From b474753b5b71c8cffa10c07108e33f202755e23a Mon Sep 17 00:00:00 2001 From: tmgordeeva Date: Tue, 10 Oct 2023 07:37:07 -0700 Subject: [PATCH 01/48] Test PR for removing replica/shard settings (#100474) Running through CI to see if we still get failures based on replica/shard settings in these tests. --- .../test/aggregate-metrics/110_field_caps.yml | 8 -------- .../test/aggregate-metrics/90_tsdb_mappings.yml | 9 --------- 2 files changed, 17 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/110_field_caps.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/110_field_caps.yml index abf367043d9c8..0c765e39656c7 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/110_field_caps.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/110_field_caps.yml @@ -9,8 +9,6 @@ setup: body: settings: index: - number_of_replicas: 0 - number_of_shards: 2 mode: time_series routing_path: [ metricset, k8s.pod.uid ] time_series: @@ -35,8 +33,6 @@ setup: body: settings: index: - number_of_replicas: 0 - number_of_shards: 2 mode: time_series routing_path: [ metricset, k8s.pod.uid ] time_series: @@ -57,10 +53,6 @@ setup: indices.create: index: test_non_time_series body: - settings: - index: - number_of_replicas: 0 - number_of_shards: 2 mappings: properties: "@timestamp": diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/90_tsdb_mappings.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/90_tsdb_mappings.yml index 05a2c640e68ef..2325a078764fc 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/90_tsdb_mappings.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/90_tsdb_mappings.yml @@ -7,10 +7,6 @@ aggregate_double_metric with time series mappings: indices.create: index: test_index body: - settings: - index: - number_of_replicas: 0 - number_of_shards: 2 mappings: properties: "@timestamp": @@ -51,10 +47,6 @@ aggregate_double_metric with wrong time series mappings: indices.create: index: tsdb_index body: - settings: - index: - number_of_replicas: 0 - number_of_shards: 2 mappings: properties: "@timestamp": @@ -95,7 +87,6 @@ aggregate_double_metric with wrong time series mappings: index: tsdb-fieldcap body: settings: - number_of_replicas: 0 mode: time_series routing_path: [field1] time_series: From 438b246709f7f06677c92e4db9f34971eda5ade1 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 10 Oct 2023 09:46:17 -0500 Subject: [PATCH 02/48] Excluding 8.10.3 from the RepositoryData BWC test (#100605) This excludes 8.10.3 from the RepositoryData BWC test, related to #100447. --- qa/repository-multi-version/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/repository-multi-version/build.gradle b/qa/repository-multi-version/build.gradle index 80d316536e09e..8398e3b8aeb1a 100644 --- a/qa/repository-multi-version/build.gradle +++ b/qa/repository-multi-version/build.gradle @@ -29,7 +29,7 @@ BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> numberOfNodes = 2 setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'xpack.security.enabled', 'false' - if (v.equals('8.10.0') || v.equals('8.10.1') || v.equals('8.10.2')) { + if (v.equals('8.10.0') || v.equals('8.10.1') || v.equals('8.10.2') || v.equals('8.10.3')) { // 8.10.x versions contain a bogus assertion that trips when reading repositories touched by newer versions // see https://github.com/elastic/elasticsearch/issues/98454 for details jvmArgs '-da' From c8ca0d1c611f228ecd282b4d9d5e7b59385930a4 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:08:13 +0100 Subject: [PATCH 03/48] ESQL Fix array Block ramBytesUsed (#100578) This commit removes the MvOrdering enum from array Block ramBytesUsed. The enum value is shared. --- .../org/elasticsearch/compute/data/BooleanArrayBlock.java | 3 +-- .../org/elasticsearch/compute/data/BytesRefArrayBlock.java | 3 +-- .../org/elasticsearch/compute/data/DoubleArrayBlock.java | 3 +-- .../org/elasticsearch/compute/data/IntArrayBlock.java | 3 +-- .../org/elasticsearch/compute/data/LongArrayBlock.java | 3 +-- .../org/elasticsearch/compute/data/X-ArrayBlock.java.st | 3 +-- .../elasticsearch/compute/data/BlockAccountingTests.java | 2 ++ .../esql/expression/function/AbstractFunctionTestCase.java | 2 +- .../expression/function/scalar/nulls/CoalesceTests.java | 7 ------- 9 files changed, 9 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java index adf1282c21fb0..9a66bf00fa71f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java @@ -93,8 +93,7 @@ public BooleanBlock expand() { public static long ramBytesEstimated(boolean[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java index f46615307f767..9e6631b6807c6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java @@ -96,8 +96,7 @@ public BytesRefBlock expand() { public static long ramBytesEstimated(BytesRefArray values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java index b0d77dd71271e..f9e1fe0c6e199 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java @@ -93,8 +93,7 @@ public DoubleBlock expand() { public static long ramBytesEstimated(double[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java index 97791a03c6044..95344bd8367c0 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java @@ -93,8 +93,7 @@ public IntBlock expand() { public static long ramBytesEstimated(int[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java index dddc5296e471e..a45abb1ed9248 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java @@ -93,8 +93,7 @@ public LongBlock expand() { public static long ramBytesEstimated(long[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st index 1f9fb93bc65c6..6a8185b43ecab 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st @@ -114,8 +114,7 @@ $endif$ public static long ramBytesEstimated($if(BytesRef)$BytesRefArray$else$$type$[]$endif$ values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java index bb1cd019273ed..c8364141d8377 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java @@ -252,6 +252,8 @@ public long accumulateObject(Object o, long shallowSize, Map fiel } else { queue.add(entry.getValue()); } + } else if (o instanceof AbstractArrayBlock && entry.getValue() instanceof Block.MvOrdering) { + // skip; MvOrdering is an enum, so instances are shared } else { queue.add(entry.getValue()); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index cdff3f0b5f2ca..3a6479215f479 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -420,7 +420,7 @@ protected void assertSimpleWithNulls(List data, Block value, int nullBlo assertTrue("argument " + nullBlock + " is null", value.isNull(0)); } - public void testEvaluateInManyThreads() throws ExecutionException, InterruptedException { + public final void testEvaluateInManyThreads() throws ExecutionException, InterruptedException { assumeTrue("nothing to do if a type error", testCase.getExpectedTypeError() == null); assumeTrue("All test data types must be representable in order to build fields", testCase.allTypesAreRepresentable()); int count = 10_000; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java index 15d37acbccfcb..8db6b1bbd0c93 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.function.Supplier; @@ -54,12 +53,6 @@ public static Iterable parameters() { return parameterSuppliersFromTypedData(builder.suppliers()); } - @Override - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100559") - public void testEvaluateInManyThreads() throws ExecutionException, InterruptedException { - super.testEvaluateInManyThreads(); - } - @Override protected void assertSimpleWithNulls(List data, Block value, int nullBlock) { for (int i = 0; i < data.size(); i++) { From 99f0b5cb995c3b2a1abaa65461e4f66c66b6ee12 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:08:50 +0100 Subject: [PATCH 04/48] Fix NPE in DocVector ramBytesUsed (#100575) This commit fixes a potential NPE in DocVector::ramBytesUsed. --- .../org/elasticsearch/compute/data/DocVector.java | 6 +++++- .../elasticsearch/compute/data/DocVectorTests.java | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java index 8abf0678593ec..44819359e8e44 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java @@ -203,6 +203,10 @@ public boolean equals(Object obj) { return shards.equals(other.shards) && segments.equals(other.segments) && docs.equals(other.docs); } + private static long ramBytesOrZero(int[] array) { + return array == null ? 0 : RamUsageEstimator.shallowSizeOf(array); + } + public static long ramBytesEstimated( IntVector shards, IntVector segments, @@ -211,7 +215,7 @@ public static long ramBytesEstimated( int[] shardSegmentDocMapBackwards ) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(shards) + RamUsageEstimator.sizeOf(segments) + RamUsageEstimator.sizeOf(docs) - + RamUsageEstimator.shallowSizeOf(shardSegmentDocMapForwards) + RamUsageEstimator.shallowSizeOf(shardSegmentDocMapBackwards); + + ramBytesOrZero(shardSegmentDocMapForwards) + ramBytesOrZero(shardSegmentDocMapBackwards); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java index 350425840a598..e2eff15fcb769 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java @@ -150,6 +150,17 @@ public void testCannotDoubleRelease() { assertThat(e.getMessage(), containsString("can't build page out of released blocks")); } + public void testRamBytesUsedWithout() { + DocVector docs = new DocVector( + IntBlock.newConstantBlockWith(0, 1).asVector(), + IntBlock.newConstantBlockWith(0, 1).asVector(), + IntBlock.newConstantBlockWith(0, 1).asVector(), + false + ); + assertThat(docs.singleSegmentNonDecreasing(), is(false)); + docs.ramBytesUsed(); // ensure non-singleSegmentNonDecreasing handles nulls in ramByteUsed + } + IntVector intRange(int startInclusive, int endExclusive) { return IntVector.range(startInclusive, endExclusive, BlockFactory.getNonBreakingInstance()); } From 07f6524bd55371744ba9c6ddc650e75dffbb53c2 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Tue, 10 Oct 2023 11:13:28 -0400 Subject: [PATCH 05/48] [ML] Refactoring inference HTTP Client to allow dynamic settings updates (#100541) * Refactoring to allow for settings updating * Allowing requests to queue * Testing getHttpClient --- .../xpack/inference/InferencePlugin.java | 49 +++-- .../inference/external/http/HttpClient.java | 55 ++---- .../external/http/HttpClientManager.java | 170 ++++++++++++++++++ .../inference/external/http/HttpSettings.java | 68 +------ .../external/http/IdleConnectionEvictor.java | 15 +- .../external/http/HttpClientManagerTests.java | 136 ++++++++++++++ .../external/http/HttpClientTests.java | 53 +++--- 7 files changed, 395 insertions(+), 151 deletions(-) create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClientManager.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientManagerTests.java diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index 25439d0bfc930..cc84a5c53c81c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -50,7 +50,7 @@ import org.elasticsearch.xpack.inference.action.TransportGetInferenceModelAction; import org.elasticsearch.xpack.inference.action.TransportInferenceAction; import org.elasticsearch.xpack.inference.action.TransportPutInferenceModelAction; -import org.elasticsearch.xpack.inference.external.http.HttpClient; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.HttpSettings; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.rest.RestDeleteInferenceModelAction; @@ -62,13 +62,16 @@ import java.util.Collection; import java.util.List; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class InferencePlugin extends Plugin implements ActionPlugin, InferenceServicePlugin, SystemIndexPlugin { public static final String NAME = "inference"; public static final String UTILITY_THREAD_POOL_NAME = "inference_utility"; + public static final String HTTP_CLIENT_SENDER_THREAD_POOL_NAME = "inference_http_client_sender"; private final Settings settings; - private final SetOnce httpClient = new SetOnce<>(); + private final SetOnce httpClientManager = new SetOnce<>(); public InferencePlugin(Settings settings) { this.settings = settings; @@ -119,8 +122,7 @@ public Collection createComponents( AllocationService allocationService, IndicesService indicesService ) { - var httpSettings = new HttpSettings(settings, clusterService); - httpClient.set(HttpClient.create(httpSettings, threadPool)); + httpClientManager.set(HttpClientManager.create(settings, threadPool, clusterService)); ModelRegistry modelRegistry = new ModelRegistry(client); return List.of(modelRegistry); @@ -154,22 +156,35 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public List> getExecutorBuilders(Settings unused) { - ScalingExecutorBuilder utility = new ScalingExecutorBuilder( - UTILITY_THREAD_POOL_NAME, - 0, - 1, - TimeValue.timeValueMinutes(10), - false, - "xpack.inference.utility_thread_pool" + public List> getExecutorBuilders(Settings settingsToUse) { + return List.of( + new ScalingExecutorBuilder( + UTILITY_THREAD_POOL_NAME, + 0, + 1, + TimeValue.timeValueMinutes(10), + false, + "xpack.inference.utility_thread_pool" + ), + /* + * This executor is specifically for enqueuing requests to be sent. The underlying + * connection pool used by the http client will block if there are no available connections to lease. + * See here for more info: https://hc.apache.org/httpcomponents-client-4.5.x/current/tutorial/html/connmgmt.html + */ + new ScalingExecutorBuilder( + HTTP_CLIENT_SENDER_THREAD_POOL_NAME, + 0, + 1, + TimeValue.timeValueMinutes(10), + false, + "xpack.inference.http_client_sender_thread_pool" + ) ); - - return List.of(utility); } @Override public List> getSettings() { - return HttpSettings.getSettings(); + return Stream.concat(HttpSettings.getSettings().stream(), HttpClientManager.getSettings().stream()).collect(Collectors.toList()); } @Override @@ -194,8 +209,8 @@ public List getInferenceServiceNamedWriteables() { @Override public void close() { - if (httpClient.get() != null) { - IOUtils.closeWhileHandlingException(httpClient.get()); + if (httpClientManager.get() != null) { + IOUtils.closeWhileHandlingException(httpClientManager.get()); } } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java index 5e3ceac875921..5622ac51ba187 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java @@ -13,9 +13,6 @@ import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; -import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; -import org.apache.http.nio.reactor.ConnectingIOReactor; -import org.apache.http.nio.reactor.IOReactorException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; @@ -29,6 +26,7 @@ import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.inference.InferencePlugin.HTTP_CLIENT_SENDER_THREAD_POOL_NAME; import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME; public class HttpClient implements Closeable { @@ -41,45 +39,19 @@ enum Status { } private final CloseableHttpAsyncClient client; - private final IdleConnectionEvictor connectionEvictor; private final AtomicReference status = new AtomicReference<>(Status.CREATED); private final ThreadPool threadPool; private final HttpSettings settings; - public static HttpClient create(HttpSettings settings, ThreadPool threadPool) { - PoolingNHttpClientConnectionManager connectionManager = createConnectionManager(); - IdleConnectionEvictor connectionEvictor = new IdleConnectionEvictor( - threadPool, - connectionManager, - settings.getEvictionInterval(), - settings.getEvictionMaxIdle() - ); + public static HttpClient create(HttpSettings settings, ThreadPool threadPool, PoolingNHttpClientConnectionManager connectionManager) { + CloseableHttpAsyncClient client = createAsyncClient(connectionManager); - int maxConnections = settings.getMaxConnections(); - CloseableHttpAsyncClient client = createAsyncClient(connectionManager, maxConnections); - - return new HttpClient(settings, client, connectionEvictor, threadPool); - } - - private static PoolingNHttpClientConnectionManager createConnectionManager() { - ConnectingIOReactor ioReactor; - try { - ioReactor = new DefaultConnectingIOReactor(); - } catch (IOReactorException e) { - var message = "Failed to initialize the inference http client"; - logger.error(message, e); - throw new ElasticsearchException(message, e); - } - - return new PoolingNHttpClientConnectionManager(ioReactor); + return new HttpClient(settings, client, threadPool); } - private static CloseableHttpAsyncClient createAsyncClient(PoolingNHttpClientConnectionManager connectionManager, int maxConnections) { + private static CloseableHttpAsyncClient createAsyncClient(PoolingNHttpClientConnectionManager connectionManager) { HttpAsyncClientBuilder clientBuilder = HttpAsyncClientBuilder.create(); - clientBuilder.setConnectionManager(connectionManager); - clientBuilder.setMaxConnPerRoute(maxConnections); - clientBuilder.setMaxConnTotal(maxConnections); // The apache client will be shared across all connections because it can be expensive to create it // so we don't want to support cookies to avoid accidental authentication for unauthorized users clientBuilder.disableCookieManagement(); @@ -88,24 +60,32 @@ private static CloseableHttpAsyncClient createAsyncClient(PoolingNHttpClientConn } // Default for testing - HttpClient(HttpSettings settings, CloseableHttpAsyncClient asyncClient, IdleConnectionEvictor evictor, ThreadPool threadPool) { + HttpClient(HttpSettings settings, CloseableHttpAsyncClient asyncClient, ThreadPool threadPool) { this.settings = settings; this.threadPool = threadPool; this.client = asyncClient; - this.connectionEvictor = evictor; } public void start() { if (status.compareAndSet(Status.CREATED, Status.STARTED)) { client.start(); - connectionEvictor.start(); } } - public void send(HttpUriRequest request, ActionListener listener) throws IOException { + public void send(HttpUriRequest request, ActionListener listener) { // The caller must call start() first before attempting to send a request assert status.get() == Status.STARTED; + threadPool.executor(HTTP_CLIENT_SENDER_THREAD_POOL_NAME).execute(() -> { + try { + doPrivilegedSend(request, listener); + } catch (IOException e) { + listener.onFailure(new ElasticsearchException(format("Failed to send request [%s]", request.getRequestLine()), e)); + } + }); + } + + private void doPrivilegedSend(HttpUriRequest request, ActionListener listener) throws IOException { SocketAccess.doPrivileged(() -> client.execute(request, new FutureCallback<>() { @Override public void completed(HttpResponse response) { @@ -144,6 +124,5 @@ private void failUsingUtilityThread(Exception exception, ActionListener MAX_CONNECTIONS = Setting.intSetting( + "xpack.inference.http.max_connections", + // TODO pick a reasonable values here + 20, + 1, + 1000, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private static final TimeValue DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME = TimeValue.timeValueSeconds(10); + static final Setting CONNECTION_EVICTION_THREAD_INTERVAL_SETTING = Setting.timeSetting( + "xpack.inference.http.connection_eviction_interval", + DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private static final TimeValue DEFAULT_CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING = DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME; + static final Setting CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING = Setting.timeSetting( + "xpack.inference.http.connection_eviction_max_idle_time", + DEFAULT_CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private final ThreadPool threadPool; + private final PoolingNHttpClientConnectionManager connectionManager; + private EvictorSettings evictorSettings; + private IdleConnectionEvictor connectionEvictor; + private final HttpClient httpClient; + + public static HttpClientManager create(Settings settings, ThreadPool threadPool, ClusterService clusterService) { + PoolingNHttpClientConnectionManager connectionManager = createConnectionManager(); + return new HttpClientManager(settings, connectionManager, threadPool, clusterService); + } + + // Default for testing + HttpClientManager( + Settings settings, + PoolingNHttpClientConnectionManager connectionManager, + ThreadPool threadPool, + ClusterService clusterService + ) { + this.threadPool = threadPool; + + this.connectionManager = connectionManager; + setMaxConnections(MAX_CONNECTIONS.get(settings)); + + this.httpClient = HttpClient.create(new HttpSettings(settings, clusterService), threadPool, connectionManager); + + evictorSettings = new EvictorSettings(settings); + connectionEvictor = createConnectionEvictor(); + + this.addSettingsUpdateConsumers(clusterService); + } + + private static PoolingNHttpClientConnectionManager createConnectionManager() { + ConnectingIOReactor ioReactor; + try { + ioReactor = new DefaultConnectingIOReactor(); + } catch (IOReactorException e) { + var message = "Failed to initialize the inference http client manager"; + logger.error(message, e); + throw new ElasticsearchException(message, e); + } + + return new PoolingNHttpClientConnectionManager(ioReactor); + } + + private void addSettingsUpdateConsumers(ClusterService clusterService) { + clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_CONNECTIONS, this::setMaxConnections); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(CONNECTION_EVICTION_THREAD_INTERVAL_SETTING, this::setEvictionInterval); + clusterService.getClusterSettings().addSettingsUpdateConsumer(CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING, this::setEvictionMaxIdle); + } + + private IdleConnectionEvictor createConnectionEvictor() { + return new IdleConnectionEvictor(threadPool, connectionManager, evictorSettings.evictionInterval, evictorSettings.evictionMaxIdle); + } + + public static List> getSettings() { + return List.of(MAX_CONNECTIONS, CONNECTION_EVICTION_THREAD_INTERVAL_SETTING, CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING); + } + + public void start() { + httpClient.start(); + connectionEvictor.start(); + } + + public HttpClient getHttpClient() { + return httpClient; + } + + @Override + public void close() throws IOException { + httpClient.close(); + connectionEvictor.stop(); + } + + private void setMaxConnections(int maxConnections) { + connectionManager.setMaxTotal(maxConnections); + connectionManager.setDefaultMaxPerRoute(maxConnections); + } + + // default for testing + void setEvictionInterval(TimeValue evictionInterval) { + evictorSettings = new EvictorSettings(evictionInterval, evictorSettings.evictionMaxIdle); + + connectionEvictor.stop(); + connectionEvictor = createConnectionEvictor(); + connectionEvictor.start(); + } + + void setEvictionMaxIdle(TimeValue evictionMaxIdle) { + evictorSettings = new EvictorSettings(evictorSettings.evictionInterval, evictionMaxIdle); + + connectionEvictor.stop(); + connectionEvictor = createConnectionEvictor(); + connectionEvictor.start(); + } + + private static class EvictorSettings { + private final TimeValue evictionInterval; + private final TimeValue evictionMaxIdle; + + EvictorSettings(Settings settings) { + this.evictionInterval = CONNECTION_EVICTION_THREAD_INTERVAL_SETTING.get(settings); + this.evictionMaxIdle = CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING.get(settings); + } + + EvictorSettings(TimeValue evictionInterval, TimeValue evictionMaxIdle) { + this.evictionInterval = evictionInterval; + this.evictionMaxIdle = evictionMaxIdle; + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java index 420f7822df06c..07d998dff956e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.core.TimeValue; import java.util.List; @@ -26,89 +25,24 @@ public class HttpSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); - static final Setting MAX_CONNECTIONS = Setting.intSetting( - "xpack.inference.http.max_connections", - 500, - 1, - // TODO pick a reasonable value here - 1000, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - private static final TimeValue DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME = TimeValue.timeValueSeconds(10); - - static final Setting CONNECTION_EVICTION_THREAD_INTERVAL_SETTING = Setting.timeSetting( - "xpack.inference.http.connection_eviction_interval", - DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - private static final TimeValue DEFAULT_CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING = DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME; - static final Setting CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING = Setting.timeSetting( - "xpack.inference.http.connection_eviction_max_idle_time", - DEFAULT_CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); private volatile ByteSizeValue maxResponseSize; - private volatile int maxConnections; - private volatile TimeValue evictionInterval; - private volatile TimeValue evictionMaxIdle; public HttpSettings(Settings settings, ClusterService clusterService) { this.maxResponseSize = MAX_HTTP_RESPONSE_SIZE.get(settings); - this.maxConnections = MAX_CONNECTIONS.get(settings); - this.evictionInterval = CONNECTION_EVICTION_THREAD_INTERVAL_SETTING.get(settings); - this.evictionMaxIdle = CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING.get(settings); clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_HTTP_RESPONSE_SIZE, this::setMaxResponseSize); - clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_CONNECTIONS, this::setMaxConnections); - clusterService.getClusterSettings() - .addSettingsUpdateConsumer(CONNECTION_EVICTION_THREAD_INTERVAL_SETTING, this::setEvictionInterval); - clusterService.getClusterSettings().addSettingsUpdateConsumer(CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING, this::setEvictionMaxIdle); } public ByteSizeValue getMaxResponseSize() { return maxResponseSize; } - public int getMaxConnections() { - return maxConnections; - } - - public TimeValue getEvictionInterval() { - return evictionInterval; - } - - public TimeValue getEvictionMaxIdle() { - return evictionMaxIdle; - } - private void setMaxResponseSize(ByteSizeValue maxResponseSize) { this.maxResponseSize = maxResponseSize; } - private void setMaxConnections(int maxConnections) { - this.maxConnections = maxConnections; - } - - private void setEvictionInterval(TimeValue evictionInterval) { - this.evictionInterval = evictionInterval; - } - - private void setEvictionMaxIdle(TimeValue evictionMaxIdle) { - this.evictionMaxIdle = evictionMaxIdle; - } - public static List> getSettings() { - return List.of( - MAX_HTTP_RESPONSE_SIZE, - MAX_CONNECTIONS, - CONNECTION_EVICTION_THREAD_INTERVAL_SETTING, - CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING - ); + return List.of(MAX_HTTP_RESPONSE_SIZE); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictor.java index 3ea0bc04848e0..f326661adc6f4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictor.java @@ -16,6 +16,7 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME; @@ -36,7 +37,7 @@ public class IdleConnectionEvictor { private final NHttpClientConnectionManager connectionManager; private final TimeValue sleepTime; private final TimeValue maxIdleTime; - private Scheduler.Cancellable cancellableTask; + private final AtomicReference cancellableTask = new AtomicReference<>(); public IdleConnectionEvictor( ThreadPool threadPool, @@ -51,13 +52,13 @@ public IdleConnectionEvictor( } public synchronized void start() { - if (cancellableTask == null) { + if (cancellableTask.get() == null) { startInternal(); } } private void startInternal() { - cancellableTask = threadPool.scheduleWithFixedDelay(() -> { + Scheduler.Cancellable task = threadPool.scheduleWithFixedDelay(() -> { try { connectionManager.closeExpiredConnections(); if (maxIdleTime != null) { @@ -67,13 +68,17 @@ private void startInternal() { logger.warn("HTTP connection eviction failed", e); } }, sleepTime, threadPool.executor(UTILITY_THREAD_POOL_NAME)); + + cancellableTask.set(task); } public void stop() { - cancellableTask.cancel(); + if (cancellableTask.get() != null) { + cancellableTask.get().cancel(); + } } public boolean isRunning() { - return cancellableTask != null && cancellableTask.isCancelled() == false; + return cancellableTask.get() != null && cancellableTask.get().isCancelled() == false; } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientManagerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientManagerTests.java new file mode 100644 index 0000000000000..a9bdee95de5fc --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientManagerTests.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http; + +import org.apache.http.HttpHeaders; +import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.http.MockResponse; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentType; +import org.junit.After; +import org.junit.Before; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.xpack.inference.external.http.HttpClientTests.createHttpPost; +import static org.elasticsearch.xpack.inference.external.http.HttpClientTests.createThreadPool; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class HttpClientManagerTests extends ESTestCase { + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + + @Before + public void init() throws Exception { + webServer.start(); + threadPool = createThreadPool(getTestName()); + } + + @After + public void shutdown() { + terminate(threadPool); + webServer.close(); + } + + public void testSend_MockServerReceivesRequest() throws Exception { + int responseCode = randomIntBetween(200, 203); + String body = randomAlphaOfLengthBetween(2, 8096); + webServer.enqueue(new MockResponse().setResponseCode(responseCode).setBody(body)); + + String paramKey = randomAlphaOfLength(3); + String paramValue = randomAlphaOfLength(3); + var httpPost = createHttpPost(webServer.getPort(), paramKey, paramValue); + + var manager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty()); + try (var httpClient = manager.getHttpClient()) { + httpClient.start(); + + PlainActionFuture listener = new PlainActionFuture<>(); + httpClient.send(httpPost, listener); + + var result = listener.actionGet(TIMEOUT); + + assertThat(result.response().getStatusLine().getStatusCode(), equalTo(responseCode)); + assertThat(new String(result.body(), StandardCharsets.UTF_8), is(body)); + assertThat(webServer.requests(), hasSize(1)); + assertThat(webServer.requests().get(0).getUri().getPath(), equalTo(httpPost.getURI().getPath())); + assertThat(webServer.requests().get(0).getUri().getQuery(), equalTo(paramKey + "=" + paramValue)); + assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType())); + } + } + + public void testStartsANewEvictor_WithNewEvictionInterval() { + var threadPool = mock(ThreadPool.class); + var manager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty()); + + var evictionInterval = TimeValue.timeValueSeconds(1); + manager.setEvictionInterval(evictionInterval); + verify(threadPool).scheduleWithFixedDelay(any(Runnable.class), eq(evictionInterval), any()); + } + + public void testStartsANewEvictor_WithNewEvictionMaxIdle() throws InterruptedException { + var mockConnectionManager = mock(PoolingNHttpClientConnectionManager.class); + + Settings settings = Settings.builder() + .put(HttpClientManager.CONNECTION_EVICTION_THREAD_INTERVAL_SETTING.getKey(), TimeValue.timeValueNanos(1)) + .build(); + var manager = new HttpClientManager(settings, mockConnectionManager, threadPool, mockClusterService(settings)); + + CountDownLatch runLatch = new CountDownLatch(1); + doAnswer(invocation -> { + manager.close(); + runLatch.countDown(); + return Void.TYPE; + }).when(mockConnectionManager).closeIdleConnections(anyLong(), any()); + + var evictionMaxIdle = TimeValue.timeValueSeconds(1); + manager.setEvictionMaxIdle(evictionMaxIdle); + runLatch.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); + + verify(mockConnectionManager, times(1)).closeIdleConnections(eq(evictionMaxIdle.millis()), eq(TimeUnit.MILLISECONDS)); + } + + private static ClusterService mockClusterServiceEmpty() { + return mockClusterService(Settings.EMPTY); + } + + private static ClusterService mockClusterService(Settings settings) { + var clusterService = mock(ClusterService.class); + + var registeredSettings = Stream.concat(HttpClientManager.getSettings().stream(), HttpSettings.getSettings().stream()) + .collect(Collectors.toSet()); + + var cSettings = new ClusterSettings(settings, registeredSettings); + when(clusterService.getClusterSettings()).thenReturn(cSettings); + + return clusterService; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java index 42c8422af3982..b0b0a34aabf97 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java @@ -45,6 +45,7 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.inference.InferencePlugin.HTTP_CLIENT_SENDER_THREAD_POOL_NAME; import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -64,17 +65,7 @@ public class HttpClientTests extends ESTestCase { @Before public void init() throws Exception { webServer.start(); - threadPool = new TestThreadPool( - getTestName(), - new ScalingExecutorBuilder( - UTILITY_THREAD_POOL_NAME, - 1, - 4, - TimeValue.timeValueMinutes(10), - false, - "xpack.inference.utility_thread_pool" - ) - ); + threadPool = createThreadPool(getTestName()); } @After @@ -92,7 +83,7 @@ public void testSend_MockServerReceivesRequest() throws Exception { String paramValue = randomAlphaOfLength(3); var httpPost = createHttpPost(webServer.getPort(), paramKey, paramValue); - try (var httpClient = HttpClient.create(emptyHttpSettings(), threadPool)) { + try (var httpClient = HttpClient.create(emptyHttpSettings(), threadPool, createConnectionManager())) { httpClient.start(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -119,10 +110,9 @@ public void testSend_FailedCallsOnFailure() throws Exception { return mock(Future.class); }).when(asyncClient).execute(any(), any()); - var evictor = createEvictor(threadPool); var httpPost = createHttpPost(webServer.getPort(), "a", "b"); - try (var client = new HttpClient(emptyHttpSettings(), asyncClient, evictor, threadPool)) { + try (var client = new HttpClient(emptyHttpSettings(), asyncClient, threadPool)) { client.start(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -143,10 +133,9 @@ public void testSend_CancelledCallsOnFailure() throws Exception { return mock(Future.class); }).when(asyncClient).execute(any(), any()); - var evictor = createEvictor(threadPool); var httpPost = createHttpPost(webServer.getPort(), "a", "b"); - try (var client = new HttpClient(emptyHttpSettings(), asyncClient, evictor, threadPool)) { + try (var client = new HttpClient(emptyHttpSettings(), asyncClient, threadPool)) { client.start(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -162,10 +151,9 @@ public void testStart_MultipleCallsOnlyStartTheClientOnce() throws Exception { var asyncClient = mock(CloseableHttpAsyncClient.class); when(asyncClient.execute(any(), any())).thenReturn(mock(Future.class)); - var evictor = createEvictor(threadPool); var httpPost = createHttpPost(webServer.getPort(), "a", "b"); - try (var client = new HttpClient(emptyHttpSettings(), asyncClient, evictor, threadPool)) { + try (var client = new HttpClient(emptyHttpSettings(), asyncClient, threadPool)) { client.start(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -188,7 +176,7 @@ public void testSend_FailsWhenMaxBytesReadIsExceeded() throws Exception { Settings settings = Settings.builder().put(HttpSettings.MAX_HTTP_RESPONSE_SIZE.getKey(), ByteSizeValue.ONE).build(); var httpSettings = createHttpSettings(settings); - try (var httpClient = HttpClient.create(httpSettings, threadPool)) { + try (var httpClient = HttpClient.create(httpSettings, threadPool, createConnectionManager())) { httpClient.start(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -199,7 +187,7 @@ public void testSend_FailsWhenMaxBytesReadIsExceeded() throws Exception { } } - private static HttpPost createHttpPost(int port, String paramKey, String paramValue) throws URISyntaxException { + public static HttpPost createHttpPost(int port, String paramKey, String paramValue) throws URISyntaxException { URI uri = new URIBuilder().setScheme("http") .setHost("localhost") .setPort(port) @@ -219,16 +207,33 @@ private static HttpPost createHttpPost(int port, String paramKey, String paramVa return httpPost; } - private static IdleConnectionEvictor createEvictor(ThreadPool threadPool) throws IOReactorException { - var manager = createConnectionManager(); - return new IdleConnectionEvictor(threadPool, manager, new TimeValue(10, TimeUnit.SECONDS), new TimeValue(10, TimeUnit.SECONDS)); + public static ThreadPool createThreadPool(String name) { + return new TestThreadPool( + name, + new ScalingExecutorBuilder( + UTILITY_THREAD_POOL_NAME, + 1, + 4, + TimeValue.timeValueMinutes(10), + false, + "xpack.inference.utility_thread_pool" + ), + new ScalingExecutorBuilder( + HTTP_CLIENT_SENDER_THREAD_POOL_NAME, + 1, + 4, + TimeValue.timeValueMinutes(10), + false, + "xpack.inference.utility_thread_pool" + ) + ); } private static PoolingNHttpClientConnectionManager createConnectionManager() throws IOReactorException { return new PoolingNHttpClientConnectionManager(new DefaultConnectingIOReactor()); } - private static HttpSettings emptyHttpSettings() { + public static HttpSettings emptyHttpSettings() { return createHttpSettings(Settings.EMPTY); } From 77b4fd12bfc9dceb5e1febc570f1dd57eed43e6a Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Tue, 10 Oct 2023 17:16:04 +0200 Subject: [PATCH 06/48] Log aws request metrics (#100272) This change logs amount of requests, exceptions and throttles from aws s3 api aggregated over a tumbling window. --- .../repositories/s3/S3BlobStore.java | 21 ++++- .../repositories/s3/S3RequestRetryStats.java | 87 +++++++++++++++++++ 2 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RequestRetryStats.java diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index f371d6f354763..3ff0497b42719 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -82,6 +82,10 @@ class S3BlobStore implements BlobStore { private final StatsCollectors statsCollectors = new StatsCollectors(); + private static final TimeValue RETRY_STATS_WINDOW = TimeValue.timeValueMinutes(5); + + private volatile S3RequestRetryStats s3RequestRetryStats; + S3BlobStore( S3Service service, String bucket, @@ -105,10 +109,23 @@ class S3BlobStore implements BlobStore { this.threadPool = threadPool; this.snapshotExecutor = threadPool.executor(ThreadPool.Names.SNAPSHOT); this.meter = meter; + s3RequestRetryStats = new S3RequestRetryStats(getMaxRetries()); + threadPool.scheduleWithFixedDelay(() -> { + var priorRetryStats = s3RequestRetryStats; + s3RequestRetryStats = new S3RequestRetryStats(getMaxRetries()); + priorRetryStats.emitMetrics(); + }, RETRY_STATS_WINDOW, threadPool.generic()); } RequestMetricCollector getMetricCollector(Operation operation, OperationPurpose purpose) { - return statsCollectors.getMetricCollector(operation, purpose); + var collector = statsCollectors.getMetricCollector(operation, purpose); + return new RequestMetricCollector() { + @Override + public void collectMetrics(Request request, Response response) { + s3RequestRetryStats.addRequest(request); + collector.collectMetrics(request, response); + } + }; } public Executor getSnapshotExecutor() { @@ -178,7 +195,7 @@ public AmazonS3Reference clientReference() { return service.client(repositoryMetadata); } - int getMaxRetries() { + final int getMaxRetries() { return service.settings(repositoryMetadata).maxRetries; } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RequestRetryStats.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RequestRetryStats.java new file mode 100644 index 0000000000000..952668f370161 --- /dev/null +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RequestRetryStats.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.repositories.s3; + +import com.amazonaws.Request; +import com.amazonaws.util.AWSRequestMetrics; +import com.amazonaws.util.TimingInfo; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.ESLogMessage; +import org.elasticsearch.common.util.Maps; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongArray; + +/** + * This class emit aws s3 metrics as logs until we have a proper apm integration + */ +public class S3RequestRetryStats { + + private static final Logger logger = LogManager.getLogger(S3RequestRetryStats.class); + + private final AtomicLong requests = new AtomicLong(); + private final AtomicLong exceptions = new AtomicLong(); + private final AtomicLong throttles = new AtomicLong(); + private final AtomicLongArray exceptionsHistogram; + private final AtomicLongArray throttlesHistogram; + + public S3RequestRetryStats(int maxRetries) { + this.exceptionsHistogram = new AtomicLongArray(maxRetries + 1); + this.throttlesHistogram = new AtomicLongArray(maxRetries + 1); + } + + public void addRequest(Request request) { + if (request == null) { + return; + } + var info = request.getAWSRequestMetrics().getTimingInfo(); + long requests = getCounter(info, AWSRequestMetrics.Field.RequestCount); + long exceptions = getCounter(info, AWSRequestMetrics.Field.Exception); + long throttles = getCounter(info, AWSRequestMetrics.Field.ThrottleException); + + this.requests.addAndGet(requests); + this.exceptions.addAndGet(exceptions); + this.throttles.addAndGet(throttles); + if (exceptions >= 0 && exceptions < this.exceptionsHistogram.length()) { + this.exceptionsHistogram.incrementAndGet((int) exceptions); + } + if (throttles >= 0 && throttles < this.throttlesHistogram.length()) { + this.throttlesHistogram.incrementAndGet((int) throttles); + } + } + + private static long getCounter(TimingInfo info, AWSRequestMetrics.Field field) { + var counter = info.getCounter(field.name()); + return counter != null ? counter.longValue() : 0L; + } + + public void emitMetrics() { + if (logger.isDebugEnabled()) { + var metrics = Maps.newMapWithExpectedSize(3); + metrics.put("elasticsearch.metrics.s3.requests", requests.get()); + metrics.put("elasticsearch.metrics.s3.exceptions", exceptions.get()); + metrics.put("elasticsearch.metrics.s3.throttles", throttles.get()); + for (int i = 0; i < exceptionsHistogram.length(); i++) { + long exceptions = exceptionsHistogram.get(i); + if (exceptions != 0) { + metrics.put("elasticsearch.metrics.s3.exceptions.h" + i, exceptions); + } + } + for (int i = 0; i < throttlesHistogram.length(); i++) { + long throttles = throttlesHistogram.get(i); + if (throttles != 0) { + metrics.put("elasticsearch.metrics.s3.throttles.h" + i, throttles); + } + } + logger.debug(new ESLogMessage().withFields(metrics)); + } + } +} From d9234149982b8fd31fbf6d91840ae0097c4cc7ca Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Tue, 10 Oct 2023 17:18:48 +0200 Subject: [PATCH 07/48] added privileges to write metrics-fleet_server* (#100574) --- .../authz/store/KibanaOwnedReservedRoleDescriptors.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index f6046cd41f25c..579638f474b21 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -193,6 +193,8 @@ static RoleDescriptor kibanaSystem(String name) { .build(), // Fleet telemetry queries Agent Logs indices in kibana task runner RoleDescriptor.IndicesPrivileges.builder().indices("logs-elastic_agent*").privileges("read").build(), + // Fleet publishes Agent metrics in kibana task runner + RoleDescriptor.IndicesPrivileges.builder().indices("metrics-fleet_server*").privileges("auto_configure", "write").build(), // Legacy "Alerts as data" used in Security Solution. // Kibana user creates these indices; reads / writes to them. RoleDescriptor.IndicesPrivileges.builder().indices(ReservedRolesStore.ALERTS_LEGACY_INDEX).privileges("all").build(), From 94d7351b48032faafddc5b4fdf22554f961a88a3 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:21:17 +0100 Subject: [PATCH 08/48] Re-enable org.elasticsearch.xpack.esql.action.EsqlActionIT.testFilterWithNullAndEvalFromIndex (#100604) This commit re-enables org.elasticsearch.xpack.esql.action.EsqlActionIT.testFilterWithNullAndEvalFromIndex, which now passes successfully. --- .../java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index fd4fe13b9c1b1..f10ca17d741d8 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -574,7 +574,6 @@ public void testStringLength() { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99826") public void testFilterWithNullAndEvalFromIndex() { // append entry, with an absent count, to the index client().prepareBulk().add(new IndexRequest("test").id("no_count").source("data", 12, "data_d", 2d, "color", "red")).get(); From a3ba9f9888c97743c414e10acdfb4879880900cf Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Tue, 10 Oct 2023 08:31:03 -0700 Subject: [PATCH 09/48] Update 8.10.3.asciidoc (#100590) (#100614) * Update 8.10.3.asciidoc * Update docs/reference/release-notes/8.10.3.asciidoc Co-authored-by: David Turner * Update docs/reference/release-notes/8.10.3.asciidoc --------- Co-authored-by: David Turner (cherry picked from commit 5ba96f2e2e766324a379fe1c93fff05c056211b3) --- docs/reference/release-notes/8.10.3.asciidoc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/reference/release-notes/8.10.3.asciidoc b/docs/reference/release-notes/8.10.3.asciidoc index a09beb26b4d27..b7828f52ad082 100644 --- a/docs/reference/release-notes/8.10.3.asciidoc +++ b/docs/reference/release-notes/8.10.3.asciidoc @@ -1,7 +1,11 @@ [[release-notes-8.10.3]] == {es} version 8.10.3 -coming[8.10.3] +[[known-issues-8.10.3]] +[float] +=== Known issues + +include::8.10.0.asciidoc[tag=repositorydata-format-change] Also see <>. From 4c96f9358b4ddea87637d34e25f345d821ec783d Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Tue, 10 Oct 2023 16:57:33 +0100 Subject: [PATCH 10/48] [ML] Unmute now fixed JobUpdateTest/testMergeWithJob (#100611) Closes #98626 --- .../elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 543360fc24d89..09ff29f768dce 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -258,7 +258,6 @@ protected JobUpdate doParseInstance(XContentParser parser) { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98626") public void testMergeWithJob() { List detectorUpdates = new ArrayList<>(); List detectionRules1 = Collections.singletonList( From 9ef5d301c299521bdec62226a639c7c1c296242a Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Tue, 10 Oct 2023 12:11:15 -0400 Subject: [PATCH 11/48] Revert "[CI] Mute MlHiddenIndicesFullClusterRestartIT.testMlIndicesBecomeHidden" (#100618) This reverts commit 725da76b70cad8e864bd150ac76f08d1f3312a8a. --- .../xpack/restart/MlHiddenIndicesFullClusterRestartIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 79a2be51197e6..aeb3dad547946 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -73,7 +73,6 @@ public void waitForMlTemplates() throws Exception { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/93521") public void testMlIndicesBecomeHidden() throws Exception { if (isRunningAgainstOldCluster()) { // trigger ML indices creation From 7ba224ea62838676d17e9b48e910119a7fc644d4 Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Tue, 10 Oct 2023 12:49:39 -0400 Subject: [PATCH 12/48] Bump versions after 7.17.14 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 16 ++++++++++++++++ .buildkite/pipelines/periodic.yml | 10 ++++++++++ .ci/bwcVersions | 1 + .ci/snapshotBwcVersions | 2 +- .../src/main/java/org/elasticsearch/Version.java | 1 + 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 6f0657c3d5e8e..4cc59424db736 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -40,7 +40,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.14", "8.10.3", "8.11.0", "8.12.0"] + BWC_VERSION: ["7.17.15", "8.10.3", "8.11.0", "8.12.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 844570d945fdf..cf8b35cb941e4 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -1056,6 +1056,22 @@ steps: env: BWC_VERSION: 7.17.14 + - label: "{{matrix.image}} / 7.17.15 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.15 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 7.17.15 + - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0 timeout_in_minutes: 300 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 8e959b07a9bc1..846cd3176593b 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -642,6 +642,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 7.17.14 + - label: 7.17.15 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.15#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 7.17.15 - label: 8.0.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest timeout_in_minutes: 300 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 536a6cbf2a3b2..29942fe6032ad 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -63,6 +63,7 @@ BWC_VERSION: - "7.17.12" - "7.17.13" - "7.17.14" + - "7.17.15" - "8.0.0" - "8.0.1" - "8.1.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 7ad88baffac95..6b586a1d49f6c 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,5 @@ BWC_VERSION: - - "7.17.14" + - "7.17.15" - "8.10.3" - "8.11.0" - "8.12.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 4d578e77e56bc..329e6eacb0bbe 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -114,6 +114,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_7_17_12 = new Version(7_17_12_99); public static final Version V_7_17_13 = new Version(7_17_13_99); public static final Version V_7_17_14 = new Version(7_17_14_99); + public static final Version V_7_17_15 = new Version(7_17_15_99); public static final Version V_8_0_0 = new Version(8_00_00_99); public static final Version V_8_0_1 = new Version(8_00_01_99); public static final Version V_8_1_0 = new Version(8_01_00_99); From 0ccbf44865eb98f1f1265b31c08ecd306460f264 Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Tue, 10 Oct 2023 12:53:11 -0400 Subject: [PATCH 13/48] Prune changelogs after 7.17.14 release --- docs/changelog/100106.yaml | 5 ----- docs/changelog/100134.yaml | 5 ----- docs/changelog/100179.yaml | 6 ------ docs/changelog/100207.yaml | 5 ----- docs/changelog/100284.yaml | 5 ----- docs/changelog/99231.yaml | 5 ----- docs/changelog/99604.yaml | 5 ----- docs/changelog/99660.yaml | 5 ----- docs/changelog/99673.yaml | 5 ----- docs/changelog/99677.yaml | 5 ----- docs/changelog/99724.yaml | 5 ----- docs/changelog/99738.yaml | 6 ------ docs/changelog/99803.yaml | 5 ----- docs/changelog/99814.yaml | 6 ------ docs/changelog/99818.yaml | 6 ------ docs/changelog/99846.yaml | 5 ----- docs/changelog/99868.yaml | 6 ------ docs/changelog/99892.yaml | 6 ------ docs/changelog/99914.yaml | 5 ----- docs/changelog/99946.yaml | 5 ----- 20 files changed, 106 deletions(-) delete mode 100644 docs/changelog/100106.yaml delete mode 100644 docs/changelog/100134.yaml delete mode 100644 docs/changelog/100179.yaml delete mode 100644 docs/changelog/100207.yaml delete mode 100644 docs/changelog/100284.yaml delete mode 100644 docs/changelog/99231.yaml delete mode 100644 docs/changelog/99604.yaml delete mode 100644 docs/changelog/99660.yaml delete mode 100644 docs/changelog/99673.yaml delete mode 100644 docs/changelog/99677.yaml delete mode 100644 docs/changelog/99724.yaml delete mode 100644 docs/changelog/99738.yaml delete mode 100644 docs/changelog/99803.yaml delete mode 100644 docs/changelog/99814.yaml delete mode 100644 docs/changelog/99818.yaml delete mode 100644 docs/changelog/99846.yaml delete mode 100644 docs/changelog/99868.yaml delete mode 100644 docs/changelog/99892.yaml delete mode 100644 docs/changelog/99914.yaml delete mode 100644 docs/changelog/99946.yaml diff --git a/docs/changelog/100106.yaml b/docs/changelog/100106.yaml deleted file mode 100644 index c3e3d50d2597a..0000000000000 --- a/docs/changelog/100106.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100106 -summary: Validate enrich index before completing policy execution -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/100134.yaml b/docs/changelog/100134.yaml deleted file mode 100644 index 3836ec2793050..0000000000000 --- a/docs/changelog/100134.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100134 -summary: Implement matches() on `SourceConfirmedTextQuery` -area: Highlighting -type: enhancement -issues: [] diff --git a/docs/changelog/100179.yaml b/docs/changelog/100179.yaml deleted file mode 100644 index 2b7824c1575e6..0000000000000 --- a/docs/changelog/100179.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 100179 -summary: ILM introduce the `check-ts-end-time-passed` step -area: ILM+SLM -type: bug -issues: - - 99696 diff --git a/docs/changelog/100207.yaml b/docs/changelog/100207.yaml deleted file mode 100644 index 10e55992f0e45..0000000000000 --- a/docs/changelog/100207.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100207 -summary: ILM the delete action waits for a TSDS index time/bounds to lapse -area: ILM+SLM -type: bug -issues: [] diff --git a/docs/changelog/100284.yaml b/docs/changelog/100284.yaml deleted file mode 100644 index 956fc472d6656..0000000000000 --- a/docs/changelog/100284.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100284 -summary: Defend against negative datafeed start times -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/99231.yaml b/docs/changelog/99231.yaml deleted file mode 100644 index 9f5dfa1137587..0000000000000 --- a/docs/changelog/99231.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99231 -summary: Add manage permission for fleet managed threat intel indices -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/99604.yaml b/docs/changelog/99604.yaml deleted file mode 100644 index 0bace7aef1b26..0000000000000 --- a/docs/changelog/99604.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99604 -summary: Show a concrete error when the enrich index does not exist rather than a NullPointerException -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/99660.yaml b/docs/changelog/99660.yaml deleted file mode 100644 index ea19e24d51fff..0000000000000 --- a/docs/changelog/99660.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99660 -summary: Close expired search contexts on SEARCH thread -area: Search -type: bug -issues: [] diff --git a/docs/changelog/99673.yaml b/docs/changelog/99673.yaml deleted file mode 100644 index b48d620b21f49..0000000000000 --- a/docs/changelog/99673.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99673 -summary: Adding retry logic for start model deployment API -area: Machine Learning -type: bug -issues: [ ] diff --git a/docs/changelog/99677.yaml b/docs/changelog/99677.yaml deleted file mode 100644 index 04c1c28cf2e12..0000000000000 --- a/docs/changelog/99677.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99677 -summary: Using 1 MB chunks for elser model storage -area: Machine Learning -type: bug -issues: [ ] diff --git a/docs/changelog/99724.yaml b/docs/changelog/99724.yaml deleted file mode 100644 index 4fe78687bf72b..0000000000000 --- a/docs/changelog/99724.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99724 -summary: Upgrade bundled JDK to Java 21 -area: Packaging -type: upgrade -issues: [] diff --git a/docs/changelog/99738.yaml b/docs/changelog/99738.yaml deleted file mode 100644 index 1b65926aed741..0000000000000 --- a/docs/changelog/99738.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99738 -summary: Ignore "index not found" error when `delete_dest_index` flag is set but the - dest index doesn't exist -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/99803.yaml b/docs/changelog/99803.yaml deleted file mode 100644 index ce0929eb20e07..0000000000000 --- a/docs/changelog/99803.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99803 -summary: Do not use PIT in the presence of remote indices in source -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/99814.yaml b/docs/changelog/99814.yaml deleted file mode 100644 index 1632be42b4e4c..0000000000000 --- a/docs/changelog/99814.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99814 -summary: Fix cardinality agg for `const_keyword` -area: Aggregations -type: bug -issues: - - 99776 diff --git a/docs/changelog/99818.yaml b/docs/changelog/99818.yaml deleted file mode 100644 index 8835bcf28e050..0000000000000 --- a/docs/changelog/99818.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99818 -summary: Add checks in term and terms queries that input terms are not too long -area: Search -type: enhancement -issues: - - 99802 diff --git a/docs/changelog/99846.yaml b/docs/changelog/99846.yaml deleted file mode 100644 index 198b0b6f939ac..0000000000000 --- a/docs/changelog/99846.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99846 -summary: Update version range in `jvm.options` for the Panama Vector API -area: Vector Search -type: bug -issues: [] diff --git a/docs/changelog/99868.yaml b/docs/changelog/99868.yaml deleted file mode 100644 index 33d582f9ebd0a..0000000000000 --- a/docs/changelog/99868.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99868 -summary: Fix fields API for `geo_point` fields inside other arrays -area: Search -type: bug -issues: - - 99781 diff --git a/docs/changelog/99892.yaml b/docs/changelog/99892.yaml deleted file mode 100644 index 5090d1d888b65..0000000000000 --- a/docs/changelog/99892.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99892 -summary: Support $ and / in restore rename replacements -area: Snapshot/Restore -type: bug -issues: - - 99078 diff --git a/docs/changelog/99914.yaml b/docs/changelog/99914.yaml deleted file mode 100644 index 8b0026a8ff9ca..0000000000000 --- a/docs/changelog/99914.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99914 -summary: Let `_stats` internally timeout if checkpoint information can not be retrieved -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/99946.yaml b/docs/changelog/99946.yaml deleted file mode 100644 index 11dc4090baa0e..0000000000000 --- a/docs/changelog/99946.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99946 -summary: Skip settings validation during desired nodes updates -area: Distributed -type: bug -issues: [] From e723c7aacdb6d466bde71f62b267c99259072ec3 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Tue, 10 Oct 2023 19:33:30 +0200 Subject: [PATCH 14/48] Switch visibility to public in ESQL REST spec (#100622) This update the visibility field in ESQL's REST spec to public. It also updates the types of quotes used for one the REST object parameter to backticks, for consistency. --- .../src/main/resources/rest-api-spec/api/esql.query.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json index ffcd30fa6c717..c038ac4f3b749 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json @@ -5,7 +5,7 @@ "description":"Executes an ESQL request" }, "stability":"experimental", - "visibility":"private", + "visibility":"public", "headers":{ "accept": [ "application/json"], "content_type": ["application/json"] @@ -32,7 +32,7 @@ } }, "body":{ - "description":"Use the `query` element to start a query. Use `time_zone` to specify an execution time zone and 'columnar' to format the answer.", + "description":"Use the `query` element to start a query. Use `time_zone` to specify an execution time zone and `columnar` to format the answer.", "required":true } } From b280a63eb70a040db7006665c67404c8a1ffdfde Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 10 Oct 2023 14:12:36 -0400 Subject: [PATCH 15/48] [CI] Disable jenkins platform-support jobs, and re-enable all Buildkite periodic pipelines (#100630) --- .buildkite/scripts/periodic.trigger.sh | 34 +++++++------------ ...icsearch+multijob+platform-support-arm.yml | 15 ++++---- ...csearch+multijob+platform-support-unix.yml | 5 +-- ...arch+multijob+platform-support-windows.yml | 15 ++++---- ...arch+periodic+platform-support-trigger.yml | 6 ---- ...lasticsearch+periodic+platform-support.yml | 3 +- 6 files changed, 33 insertions(+), 45 deletions(-) delete mode 100644 .ci/jobs.t/elastic+elasticsearch+periodic+platform-support-trigger.yml diff --git a/.buildkite/scripts/periodic.trigger.sh b/.buildkite/scripts/periodic.trigger.sh index 754c701927185..3571d112c5b6d 100755 --- a/.buildkite/scripts/periodic.trigger.sh +++ b/.buildkite/scripts/periodic.trigger.sh @@ -12,6 +12,18 @@ for BRANCH in "${BRANCHES[@]}"; do LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit') cat < Date: Tue, 10 Oct 2023 13:10:38 -0500 Subject: [PATCH 16/48] Bump versions after 8.10.3 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 16 ++++++++++++++++ .buildkite/pipelines/periodic.yml | 10 ++++++++++ .ci/bwcVersions | 1 + .ci/snapshotBwcVersions | 2 +- .../src/main/java/org/elasticsearch/Version.java | 1 + 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 4cc59424db736..06b9f1dfbb6bf 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -40,7 +40,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.15", "8.10.3", "8.11.0", "8.12.0"] + BWC_VERSION: ["7.17.15", "8.10.4", "8.11.0", "8.12.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index cf8b35cb941e4..a265f7cd0cd4c 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -1664,6 +1664,22 @@ steps: env: BWC_VERSION: 8.10.3 + - label: "{{matrix.image}} / 8.10.4 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.4 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.10.4 + - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0 timeout_in_minutes: 300 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 846cd3176593b..8143110607da2 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -1022,6 +1022,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 8.10.3 + - label: 8.10.4 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.4#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.10.4 - label: 8.11.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest timeout_in_minutes: 300 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 29942fe6032ad..0a17a776e44a2 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -101,5 +101,6 @@ BWC_VERSION: - "8.10.1" - "8.10.2" - "8.10.3" + - "8.10.4" - "8.11.0" - "8.12.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 6b586a1d49f6c..a27c1532720fa 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,5 @@ BWC_VERSION: - "7.17.15" - - "8.10.3" + - "8.10.4" - "8.11.0" - "8.12.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 329e6eacb0bbe..69eaf17addb88 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -152,6 +152,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_10_1 = new Version(8_10_01_99); public static final Version V_8_10_2 = new Version(8_10_02_99); public static final Version V_8_10_3 = new Version(8_10_03_99); + public static final Version V_8_10_4 = new Version(8_10_04_99); public static final Version V_8_11_0 = new Version(8_11_00_99); public static final Version V_8_12_0 = new Version(8_12_00_99); public static final Version CURRENT = V_8_12_0; From 4b0a6cd59895817f45c5e67dd9cf363fadd6952f Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Tue, 10 Oct 2023 11:24:54 -0700 Subject: [PATCH 17/48] Capture JVM crash dump logs in uploaded artifact bundle (#100627) --- .../src/main/groovy/elasticsearch.build-complete.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-complete.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-complete.gradle index 7cb66be729374..f80cfa566a518 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-complete.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-complete.gradle @@ -28,6 +28,7 @@ if (buildNumber && performanceTest == null && GradleUtils.isIncludedBuild(projec include("**/build/test-results/**/*.xml") include("**/build/testclusters/**") include("**/build/testrun/*/temp/**") + include("**/build/**/hs_err_pid*.log") exclude("**/build/testclusters/**/data/**") exclude("**/build/testclusters/**/distro/**") exclude("**/build/testclusters/**/repo/**") From 3465a2bf183d157397aaea2772c59d2e28dbbe45 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Tue, 10 Oct 2023 20:28:02 +0200 Subject: [PATCH 18/48] Fix metric gauge creation model (#100609) OTEL gauges should follow the callback model otherwise they will not be sent by apm java agent. (or use BatchCallback) This commit changes the gagues creation model to return Observable*Gauge and uses AtomicLong/Double to store current value which will be polled when metrics are exported (and callback is called) --- docs/changelog/100609.yaml | 5 + .../internal/metrics/DoubleGaugeAdapter.java | 25 +++- .../internal/metrics/LongGaugeAdapter.java | 20 ++- .../internal/metrics/GaugeAdapterTests.java | 123 ++++++++++++++++++ 4 files changed, 162 insertions(+), 11 deletions(-) create mode 100644 docs/changelog/100609.yaml create mode 100644 modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/GaugeAdapterTests.java diff --git a/docs/changelog/100609.yaml b/docs/changelog/100609.yaml new file mode 100644 index 0000000000000..c1c63c1af5d4d --- /dev/null +++ b/docs/changelog/100609.yaml @@ -0,0 +1,5 @@ +pr: 100609 +summary: Fix metric gauge creation model +area: Infra/Core +type: bug +issues: [] diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java index 9d55d475d4a93..54f33be21698b 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java @@ -10,33 +10,46 @@ import io.opentelemetry.api.metrics.Meter; +import java.util.Collections; import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; /** * DoubleGaugeAdapter wraps an otel ObservableDoubleMeasurement */ -public class DoubleGaugeAdapter extends AbstractInstrument +public class DoubleGaugeAdapter extends AbstractInstrument implements org.elasticsearch.telemetry.metric.DoubleGauge { + private final AtomicReference valueWithAttributes; + public DoubleGaugeAdapter(Meter meter, String name, String description, String unit) { super(meter, name, description, unit); + this.valueWithAttributes = new AtomicReference<>(new ValueWithAttributes(0.0, Collections.emptyMap())); } @Override - io.opentelemetry.api.metrics.ObservableDoubleMeasurement buildInstrument(Meter meter) { - var builder = Objects.requireNonNull(meter).gaugeBuilder(getName()); - return builder.setDescription(getDescription()).setUnit(getUnit()).buildObserver(); + io.opentelemetry.api.metrics.ObservableDoubleGauge buildInstrument(Meter meter) { + return Objects.requireNonNull(meter) + .gaugeBuilder(getName()) + .setDescription(getDescription()) + .setUnit(getUnit()) + .buildWithCallback(measurement -> { + var localValueWithAttributed = valueWithAttributes.get(); + measurement.record(localValueWithAttributed.value(), OtelHelper.fromMap(localValueWithAttributed.attributes())); + }); } @Override public void record(double value) { - getInstrument().record(value); + record(value, Collections.emptyMap()); } @Override public void record(double value, Map attributes) { - getInstrument().record(value, OtelHelper.fromMap(attributes)); + this.valueWithAttributes.set(new ValueWithAttributes(value, attributes)); } + + private record ValueWithAttributes(double value, Map attributes) {} } diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java index 48430285a5173..66d2287a765dc 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java @@ -10,37 +10,47 @@ import io.opentelemetry.api.metrics.Meter; +import java.util.Collections; import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; /** * LongGaugeAdapter wraps an otel ObservableLongMeasurement */ -public class LongGaugeAdapter extends AbstractInstrument +public class LongGaugeAdapter extends AbstractInstrument implements org.elasticsearch.telemetry.metric.LongGauge { + private final AtomicReference valueWithAttributes; public LongGaugeAdapter(Meter meter, String name, String description, String unit) { super(meter, name, description, unit); + this.valueWithAttributes = new AtomicReference<>(new ValueWithAttributes(0L, Collections.emptyMap())); } @Override - io.opentelemetry.api.metrics.ObservableLongMeasurement buildInstrument(Meter meter) { + io.opentelemetry.api.metrics.ObservableLongGauge buildInstrument(Meter meter) { + return Objects.requireNonNull(meter) .gaugeBuilder(getName()) .ofLongs() .setDescription(getDescription()) .setUnit(getUnit()) - .buildObserver(); + .buildWithCallback(measurement -> { + var localValueWithAttributed = valueWithAttributes.get(); + measurement.record(localValueWithAttributed.value(), OtelHelper.fromMap(localValueWithAttributed.attributes())); + }); } @Override public void record(long value) { - getInstrument().record(value); + record(value, Collections.emptyMap()); } @Override public void record(long value, Map attributes) { - getInstrument().record(value, OtelHelper.fromMap(attributes)); + this.valueWithAttributes.set(new ValueWithAttributes(value, attributes)); } + + private record ValueWithAttributes(long value, Map attributes) {} } diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/GaugeAdapterTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/GaugeAdapterTests.java new file mode 100644 index 0000000000000..1e230eefe32dc --- /dev/null +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/GaugeAdapterTests.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.telemetry.apm.internal.metrics; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleGaugeBuilder; +import io.opentelemetry.api.metrics.LongGaugeBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.ObservableDoubleMeasurement; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; + +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.util.Map; +import java.util.function.Consumer; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class GaugeAdapterTests extends ESTestCase { + Meter testMeter = Mockito.mock(Meter.class); + LongGaugeBuilder longGaugeBuilder = Mockito.mock(LongGaugeBuilder.class); + DoubleGaugeBuilder mockDoubleGaugeBuilder = Mockito.mock(DoubleGaugeBuilder.class); + + @Before + public void init() { + when(longGaugeBuilder.setDescription(Mockito.anyString())).thenReturn(longGaugeBuilder); + when(longGaugeBuilder.setUnit(Mockito.anyString())).thenReturn(longGaugeBuilder); + + + when(mockDoubleGaugeBuilder.ofLongs()).thenReturn(longGaugeBuilder); + when(mockDoubleGaugeBuilder.setUnit(Mockito.anyString())).thenReturn(mockDoubleGaugeBuilder); + when(mockDoubleGaugeBuilder.setDescription(Mockito.anyString())).thenReturn(mockDoubleGaugeBuilder); + when(testMeter.gaugeBuilder(anyString())).thenReturn(mockDoubleGaugeBuilder); + } + + // testing that a value reported is then used in a callback + @SuppressWarnings("unchecked") + public void testLongGaugeRecord() { + LongGaugeAdapter longGaugeAdapter = new LongGaugeAdapter(testMeter, "name", "desc", "unit"); + + // recording a value + longGaugeAdapter.record(1L, Map.of("k", 1L)); + + // upon metric export, the consumer will be called + ArgumentCaptor> captor = ArgumentCaptor.forClass(Consumer.class); + verify(longGaugeBuilder).buildWithCallback(captor.capture()); + + Consumer value = captor.getValue(); + // making sure that a consumer will fetch the value passed down upon recording of a value + TestLongMeasurement testLongMeasurement = new TestLongMeasurement(); + value.accept(testLongMeasurement); + + assertThat(testLongMeasurement.value, Matchers.equalTo(1L)); + assertThat(testLongMeasurement.attributes, Matchers.equalTo(Attributes.builder().put("k", 1).build())); + } + + // testing that a value reported is then used in a callback + @SuppressWarnings("unchecked") + public void testDoubleGaugeRecord() { + DoubleGaugeAdapter doubleGaugeAdapter = new DoubleGaugeAdapter(testMeter, "name", "desc", "unit"); + + // recording a value + doubleGaugeAdapter.record(1.0, Map.of("k", 1.0)); + + // upon metric export, the consumer will be called + ArgumentCaptor> captor = ArgumentCaptor.forClass(Consumer.class); + verify(mockDoubleGaugeBuilder).buildWithCallback(captor.capture()); + + Consumer value = captor.getValue(); + // making sure that a consumer will fetch the value passed down upon recording of a value + TestDoubleMeasurement testLongMeasurement = new TestDoubleMeasurement(); + value.accept(testLongMeasurement); + + assertThat(testLongMeasurement.value, Matchers.equalTo(1.0)); + assertThat(testLongMeasurement.attributes, Matchers.equalTo(Attributes.builder().put("k", 1.0).build())); + } + + private static class TestDoubleMeasurement implements ObservableDoubleMeasurement { + double value; + Attributes attributes; + + @Override + public void record(double value) { + this.value = value; + } + + @Override + public void record(double value, Attributes attributes) { + this.value = value; + this.attributes = attributes; + + } + } + + private static class TestLongMeasurement implements ObservableLongMeasurement { + long value; + Attributes attributes; + + @Override + public void record(long value) { + this.value = value; + } + + @Override + public void record(long value, Attributes attributes) { + this.value = value; + this.attributes = attributes; + + } + } +} From 7836c18e58c994ac4543a4b5bdd17ebdf633437c Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Tue, 10 Oct 2023 20:41:42 +0100 Subject: [PATCH 19/48] fix BlockAccountingTests and add a new filter test scenario (#100600) This commit fixes a failure in BlockAccountingTests, and adds a new filter test scenario that filters on the last position of a multivalue. --- .../compute/data/BlockAccountingTests.java | 72 ++++++++++--------- .../compute/data/FilteredBlockTests.java | 69 ++++++++++++++++++ 2 files changed, 106 insertions(+), 35 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java index c8364141d8377..d62fd75abbcdd 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java @@ -45,7 +45,7 @@ public void testBooleanVector() { Vector emptyPlusOne = new BooleanArrayVector(new boolean[] { randomBoolean() }, 1); assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + 1))); - boolean[] randomData = new boolean[randomIntBetween(1, 1024)]; + boolean[] randomData = new boolean[randomIntBetween(2, 1024)]; Vector emptyPlusSome = new BooleanArrayVector(randomData, randomData.length); assertThat(emptyPlusSome.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + randomData.length))); @@ -61,7 +61,7 @@ public void testIntVector() { Vector emptyPlusOne = new IntArrayVector(new int[] { randomInt() }, 1); assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Integer.BYTES))); - int[] randomData = new int[randomIntBetween(1, 1024)]; + int[] randomData = new int[randomIntBetween(2, 1024)]; Vector emptyPlusSome = new IntArrayVector(randomData, randomData.length); assertThat(emptyPlusSome.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + (long) Integer.BYTES * randomData.length))); @@ -77,7 +77,7 @@ public void testLongVector() { Vector emptyPlusOne = new LongArrayVector(new long[] { randomLong() }, 1); assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + Long.BYTES)); - long[] randomData = new long[randomIntBetween(1, 1024)]; + long[] randomData = new long[randomIntBetween(2, 1024)]; Vector emptyPlusSome = new LongArrayVector(randomData, randomData.length); assertThat(emptyPlusSome.ramBytesUsed(), is(empty.ramBytesUsed() + (long) Long.BYTES * randomData.length)); @@ -93,7 +93,7 @@ public void testDoubleVector() { Vector emptyPlusOne = new DoubleArrayVector(new double[] { randomDouble() }, 1); assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + Double.BYTES)); - double[] randomData = new double[randomIntBetween(1, 1024)]; + double[] randomData = new double[randomIntBetween(2, 1024)]; Vector emptyPlusSome = new DoubleArrayVector(randomData, randomData.length); assertThat(emptyPlusSome.ramBytesUsed(), is(empty.ramBytesUsed() + (long) Double.BYTES * randomData.length)); @@ -130,13 +130,11 @@ public void testBooleanBlock() { Block emptyPlusOne = new BooleanArrayBlock(new boolean[] { randomBoolean() }, 1, new int[] { 0 }, null, Block.MvOrdering.UNORDERED); assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + 1) + alignObjectSize(Integer.BYTES))); - boolean[] randomData = new boolean[randomIntBetween(1, 1024)]; - int[] valueIndices = IntStream.range(0, randomData.length).toArray(); + boolean[] randomData = new boolean[randomIntBetween(2, 1024)]; + int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); Block emptyPlusSome = new BooleanArrayBlock(randomData, randomData.length, valueIndices, null, Block.MvOrdering.UNORDERED); - assertThat( - emptyPlusSome.ramBytesUsed(), - is(alignObjectSize(empty.ramBytesUsed() + randomData.length) + alignObjectSize(valueIndices.length * Integer.BYTES)) - ); + long expected = empty.ramBytesUsed() + ramBytesForBooleanArray(randomData) + ramBytesForIntArray(valueIndices); + assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); assertThat(filterBlock.ramBytesUsed(), lessThan(emptyPlusOne.ramBytesUsed())); @@ -148,7 +146,6 @@ public void testBooleanBlockWithNullFirstValues() { assertThat(empty.ramBytesUsed(), lessThanOrEqualTo(expectedEmptyUsed)); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100586") public void testIntBlock() { Block empty = new IntArrayBlock(new int[] {}, 0, new int[] {}, null, Block.MvOrdering.UNORDERED); long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR); @@ -157,10 +154,11 @@ public void testIntBlock() { Block emptyPlusOne = new IntArrayBlock(new int[] { randomInt() }, 1, new int[] { 0 }, null, Block.MvOrdering.UNORDERED); assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + alignObjectSize(Integer.BYTES) + alignObjectSize(Integer.BYTES))); - int[] randomData = new int[randomIntBetween(1, 1024)]; - int[] valueIndices = IntStream.range(0, randomData.length).toArray(); + int[] randomData = new int[randomIntBetween(2, 1024)]; + int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); Block emptyPlusSome = new IntArrayBlock(randomData, randomData.length, valueIndices, null, Block.MvOrdering.UNORDERED); - assertThat(emptyPlusSome.ramBytesUsed(), is(empty.ramBytesUsed() + alignObjectSize((long) Integer.BYTES * randomData.length) * 2)); + long expected = empty.ramBytesUsed() + ramBytesForIntArray(randomData) + ramBytesForIntArray(valueIndices); + assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); assertThat(filterBlock.ramBytesUsed(), lessThan(emptyPlusOne.ramBytesUsed())); @@ -180,17 +178,11 @@ public void testLongBlock() { Block emptyPlusOne = new LongArrayBlock(new long[] { randomInt() }, 1, new int[] { 0 }, null, Block.MvOrdering.UNORDERED); assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Long.BYTES) + alignObjectSize(Integer.BYTES))); - long[] randomData = new long[randomIntBetween(1, 1024)]; - int[] valueIndices = IntStream.range(0, randomData.length).toArray(); + long[] randomData = new long[randomIntBetween(2, 1024)]; + int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); Block emptyPlusSome = new LongArrayBlock(randomData, randomData.length, valueIndices, null, Block.MvOrdering.UNORDERED); - assertThat( - emptyPlusSome.ramBytesUsed(), - is( - alignObjectSize(empty.ramBytesUsed() + (long) Long.BYTES * randomData.length) + alignObjectSize( - (long) valueIndices.length * Integer.BYTES - ) - ) - ); + long expected = empty.ramBytesUsed() + ramBytesForLongArray(randomData) + ramBytesForIntArray(valueIndices); + assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); assertThat(filterBlock.ramBytesUsed(), lessThan(emptyPlusOne.ramBytesUsed())); @@ -210,17 +202,11 @@ public void testDoubleBlock() { Block emptyPlusOne = new DoubleArrayBlock(new double[] { randomInt() }, 1, new int[] { 0 }, null, Block.MvOrdering.UNORDERED); assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Double.BYTES) + alignObjectSize(Integer.BYTES))); - double[] randomData = new double[randomIntBetween(1, 1024)]; - int[] valueIndices = IntStream.range(0, randomData.length).toArray(); + double[] randomData = new double[randomIntBetween(2, 1024)]; + int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); Block emptyPlusSome = new DoubleArrayBlock(randomData, randomData.length, valueIndices, null, Block.MvOrdering.UNORDERED); - assertThat( - emptyPlusSome.ramBytesUsed(), - is( - alignObjectSize(empty.ramBytesUsed() + (long) Double.BYTES * randomData.length) + alignObjectSize( - valueIndices.length * Integer.BYTES - ) - ) - ); + long expected = empty.ramBytesUsed() + ramBytesForDoubleArray(randomData) + ramBytesForIntArray(valueIndices); + assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); assertThat(filterBlock.ramBytesUsed(), lessThan(emptyPlusOne.ramBytesUsed())); @@ -260,5 +246,21 @@ public long accumulateObject(Object o, long shallowSize, Map fiel } return shallowSize; } - }; + } + + static long ramBytesForBooleanArray(boolean[] arr) { + return alignObjectSize((long) Byte.BYTES * arr.length); + } + + static long ramBytesForIntArray(int[] arr) { + return alignObjectSize((long) Integer.BYTES * arr.length); + } + + static long ramBytesForLongArray(long[] arr) { + return alignObjectSize((long) Long.BYTES * arr.length); + } + + static long ramBytesForDoubleArray(double[] arr) { + return alignObjectSize((long) Long.BYTES * arr.length); + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/FilteredBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/FilteredBlockTests.java index 28721be14f548..f43159b7ce9bd 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/FilteredBlockTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/FilteredBlockTests.java @@ -316,6 +316,75 @@ public void testFilterToStringMultiValue() { } } + /** Tests filtering on the last position of a block with multi-values. */ + public void testFilterOnLastPositionWithMultiValues() { + { + var builder = blockFactory.newBooleanBlockBuilder(0); + builder.beginPositionEntry().appendBoolean(true).appendBoolean(false).endPositionEntry(); + builder.beginPositionEntry().appendBoolean(false).appendBoolean(true).endPositionEntry(); + BooleanBlock block = builder.build(); + var filter = block.filter(1); + assertThat(filter.getPositionCount(), is(1)); + assertThat(filter.getValueCount(0), is(2)); + assertThat(filter.getBoolean(filter.getFirstValueIndex(0)), is(false)); + assertThat(filter.getBoolean(filter.getFirstValueIndex(0) + 1), is(true)); + Releasables.close(builder, block); + releaseAndAssertBreaker(filter); + } + { + var builder = blockFactory.newIntBlockBuilder(6); + builder.beginPositionEntry().appendInt(0).appendInt(10).endPositionEntry(); + builder.beginPositionEntry().appendInt(20).appendInt(50).endPositionEntry(); + var block = builder.build(); + var filter = block.filter(1); + assertThat(filter.getPositionCount(), is(1)); + assertThat(filter.getInt(filter.getFirstValueIndex(0)), is(20)); + assertThat(filter.getInt(filter.getFirstValueIndex(0) + 1), is(50)); + assertThat(filter.getValueCount(0), is(2)); + Releasables.close(builder, block); + releaseAndAssertBreaker(filter); + } + { + var builder = blockFactory.newLongBlockBuilder(6); + builder.beginPositionEntry().appendLong(0).appendLong(10).endPositionEntry(); + builder.beginPositionEntry().appendLong(20).appendLong(50).endPositionEntry(); + var block = builder.build(); + var filter = block.filter(1); + assertThat(filter.getPositionCount(), is(1)); + assertThat(filter.getValueCount(0), is(2)); + assertThat(filter.getLong(filter.getFirstValueIndex(0)), is(20L)); + assertThat(filter.getLong(filter.getFirstValueIndex(0) + 1), is(50L)); + Releasables.close(builder, block); + releaseAndAssertBreaker(filter); + } + { + var builder = blockFactory.newDoubleBlockBuilder(6); + builder.beginPositionEntry().appendDouble(0).appendDouble(10).endPositionEntry(); + builder.beginPositionEntry().appendDouble(0.002).appendDouble(10e8).endPositionEntry(); + var block = builder.build(); + var filter = block.filter(1); + assertThat(filter.getPositionCount(), is(1)); + assertThat(filter.getValueCount(0), is(2)); + assertThat(filter.getDouble(filter.getFirstValueIndex(0)), is(0.002)); + assertThat(filter.getDouble(filter.getFirstValueIndex(0) + 1), is(10e8)); + Releasables.close(builder, block); + releaseAndAssertBreaker(filter); + } + { + var builder = blockFactory.newBytesRefBlockBuilder(6); + builder.beginPositionEntry().appendBytesRef(new BytesRef("cat")).appendBytesRef(new BytesRef("dog")).endPositionEntry(); + builder.beginPositionEntry().appendBytesRef(new BytesRef("pig")).appendBytesRef(new BytesRef("chicken")).endPositionEntry(); + var block = builder.build(); + var filter = block.filter(1); + assertThat(filter.getPositionCount(), is(1)); + assertThat(filter.getValueCount(0), is(2)); + assertThat(filter.getBytesRef(filter.getFirstValueIndex(0), new BytesRef()), equalTo(new BytesRef("pig"))); + assertThat(filter.getBytesRef(filter.getFirstValueIndex(0) + 1, new BytesRef()), equalTo(new BytesRef("chicken"))); + Releasables.close(builder, block); + releaseAndAssertBreaker(filter); + } + } + static int randomPosition(int positionCount) { return positionCount == 1 ? 0 : randomIntBetween(0, positionCount - 1); } From 15828c9b6a56522aaefef506bf4f96cbe4958b46 Mon Sep 17 00:00:00 2001 From: Volodymyr Krasnikov <129072588+volodk85@users.noreply.github.com> Date: Tue, 10 Oct 2023 13:11:11 -0700 Subject: [PATCH 20/48] Add support for reindex over CCS (#96968) * Allow prefix index naming while reindexing from remote * Update docs/changelog/96968.yaml * Add ignore_unavailable index option to query parameters * Code style * Fix asciidoc * Exclude remote index names from reindex alias validation * spotless fix * Fix test * Fix test * code style fix * Do not eval remote expr locally + IT test * Fix test * in progress * Reverting back a bit (sync with main) * Ignore remote names in ReindexValidator * Add IT test, fix double re-indexing * codestyle * reduce scope of PR (do not handle ignore_unavailable request option) * minus api specs file * add datemath index name tests for within-cluster reindexing * Move out (to separate PR) logic which handles complex datemath expressions --- docs/changelog/96968.yaml | 6 + .../index/reindex/CrossClusterReindexIT.java | 162 ++++++++++++++++++ .../reindex/ReindexValidator.java | 18 +- 3 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/96968.yaml create mode 100644 modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java diff --git a/docs/changelog/96968.yaml b/docs/changelog/96968.yaml new file mode 100644 index 0000000000000..8cc6d4ac4c284 --- /dev/null +++ b/docs/changelog/96968.yaml @@ -0,0 +1,6 @@ +pr: 96968 +summary: Allow prefix index naming while reindexing from remote +area: Reindex +type: bug +issues: + - 89120 diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java new file mode 100644 index 0000000000000..b182d9e8c2bde --- /dev/null +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.reindex; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.test.AbstractMultiClustersTestCase; + +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class CrossClusterReindexIT extends AbstractMultiClustersTestCase { + + private static final String REMOTE_CLUSTER = "remote-cluster"; + + @Override + protected boolean reuseClusters() { + return false; + } + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE_CLUSTER); + } + + @Override + protected Collection> nodePlugins(String clusterAlias) { + return List.of(ReindexPlugin.class); + } + + private int indexDocs(Client client, String index) { + int numDocs = between(1, 100); + for (int i = 0; i < numDocs; i++) { + client.prepareIndex(index).setSource("f", "v").get(); + } + client.admin().indices().prepareRefresh(index).get(); + return numDocs; + } + + public void testReindexFromRemoteGivenIndexExists() throws Exception { + assertAcked(client(REMOTE_CLUSTER).admin().indices().prepareCreate("source-index-001")); + final int docsNumber = indexDocs(client(REMOTE_CLUSTER), "source-index-001"); + + final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "source-index-001"; + new ReindexRequestBuilder(client(LOCAL_CLUSTER), ReindexAction.INSTANCE).source(sourceIndexInRemote) + .destination("desc-index-001") + .get(); + + assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { + SearchResponse resp = client(LOCAL_CLUSTER).prepareSearch("desc-index-001") + .setQuery(new MatchAllQueryBuilder()) + .setSize(1000) + .get(); + final TotalHits totalHits = resp.getHits().getTotalHits(); + return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber; + })); + } + + public void testReindexFromRemoteGivenSameIndexNames() throws Exception { + assertAcked(client(REMOTE_CLUSTER).admin().indices().prepareCreate("test-index-001")); + final int docsNumber = indexDocs(client(REMOTE_CLUSTER), "test-index-001"); + + final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "test-index-001"; + new ReindexRequestBuilder(client(LOCAL_CLUSTER), ReindexAction.INSTANCE).source(sourceIndexInRemote) + .destination("test-index-001") + .get(); + + assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { + SearchResponse resp = client(LOCAL_CLUSTER).prepareSearch("test-index-001") + .setQuery(new MatchAllQueryBuilder()) + .setSize(1000) + .get(); + final TotalHits totalHits = resp.getHits().getTotalHits(); + return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber; + })); + } + + public void testReindexManyTimesFromRemoteGivenSameIndexNames() throws Exception { + assertAcked(client(REMOTE_CLUSTER).admin().indices().prepareCreate("test-index-001")); + final long docsNumber = indexDocs(client(REMOTE_CLUSTER), "test-index-001"); + + final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "test-index-001"; + + int N = randomIntBetween(2, 10); + for (int attempt = 0; attempt < N; attempt++) { + + BulkByScrollResponse response = new ReindexRequestBuilder(client(LOCAL_CLUSTER), ReindexAction.INSTANCE).source( + sourceIndexInRemote + ).destination("test-index-001").get(); + + if (attempt == 0) { + assertThat(response.getCreated(), equalTo(docsNumber)); + assertThat(response.getUpdated(), equalTo(0L)); + } else { + assertThat(response.getCreated(), equalTo(0L)); + assertThat(response.getUpdated(), equalTo(docsNumber)); + } + + assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { + SearchResponse resp = client(LOCAL_CLUSTER).prepareSearch("test-index-001") + .setQuery(new MatchAllQueryBuilder()) + .setSize(1000) + .get(); + final TotalHits totalHits = resp.getHits().getTotalHits(); + return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber; + })); + } + } + + public void testReindexFromRemoteThrowOnUnavailableIndex() throws Exception { + + final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "no-such-source-index-001"; + expectThrows( + IndexNotFoundException.class, + () -> new ReindexRequestBuilder(client(LOCAL_CLUSTER), ReindexAction.INSTANCE).source(sourceIndexInRemote) + .destination("desc-index-001") + .get() + ); + + // assert that local index was not created either + final IndexNotFoundException e = expectThrows( + IndexNotFoundException.class, + () -> client(LOCAL_CLUSTER).prepareSearch("desc-index-001").setQuery(new MatchAllQueryBuilder()).setSize(1000).get() + ); + assertThat(e.getMessage(), containsString("no such index [desc-index-001]")); + } + + public void testReindexFromRemoteGivenSimpleDateMathIndexName() throws InterruptedException { + assertAcked(client(REMOTE_CLUSTER).admin().indices().prepareCreate("datemath-2001-01-02")); + final int docsNumber = indexDocs(client(REMOTE_CLUSTER), "datemath-2001-01-02"); + + final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + ""; + new ReindexRequestBuilder(client(LOCAL_CLUSTER), ReindexAction.INSTANCE).source(sourceIndexInRemote) + .destination("desc-index-001") + .get(); + + assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { + SearchResponse resp = client(LOCAL_CLUSTER).prepareSearch("desc-index-001") + .setQuery(new MatchAllQueryBuilder()) + .setSize(1000) + .get(); + final TotalHits totalHits = resp.getHits().getTotalHits(); + return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber; + })); + } + +} diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java index aad38f64f64a5..a874dd1846e68 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.reindex.RemoteInfo; import org.elasticsearch.search.builder.SearchSourceBuilder; +import java.util.Arrays; import java.util.List; public class ReindexValidator { @@ -138,7 +139,12 @@ static void validateAgainstAliases( */ target = indexNameExpressionResolver.concreteWriteIndex(clusterState, destination).getName(); } - for (String sourceIndex : indexNameExpressionResolver.concreteIndexNames(clusterState, source)) { + SearchRequest filteredSource = skipRemoteIndexNames(source); + if (filteredSource.indices().length == 0) { + return; + } + String[] sourceIndexNames = indexNameExpressionResolver.concreteIndexNames(clusterState, filteredSource); + for (String sourceIndex : sourceIndexNames) { if (sourceIndex.equals(target)) { ActionRequestValidationException e = new ActionRequestValidationException(); e.addValidationError("reindex cannot write into an index its reading from [" + target + ']'); @@ -146,4 +152,14 @@ static void validateAgainstAliases( } } } + + private static SearchRequest skipRemoteIndexNames(SearchRequest source) { + return new SearchRequest(source).indices( + Arrays.stream(source.indices()).filter(name -> isRemoteExpression(name) == false).toArray(String[]::new) + ); + } + + private static boolean isRemoteExpression(String expression) { + return expression.contains(":"); + } } From 63b4ee128cbdd44d9f151ff0fe58c220c9cd1dcc Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 10 Oct 2023 22:12:35 +0100 Subject: [PATCH 21/48] Increase timeout in MixedClusterClientYamlTestSuiteIT (#100585) This suite now has a couple of thousand tests, some of which take a couple of seconds, so it times out occasionally. Relaxing the timeout further. --- .../backwards/MixedClusterClientYamlTestSuiteIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java index 1b53a64fb096d..f7caf4805be15 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java @@ -15,7 +15,7 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -@TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell +@TimeoutSuite(millis = 60 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell public class MixedClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { public MixedClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) { From c3b49c56c554690252e71d8376016393a5c6698b Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Wed, 11 Oct 2023 02:42:58 +0300 Subject: [PATCH 22/48] ESQL: Handle queries with non-existing enrich policies and no field (#100647) When dealing with non-existing policies, the validation code kept trying to determine the matching field resulting in a NPE. Fix #100593 --- docs/changelog/100647.yaml | 6 ++++++ .../org/elasticsearch/xpack/esql/analysis/Analyzer.java | 2 +- .../elasticsearch/xpack/esql/analysis/AnalyzerTests.java | 8 ++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/100647.yaml diff --git a/docs/changelog/100647.yaml b/docs/changelog/100647.yaml new file mode 100644 index 0000000000000..399407146af68 --- /dev/null +++ b/docs/changelog/100647.yaml @@ -0,0 +1,6 @@ +pr: 100647 +summary: "ESQL: Handle queries with non-existing enrich policies and no field" +area: ES|QL +type: bug +issues: + - 100593 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 8732321e8d068..818d58e91a91c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -219,7 +219,7 @@ protected LogicalPlan rule(Enrich plan, AnalyzerContext context) { ) : plan.policyName(); - var matchField = plan.matchField() == null || plan.matchField() instanceof EmptyAttribute + var matchField = policy != null && (plan.matchField() == null || plan.matchField() instanceof EmptyAttribute) ? new UnresolvedAttribute(plan.source(), policy.getMatchField()) : plan.matchField(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 1ee90256b95dd..6cbc1f93bcdf1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1256,6 +1256,14 @@ public void testNonExistingEnrichPolicy() { assertThat(e.getMessage(), containsString("unresolved enrich policy [foo]")); } + public void testNonExistingEnrichNoMatchField() { + var e = expectThrows(VerificationException.class, () -> analyze(""" + from test + | enrich foo + """)); + assertThat(e.getMessage(), containsString("unresolved enrich policy [foo]")); + } + public void testNonExistingEnrichPolicyWithSimilarName() { var e = expectThrows(VerificationException.class, () -> analyze(""" from test From 939de19eab594debacf7544512d1e9cd467da834 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Wed, 11 Oct 2023 04:25:46 +0300 Subject: [PATCH 23/48] ESQL: Graceful handling of non-bool condition in the filter (#100645) Improve the Verifier to handle queries with non-boolean expressions used in the WHERE clause (where 10) In the process improve the readability of the Verifier class by extracting the checks into their own methods Fix #100049 Fix #100409 --- docs/changelog/100645.yaml | 7 + .../xpack/esql/analysis/Verifier.java | 187 ++++++++++-------- .../xpack/esql/analysis/VerifierTests.java | 8 + 3 files changed, 118 insertions(+), 84 deletions(-) create mode 100644 docs/changelog/100645.yaml diff --git a/docs/changelog/100645.yaml b/docs/changelog/100645.yaml new file mode 100644 index 0000000000000..e6bb6ab0fd653 --- /dev/null +++ b/docs/changelog/100645.yaml @@ -0,0 +1,7 @@ +pr: 100645 +summary: "ESQL: Graceful handling of non-bool condition in the filter" +area: ES|QL +type: bug +issues: + - 100049 + - 100409 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 59c6e2782b014..40f81d0247b33 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -55,6 +55,7 @@ import static org.elasticsearch.xpack.esql.stats.FeatureMetric.SORT; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.STATS; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.WHERE; +import static org.elasticsearch.xpack.ql.analyzer.VerifierChecks.checkFilterConditionType; import static org.elasticsearch.xpack.ql.common.Failure.fail; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.FIRST; @@ -121,87 +122,128 @@ else if (p.resolved()) { // Concrete verifications plan.forEachDown(p -> { - if (p instanceof Aggregate agg) { - agg.aggregates().forEach(e -> { - var exp = e instanceof Alias ? ((Alias) e).child() : e; - if (exp instanceof AggregateFunction aggFunc) { - Expression field = aggFunc.field(); - - // TODO: allow an expression? - if ((field instanceof FieldAttribute - || field instanceof MetadataAttribute - || field instanceof ReferenceAttribute - || field instanceof Literal) == false) { - failures.add( - fail( - e, - "aggregate function's field must be an attribute or literal; found [" - + field.sourceText() - + "] of type [" - + field.nodeName() - + "]" - ) - ); - } - } else if (agg.groupings().contains(exp) == false) { // TODO: allow an expression? + // if the children are unresolved, so will this node; counting it will only add noise + if (p.childrenResolved() == false) { + return; + } + checkFilterConditionType(p, failures); + checkAggregate(p, failures); + checkRegexExtractOnlyOnStrings(p, failures); + + checkRow(p, failures); + checkEvalFields(p, failures); + + checkOperationsOnUnsignedLong(p, failures); + checkBinaryComparison(p, failures); + }); + + // gather metrics + if (failures.isEmpty()) { + gatherMetrics(plan); + } + + return failures; + } + + private static void checkAggregate(LogicalPlan p, Set failures) { + if (p instanceof Aggregate agg) { + agg.aggregates().forEach(e -> { + var exp = e instanceof Alias ? ((Alias) e).child() : e; + if (exp instanceof AggregateFunction aggFunc) { + Expression field = aggFunc.field(); + + // TODO: allow an expression? + if ((field instanceof FieldAttribute + || field instanceof MetadataAttribute + || field instanceof ReferenceAttribute + || field instanceof Literal) == false) { failures.add( fail( - exp, - "expected an aggregate function or group but got [" - + exp.sourceText() + e, + "aggregate function's field must be an attribute or literal; found [" + + field.sourceText() + "] of type [" - + exp.nodeName() + + field.nodeName() + "]" ) ); } - }); - } else if (p instanceof RegexExtract re) { - Expression expr = re.input(); - DataType type = expr.dataType(); - if (EsqlDataTypes.isString(type) == false) { + } else if (agg.groupings().contains(exp) == false) { // TODO: allow an expression? failures.add( fail( - expr, - "{} only supports KEYWORD or TEXT values, found expression [{}] type [{}]", - re.getClass().getSimpleName(), - expr.sourceText(), - type + exp, + "expected an aggregate function or group but got [" + exp.sourceText() + "] of type [" + exp.nodeName() + "]" ) ); } - } else if (p instanceof Row row) { - failures.addAll(validateRow(row)); - } else if (p instanceof Eval eval) { - failures.addAll(validateEval(eval)); + }); + } + } + + private static void checkRegexExtractOnlyOnStrings(LogicalPlan p, Set failures) { + if (p instanceof RegexExtract re) { + Expression expr = re.input(); + DataType type = expr.dataType(); + if (EsqlDataTypes.isString(type) == false) { + failures.add( + fail( + expr, + "{} only supports KEYWORD or TEXT values, found expression [{}] type [{}]", + re.getClass().getSimpleName(), + expr.sourceText(), + type + ) + ); } + } + } - p.forEachExpression(BinaryOperator.class, bo -> { - Failure f = validateUnsignedLongOperator(bo); - if (f != null) { - failures.add(f); + private static void checkRow(LogicalPlan p, Set failures) { + if (p instanceof Row row) { + row.fields().forEach(a -> { + if (EsqlDataTypes.isRepresentable(a.dataType()) == false) { + failures.add(fail(a, "cannot use [{}] directly in a row assignment", a.child().sourceText())); } }); - p.forEachExpression(BinaryComparison.class, bc -> { - Failure f = validateBinaryComparison(bc); - if (f != null) { - failures.add(f); - } - }); - p.forEachExpression(Neg.class, neg -> { - Failure f = validateUnsignedLongNegation(neg); - if (f != null) { - failures.add(f); + } + } + + private static void checkEvalFields(LogicalPlan p, Set failures) { + if (p instanceof Eval eval) { + eval.fields().forEach(field -> { + DataType dataType = field.dataType(); + if (EsqlDataTypes.isRepresentable(dataType) == false) { + failures.add( + fail(field, "EVAL does not support type [{}] in expression [{}]", dataType.typeName(), field.child().sourceText()) + ); } }); - }); - - // gather metrics - if (failures.isEmpty()) { - gatherMetrics(plan); } + } - return failures; + private static void checkOperationsOnUnsignedLong(LogicalPlan p, Set failures) { + p.forEachExpression(e -> { + Failure f = null; + + if (e instanceof BinaryOperator bo) { + f = validateUnsignedLongOperator(bo); + } else if (e instanceof Neg neg) { + f = validateUnsignedLongNegation(neg); + } + + if (f != null) { + failures.add(f); + } + }); + } + + private static void checkBinaryComparison(LogicalPlan p, Set failures) { + p.forEachExpression(BinaryComparison.class, bc -> { + Failure f = validateBinaryComparison(bc); + if (f != null) { + failures.add(f); + } + }); } private void gatherMetrics(LogicalPlan plan) { @@ -228,29 +270,6 @@ private void gatherMetrics(LogicalPlan plan) { } } - private static Collection validateRow(Row row) { - List failures = new ArrayList<>(row.fields().size()); - row.fields().forEach(a -> { - if (EsqlDataTypes.isRepresentable(a.dataType()) == false) { - failures.add(fail(a, "cannot use [{}] directly in a row assignment", a.child().sourceText())); - } - }); - return failures; - } - - private static Collection validateEval(Eval eval) { - List failures = new ArrayList<>(eval.fields().size()); - eval.fields().forEach(field -> { - DataType dataType = field.dataType(); - if (EsqlDataTypes.isRepresentable(dataType) == false) { - failures.add( - fail(field, "EVAL does not support type [{}] in expression [{}]", dataType.typeName(), field.child().sourceText()) - ); - } - }); - return failures; - } - /** * Limit QL's comparisons to types we support. */ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 10f134432a0a2..cd1c9d8fbe830 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -292,6 +292,14 @@ public void testPeriodAndDurationInEval() { } } + public void testFilterNonBoolField() { + assertEquals("1:19: Condition expression needs to be boolean, found [INTEGER]", error("from test | where emp_no")); + } + + public void testFilterDateConstant() { + assertEquals("1:19: Condition expression needs to be boolean, found [DATE_PERIOD]", error("from test | where 1 year")); + } + private String error(String query) { return error(query, defaultAnalyzer); From c715c03da83914225b9883c5cc12d56487e97c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Wed, 11 Oct 2023 06:57:07 +0200 Subject: [PATCH 24/48] [Transform] Make Transform Feature Reset really wait for all the tasks (#100624) --- docs/changelog/100624.yaml | 5 +++++ .../xpack/transform/integration/TestFeatureResetIT.java | 3 ++- .../java/org/elasticsearch/xpack/transform/Transform.java | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/100624.yaml diff --git a/docs/changelog/100624.yaml b/docs/changelog/100624.yaml new file mode 100644 index 0000000000000..247343bf03ed8 --- /dev/null +++ b/docs/changelog/100624.yaml @@ -0,0 +1,5 @@ +pr: 100624 +summary: Make Transform Feature Reset really wait for all the tasks +area: Transform +type: bug +issues: [] diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java index 32cdcee280d6e..6ba0f572a2f9f 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java @@ -114,7 +114,8 @@ public void testTransformFeatureReset() throws Exception { ); // assert transform indices are gone - assertThat(ESRestTestCase.entityAsMap(adminClient().performRequest(new Request("GET", ".transform-*"))), is(anEmptyMap())); + Map transformIndices = ESRestTestCase.entityAsMap(adminClient().performRequest(new Request("GET", ".transform-*"))); + assertThat("Indices were: " + transformIndices, transformIndices, is(anEmptyMap())); } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index c1964448c2662..81a719e24f633 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -426,7 +426,7 @@ public void cleanUpFeature( client.admin() .cluster() .prepareListTasks() - .setActions(TransformField.TASK_NAME) + .setActions(TransformField.TASK_NAME + "*") .setWaitForCompletion(true) .execute(ActionListener.wrap(listTransformTasks -> { listTransformTasks.rethrowFailures("Waiting for transform tasks"); From 2a7a74c8b93f5d8274c9ae24c6dea62e452cd662 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Wed, 11 Oct 2023 16:00:42 +1100 Subject: [PATCH 25/48] [Monitoring] Dont get cluster state until recovery (#100565) The LocalExporter would call `clusterService.state()` as part of a scheduled runnable, however this could end up running before the cluster state was recovered, and calling state() before recovery is not permitted (this trips an assertion in tests) The class already listened to cluster events and detected when cluster state recovery was complete, this commit causes the scheduled cleanup method to do nothing if the recovery event has not yet been received. --- docs/changelog/100565.yaml | 5 +++ .../exporter/local/LocalExporter.java | 5 +++ .../exporter/local/LocalExporterTests.java | 33 +++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 docs/changelog/100565.yaml diff --git a/docs/changelog/100565.yaml b/docs/changelog/100565.yaml new file mode 100644 index 0000000000000..066e9bbb4b227 --- /dev/null +++ b/docs/changelog/100565.yaml @@ -0,0 +1,5 @@ +pr: 100565 +summary: "[Monitoring] Dont get cluster state until recovery" +area: Monitoring +type: bug +issues: [] diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java index 467378f4cd738..ba43cf82d1458 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java @@ -596,6 +596,11 @@ private boolean canUseWatcher() { @Override public void onCleanUpIndices(TimeValue retention) { + if (stateInitialized.get() == false) { + // ^ this is once the cluster state is recovered. Don't try to interact with the cluster service until that happens + logger.debug("exporter not yet initialized"); + return; + } ClusterState clusterState = clusterService.state(); if (clusterService.localNode() == null || clusterState == null diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java index 3b0d301099d72..a30975be1055d 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java @@ -7,17 +7,25 @@ package org.elasticsearch.xpack.monitoring.exporter.local; +import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.xpack.monitoring.cleaner.CleanerService; import org.elasticsearch.xpack.monitoring.exporter.Exporter; import org.elasticsearch.xpack.monitoring.exporter.MonitoringMigrationCoordinator; +import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; public class LocalExporterTests extends ESTestCase { @@ -37,4 +45,29 @@ public void testLocalExporterRemovesListenersOnClose() { verify(licenseState).removeListener(exporter); } + public void testLocalExporterDoesNotInteractWithClusterServiceUntilStateIsRecovered() { + final ClusterService clusterService = mock(ClusterService.class); + final XPackLicenseState licenseState = mock(XPackLicenseState.class); + final Exporter.Config config = new Exporter.Config("name", "type", Settings.EMPTY, clusterService, licenseState); + final CleanerService cleanerService = mock(CleanerService.class); + final MonitoringMigrationCoordinator migrationCoordinator = new MonitoringMigrationCoordinator(); + try (Client client = new NoOpClient(getTestName())) { + final LocalExporter exporter = new LocalExporter(config, client, migrationCoordinator, cleanerService); + + final TimeValue retention = TimeValue.timeValueDays(randomIntBetween(1, 90)); + exporter.onCleanUpIndices(retention); + + verify(clusterService).addListener(same(exporter)); + verifyNoMoreInteractions(clusterService); + + final ClusterState oldState = ClusterState.EMPTY_STATE; + final ClusterState newState = ClusterStateCreationUtils.stateWithNoShard(); + exporter.clusterChanged(new ClusterChangedEvent(getTestName(), newState, oldState)); + verify(clusterService).localNode(); + + exporter.onCleanUpIndices(retention); + verify(clusterService).state(); + verify(clusterService, times(2)).localNode(); + } + } } From f35c3b49b535d7510c9ad2adae35d6cd4dd1f785 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 11 Oct 2023 06:03:34 +0100 Subject: [PATCH 26/48] AwaitsFix for #100653 --- .../xpack/downsample/DownsampleClusterDisruptionIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java index 84b55a5fa8009..cf234e31f1f7c 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java @@ -209,6 +209,7 @@ public boolean validateClusterForming() { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100653") public void testDownsampleIndexWithRollingRestart() throws Exception { try (InternalTestCluster cluster = internalCluster()) { final List masterNodes = cluster.startMasterOnlyNodes(1); From b5843e4b98d2856310b871ef03e2db09927e4b6a Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 11 Oct 2023 07:03:31 +0100 Subject: [PATCH 27/48] Encapsulate snapshots deletion process (#100617) Introduces the `SnapshotsDeletion` class which encapsulates the process of deleting some collection of snapshots. In particular this class gives us somewhere to store various deletion-wide data which significantly reduces the length of some argument lists. Relates #100568 --- .../test/repository_url/10_basic.yml | 4 +- .../SharedClusterSnapshotRestoreIT.java | 2 +- .../blobstore/BlobStoreRepository.java | 796 +++++++++--------- 3 files changed, 418 insertions(+), 384 deletions(-) diff --git a/modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/10_basic.yml b/modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/10_basic.yml index 4508dacbfe7e9..01152a5930f47 100644 --- a/modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/10_basic.yml +++ b/modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/10_basic.yml @@ -167,7 +167,7 @@ teardown: - match: {count: 3} - do: - catch: /cannot delete snapshot from a readonly repository/ + catch: /repository is readonly/ snapshot.delete: repository: repository-url snapshot: snapshot-two @@ -229,7 +229,7 @@ teardown: - match: {count: 3} - do: - catch: /cannot delete snapshot from a readonly repository/ + catch: /repository is readonly/ snapshot.delete: repository: repository-file snapshot: snapshot-one diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 7fa59f0b47b61..71d036cc6b0f0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -1050,7 +1050,7 @@ public void testReadonlyRepository() throws Exception { assertRequestBuilderThrows( client.admin().cluster().prepareDeleteSnapshot("readonly-repo", "test-snap"), RepositoryException.class, - "cannot delete snapshot from a readonly repository" + "repository is readonly" ); logger.info("--> try making another snapshot"); diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 9a2d53312d577..98d725b9d1367 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -835,432 +835,466 @@ public void deleteSnapshots( long repositoryDataGeneration, IndexVersion repositoryFormatIndexVersion, SnapshotDeleteListener listener + ) { + createSnapshotsDeletion(snapshotIds, repositoryDataGeneration, repositoryFormatIndexVersion, new ActionListener<>() { + @Override + public void onResponse(SnapshotsDeletion snapshotsDeletion) { + snapshotsDeletion.runDelete(listener); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + + private void createSnapshotsDeletion( + Collection snapshotIds, + long repositoryDataGeneration, + IndexVersion repositoryFormatIndexVersion, + ActionListener listener ) { if (isReadOnly()) { - listener.onFailure(new RepositoryException(metadata.name(), "cannot delete snapshot from a readonly repository")); + listener.onFailure(new RepositoryException(metadata.name(), "repository is readonly")); } else { - threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new AbstractRunnable() { - @Override - protected void doRun() throws Exception { - final Map rootBlobs = blobContainer().listBlobs(OperationPurpose.SNAPSHOT); - final RepositoryData repositoryData = safeRepositoryData(repositoryDataGeneration, rootBlobs); - // Cache the indices that were found before writing out the new index-N blob so that a stuck master will never - // delete an index that was created by another master node after writing this index-N blob. - final Map foundIndices = blobStore().blobContainer(indicesPath()) - .children(OperationPurpose.SNAPSHOT); - doDeleteShardSnapshots( - snapshotIds, - repositoryDataGeneration, - foundIndices, - rootBlobs, - repositoryData, - repositoryFormatIndexVersion, - listener - ); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(new RepositoryException(metadata.name(), "failed to delete snapshots " + snapshotIds, e)); - } - }); + threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.supply(listener, () -> { + final var originalRootBlobs = blobContainer().listBlobs(OperationPurpose.SNAPSHOT); + return new SnapshotsDeletion( + snapshotIds, + repositoryDataGeneration, + repositoryFormatIndexVersion, + originalRootBlobs, + blobStore().blobContainer(indicesPath()).children(OperationPurpose.SNAPSHOT), + safeRepositoryData(repositoryDataGeneration, originalRootBlobs) + ); + })); } } /** - * The result of removing a snapshot from a shard folder in the repository. - * - * @param indexId Index that the snapshot was removed from - * @param shardId Shard id that the snapshot was removed from - * @param newGeneration Id of the new index-${uuid} blob that does not include the snapshot any more - * @param blobsToDelete Blob names in the shard directory that have become unreferenced in the new shard generation + *

+ * Represents the process of deleting some collection of snapshots within this repository which since 7.6.0 looks like this: + *

+ *
    + *
  • Write a new {@link BlobStoreIndexShardSnapshots} for each affected shard, and compute the blobs to delete.
    • + *
  • Update the {@link RepositoryData} to remove references to deleted snapshots/indices and point to the new + * {@link BlobStoreIndexShardSnapshots} files.
    • + *
  • Remove up any now-unreferenced blobs.
    • + *
+ *

+ * Until the {@link RepositoryData} is updated there should be no other activities in the repository, and in particular the root + * blob must not change until it is updated by this deletion and {@link SnapshotDeleteListener#onRepositoryDataWritten} is called. + *

*/ - private record ShardSnapshotMetaDeleteResult( - IndexId indexId, - int shardId, - ShardGeneration newGeneration, - Collection blobsToDelete - ) {} - - // --------------------------------------------------------------------------------------------------------------------------------- - // The overall flow of execution + class SnapshotsDeletion { + + /** + * The IDs of the snapshots to delete. + */ + private final Collection snapshotIds; + + /** + * The {@link RepositoryData} generation at the start of the process, to ensure that the {@link RepositoryData} does not change + * while the new {@link BlobStoreIndexShardSnapshots} are being written. + */ + private final long originalRepositoryDataGeneration; + + /** + * The minimum {@link IndexVersion} of the nodes in the cluster and the snapshots remaining in the repository. The repository must + * remain readable by all node versions which support this {@link IndexVersion}. + */ + private final IndexVersion repositoryFormatIndexVersion; + + /** + * Whether the {@link #repositoryFormatIndexVersion} is new enough to support naming {@link BlobStoreIndexShardSnapshots} blobs with + * UUIDs (i.e. does not need to remain compatible with versions before v7.6.0). Older repositories use (unsafe) numeric indices for + * these blobs instead. + */ + private final boolean useShardGenerations; + + /** + * All blobs in the repository root at the start of the operation, obtained by listing the repository contents. Note that this may + * include some blobs which are no longer referenced by the current {@link RepositoryData}, but which have not yet been removed by + * the cleanup that follows an earlier deletion. This cleanup may still be ongoing (we do not wait for it to complete before + * starting the next repository operation) or it may have failed before completion (it could have been running on a different node, + * which crashed for unrelated reasons) so we track all the blobs here and clean them up again at the end. + */ + private final Map originalRootBlobs; + + /** + * All index containers at the start of the operation, obtained by listing the repository contents. Note that this may include some + * containers which are no longer referenced by the current {@link RepositoryData}, but which have not yet been removed by + * the cleanup that follows an earlier deletion. This cleanup may or may not still be ongoing (it could have been running on a + * different node, which died before completing it) so we track all the blobs here and clean them up again at the end. + */ + private final Map originalIndexContainers; + + /** + * The {@link RepositoryData} at the start of the operation, obtained after verifying that {@link #originalRootBlobs} contains no + * {@link RepositoryData} blob newer than the one identified by {@link #originalRepositoryDataGeneration}. + */ + private final RepositoryData originalRepositoryData; + + /** + * Executor to use for all repository interactions. + */ + private final Executor snapshotExecutor = threadPool.executor(ThreadPool.Names.SNAPSHOT); + + SnapshotsDeletion( + Collection snapshotIds, + long originalRepositoryDataGeneration, + IndexVersion repositoryFormatIndexVersion, + Map originalRootBlobs, + Map originalIndexContainers, + RepositoryData originalRepositoryData + ) { + this.snapshotIds = snapshotIds; + this.originalRepositoryDataGeneration = originalRepositoryDataGeneration; + this.repositoryFormatIndexVersion = repositoryFormatIndexVersion; + this.useShardGenerations = SnapshotsService.useShardGenerations(repositoryFormatIndexVersion); + this.originalRootBlobs = originalRootBlobs; + this.originalIndexContainers = originalIndexContainers; + this.originalRepositoryData = originalRepositoryData; + } + + /** + * The result of removing a snapshot from a shard folder in the repository. + * + * @param indexId Index that the snapshot was removed from + * @param shardId Shard id that the snapshot was removed from + * @param newGeneration Id of the new index-${uuid} blob that does not include the snapshot any more + * @param blobsToDelete Blob names in the shard directory that have become unreferenced in the new shard generation + */ + private record ShardSnapshotMetaDeleteResult( + IndexId indexId, + int shardId, + ShardGeneration newGeneration, + Collection blobsToDelete + ) {} - /** - * After updating the {@link RepositoryData} each of the shards directories is individually first moved to the next shard generation - * and then has all now unreferenced blobs in it deleted. - * - * @param snapshotIds SnapshotIds to delete - * @param originalRepositoryDataGeneration {@link RepositoryData} generation at the start of the process. - * @param originalIndexContainers All index containers at the start of the operation, obtained by listing the repository - * contents. - * @param originalRootBlobs All blobs found at the root of the repository at the start of the operation, obtained by - * listing the repository contents. - * @param originalRepositoryData {@link RepositoryData} at the start of the operation. - * @param repositoryFormatIndexVersion The minimum {@link IndexVersion} of the nodes in the cluster and the snapshots remaining in - * the repository. - * @param listener Listener to invoke once finished - */ - private void doDeleteShardSnapshots( - Collection snapshotIds, - long originalRepositoryDataGeneration, - Map originalIndexContainers, - Map originalRootBlobs, - RepositoryData originalRepositoryData, - IndexVersion repositoryFormatIndexVersion, - SnapshotDeleteListener listener - ) { - if (SnapshotsService.useShardGenerations(repositoryFormatIndexVersion)) { - // First write the new shard state metadata (with the removed snapshot) and compute deletion targets - final ListenableFuture> writeShardMetaDataAndComputeDeletesStep = - new ListenableFuture<>(); - writeUpdatedShardMetaDataAndComputeDeletes(snapshotIds, originalRepositoryData, true, writeShardMetaDataAndComputeDeletesStep); - // Once we have put the new shard-level metadata into place, we can update the repository metadata as follows: - // 1. Remove the snapshots from the list of existing snapshots - // 2. Update the index shard generations of all updated shard folders - // - // Note: If we fail updating any of the individual shard paths, none of them are changed since the newly created - // index-${gen_uuid} will not be referenced by the existing RepositoryData and new RepositoryData is only - // written if all shard paths have been successfully updated. - final ListenableFuture writeUpdatedRepoDataStep = new ListenableFuture<>(); - writeShardMetaDataAndComputeDeletesStep.addListener(ActionListener.wrap(shardDeleteResults -> { - final ShardGenerations.Builder builder = ShardGenerations.builder(); - for (ShardSnapshotMetaDeleteResult newGen : shardDeleteResults) { - builder.put(newGen.indexId, newGen.shardId, newGen.newGeneration); - } - final RepositoryData newRepositoryData = originalRepositoryData.removeSnapshots(snapshotIds, builder.build()); + // --------------------------------------------------------------------------------------------------------------------------------- + // The overall flow of execution + + private void runDelete(SnapshotDeleteListener listener) { + if (useShardGenerations) { + // First write the new shard state metadata (with the removed snapshot) and compute deletion targets + final ListenableFuture> writeShardMetaDataAndComputeDeletesStep = + new ListenableFuture<>(); + writeUpdatedShardMetaDataAndComputeDeletes(writeShardMetaDataAndComputeDeletesStep); + // Once we have put the new shard-level metadata into place, we can update the repository metadata as follows: + // 1. Remove the snapshots from the list of existing snapshots + // 2. Update the index shard generations of all updated shard folders + // + // Note: If we fail updating any of the individual shard paths, none of them are changed since the newly created + // index-${gen_uuid} will not be referenced by the existing RepositoryData and new RepositoryData is only + // written if all shard paths have been successfully updated. + final ListenableFuture writeUpdatedRepoDataStep = new ListenableFuture<>(); + writeShardMetaDataAndComputeDeletesStep.addListener(ActionListener.wrap(shardDeleteResults -> { + final ShardGenerations.Builder builder = ShardGenerations.builder(); + for (ShardSnapshotMetaDeleteResult newGen : shardDeleteResults) { + builder.put(newGen.indexId, newGen.shardId, newGen.newGeneration); + } + final RepositoryData newRepositoryData = originalRepositoryData.removeSnapshots(snapshotIds, builder.build()); + writeIndexGen( + newRepositoryData, + originalRepositoryDataGeneration, + repositoryFormatIndexVersion, + Function.identity(), + ActionListener.wrap(writeUpdatedRepoDataStep::onResponse, listener::onFailure) + ); + }, listener::onFailure)); + // Once we have updated the repository, run the clean-ups + writeUpdatedRepoDataStep.addListener(ActionListener.wrap(newRepositoryData -> { + listener.onRepositoryDataWritten(newRepositoryData); + // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion + try (var refs = new RefCountingRunnable(listener::onDone)) { + cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener()); + cleanupUnlinkedShardLevelBlobs(writeShardMetaDataAndComputeDeletesStep.result(), refs.acquireListener()); + } + }, listener::onFailure)); + } else { + // Write the new repository data first (with the removed snapshot), using no shard generations writeIndexGen( - newRepositoryData, + originalRepositoryData.removeSnapshots(snapshotIds, ShardGenerations.EMPTY), originalRepositoryDataGeneration, repositoryFormatIndexVersion, Function.identity(), - ActionListener.wrap(writeUpdatedRepoDataStep::onResponse, listener::onFailure) - ); - }, listener::onFailure)); - // Once we have updated the repository, run the clean-ups - writeUpdatedRepoDataStep.addListener(ActionListener.wrap(newRepositoryData -> { - listener.onRepositoryDataWritten(newRepositoryData); - // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion - try (var refs = new RefCountingRunnable(listener::onDone)) { - cleanupUnlinkedRootAndIndicesBlobs( - snapshotIds, - originalIndexContainers, - originalRootBlobs, - newRepositoryData, - refs.acquireListener() - ); - cleanupUnlinkedShardLevelBlobs( - originalRepositoryData, - snapshotIds, - writeShardMetaDataAndComputeDeletesStep.result(), - refs.acquireListener() - ); - } - }, listener::onFailure)); - } else { - // Write the new repository data first (with the removed snapshot), using no shard generations - writeIndexGen( - originalRepositoryData.removeSnapshots(snapshotIds, ShardGenerations.EMPTY), - originalRepositoryDataGeneration, - repositoryFormatIndexVersion, - Function.identity(), - ActionListener.wrap(newRepositoryData -> { - try (var refs = new RefCountingRunnable(() -> { - listener.onRepositoryDataWritten(newRepositoryData); - listener.onDone(); - })) { - // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion - cleanupUnlinkedRootAndIndicesBlobs( - snapshotIds, - originalIndexContainers, - originalRootBlobs, - newRepositoryData, - refs.acquireListener() - ); - - // writeIndexGen finishes on master-service thread so must fork here. - threadPool.executor(ThreadPool.Names.SNAPSHOT) - .execute( + ActionListener.wrap(newRepositoryData -> { + try (var refs = new RefCountingRunnable(() -> { + listener.onRepositoryDataWritten(newRepositoryData); + listener.onDone(); + })) { + // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion + cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener()); + + // writeIndexGen finishes on master-service thread so must fork here. + snapshotExecutor.execute( ActionRunnable.wrap( refs.acquireListener(), l0 -> writeUpdatedShardMetaDataAndComputeDeletes( - snapshotIds, - originalRepositoryData, - false, - l0.delegateFailure( - (l, deleteResults) -> cleanupUnlinkedShardLevelBlobs( - originalRepositoryData, - snapshotIds, - deleteResults, - l - ) - ) + l0.delegateFailure((l, shardDeleteResults) -> cleanupUnlinkedShardLevelBlobs(shardDeleteResults, l)) ) ) ); - } - }, listener::onFailure) - ); + } + }, listener::onFailure) + ); + } } - } - // --------------------------------------------------------------------------------------------------------------------------------- - // Updating the shard-level metadata and accumulating results + // --------------------------------------------------------------------------------------------------------------------------------- + // Updating the shard-level metadata and accumulating results - // updates the shard state metadata for shards of a snapshot that is to be deleted. Also computes the files to be cleaned up. - private void writeUpdatedShardMetaDataAndComputeDeletes( - Collection snapshotIds, - RepositoryData originalRepositoryData, - boolean useShardGenerations, - ActionListener> onAllShardsCompleted - ) { - - final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); - final List indices = originalRepositoryData.indicesToUpdateAfterRemovingSnapshot(snapshotIds); - - if (indices.isEmpty()) { - onAllShardsCompleted.onResponse(Collections.emptyList()); - return; - } + // updates the shard state metadata for shards of a snapshot that is to be deleted. Also computes the files to be cleaned up. + private void writeUpdatedShardMetaDataAndComputeDeletes( + ActionListener> onAllShardsCompleted + ) { - // Listener that flattens out the delete results for each index - final ActionListener> deleteIndexMetadataListener = new GroupedActionListener<>( - indices.size(), - onAllShardsCompleted.map(res -> res.stream().flatMap(Collection::stream).toList()) - ); + final List indices = originalRepositoryData.indicesToUpdateAfterRemovingSnapshot(snapshotIds); - for (IndexId indexId : indices) { - final Set snapshotsWithIndex = Set.copyOf(originalRepositoryData.getSnapshots(indexId)); - final Set survivingSnapshots = snapshotsWithIndex.stream() - .filter(id -> snapshotIds.contains(id) == false) - .collect(Collectors.toSet()); - final ListenableFuture> shardCountListener = new ListenableFuture<>(); - final Collection indexMetaGenerations = snapshotIds.stream() - .filter(snapshotsWithIndex::contains) - .map(id -> originalRepositoryData.indexMetaDataGenerations().indexMetaBlobId(id, indexId)) - .collect(Collectors.toSet()); - final ActionListener allShardCountsListener = new GroupedActionListener<>( - indexMetaGenerations.size(), - shardCountListener - ); - final BlobContainer indexContainer = indexContainer(indexId); - for (String indexMetaGeneration : indexMetaGenerations) { - executor.execute(ActionRunnable.supply(allShardCountsListener, () -> { - try { - return INDEX_METADATA_FORMAT.read(metadata.name(), indexContainer, indexMetaGeneration, namedXContentRegistry) - .getNumberOfShards(); - } catch (Exception ex) { - logger.warn( - () -> format("[%s] [%s] failed to read metadata for index", indexMetaGeneration, indexId.getName()), - ex - ); - // Just invoke the listener without any shard generations to count it down, this index will be cleaned up - // by the stale data cleanup in the end. - // TODO: Getting here means repository corruption. We should find a way of dealing with this instead of just - // ignoring it and letting the cleanup deal with it. - return null; - } - })); + if (indices.isEmpty()) { + onAllShardsCompleted.onResponse(Collections.emptyList()); + return; } - // ----------------------------------------------------------------------------------------------------------------------------- - // Determining the shard count - - shardCountListener.addListener(deleteIndexMetadataListener.delegateFailureAndWrap((delegate, counts) -> { - final int shardCount = counts.stream().mapToInt(i -> i).max().orElse(0); - if (shardCount == 0) { - delegate.onResponse(null); - return; - } - // Listener for collecting the results of removing the snapshot from each shard's metadata in the current index - final ActionListener allShardsListener = new GroupedActionListener<>(shardCount, delegate); - for (int i = 0; i < shardCount; i++) { - final int shardId = i; - executor.execute(new AbstractRunnable() { - @Override - protected void doRun() throws Exception { - final BlobContainer shardContainer = shardContainer(indexId, shardId); - final Set originalShardBlobs = shardContainer.listBlobs(OperationPurpose.SNAPSHOT).keySet(); - final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots; - final long newGen; - if (useShardGenerations) { - newGen = -1L; - blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots( - originalShardBlobs, - shardContainer, - originalRepositoryData.shardGenerations().getShardGen(indexId, shardId) - ).v1(); - } else { - Tuple tuple = buildBlobStoreIndexShardSnapshots( - originalShardBlobs, - shardContainer - ); - newGen = tuple.v2() + 1; - blobStoreIndexShardSnapshots = tuple.v1(); - } - allShardsListener.onResponse( - deleteFromShardSnapshotMeta( - survivingSnapshots, - indexId, - shardId, - snapshotIds, - shardContainer, - originalShardBlobs, - blobStoreIndexShardSnapshots, - newGen - ) - ); - } + // Listener that flattens out the delete results for each index + final ActionListener> deleteIndexMetadataListener = new GroupedActionListener<>( + indices.size(), + onAllShardsCompleted.map(res -> res.stream().flatMap(Collection::stream).toList()) + ); - @Override - public void onFailure(Exception ex) { + for (IndexId indexId : indices) { + final Set snapshotsWithIndex = Set.copyOf(originalRepositoryData.getSnapshots(indexId)); + final Set survivingSnapshots = snapshotsWithIndex.stream() + .filter(id -> snapshotIds.contains(id) == false) + .collect(Collectors.toSet()); + final ListenableFuture> shardCountListener = new ListenableFuture<>(); + final Collection indexMetaGenerations = snapshotIds.stream() + .filter(snapshotsWithIndex::contains) + .map(id -> originalRepositoryData.indexMetaDataGenerations().indexMetaBlobId(id, indexId)) + .collect(Collectors.toSet()); + final ActionListener allShardCountsListener = new GroupedActionListener<>( + indexMetaGenerations.size(), + shardCountListener + ); + final BlobContainer indexContainer = indexContainer(indexId); + for (String indexMetaGeneration : indexMetaGenerations) { + snapshotExecutor.execute(ActionRunnable.supply(allShardCountsListener, () -> { + try { + return INDEX_METADATA_FORMAT.read(metadata.name(), indexContainer, indexMetaGeneration, namedXContentRegistry) + .getNumberOfShards(); + } catch (Exception ex) { logger.warn( - () -> format("%s failed to delete shard data for shard [%s][%s]", snapshotIds, indexId.getName(), shardId), + () -> format("[%s] [%s] failed to read metadata for index", indexMetaGeneration, indexId.getName()), ex ); - // Just passing null here to count down the listener instead of failing it, the stale data left behind - // here will be retried in the next delete or repository cleanup - allShardsListener.onResponse(null); + // Just invoke the listener without any shard generations to count it down, this index will be cleaned up + // by the stale data cleanup in the end. + // TODO: Getting here means repository corruption. We should find a way of dealing with this instead of just + // ignoring it and letting the cleanup deal with it. + return null; } - }); + })); } - })); - } - } - // ----------------------------------------------------------------------------------------------------------------------------- - // Updating each shard + // ------------------------------------------------------------------------------------------------------------------------- + // Determining the shard count - /** - * Delete snapshot from shard level metadata. - * - * @param indexGeneration generation to write the new shard level level metadata to. If negative a uuid id shard generation should be - * used - */ - private ShardSnapshotMetaDeleteResult deleteFromShardSnapshotMeta( - Set survivingSnapshots, - IndexId indexId, - int shardId, - Collection snapshotIds, - BlobContainer shardContainer, - Set originalShardBlobs, - BlobStoreIndexShardSnapshots snapshots, - long indexGeneration - ) { - // Build a list of snapshots that should be preserved - final BlobStoreIndexShardSnapshots updatedSnapshots = snapshots.withRetainedSnapshots(survivingSnapshots); - ShardGeneration writtenGeneration = null; - try { - if (updatedSnapshots.snapshots().isEmpty()) { - return new ShardSnapshotMetaDeleteResult(indexId, shardId, ShardGenerations.DELETED_SHARD_GEN, originalShardBlobs); - } else { - if (indexGeneration < 0L) { - writtenGeneration = ShardGeneration.newGeneration(); - INDEX_SHARD_SNAPSHOTS_FORMAT.write(updatedSnapshots, shardContainer, writtenGeneration.toBlobNamePart(), compress); + shardCountListener.addListener(deleteIndexMetadataListener.delegateFailureAndWrap((delegate, counts) -> { + final int shardCount = counts.stream().mapToInt(i -> i).max().orElse(0); + if (shardCount == 0) { + delegate.onResponse(null); + return; + } + // Listener for collecting the results of removing the snapshot from each shard's metadata in the current index + final ActionListener allShardsListener = new GroupedActionListener<>( + shardCount, + delegate + ); + for (int i = 0; i < shardCount; i++) { + final int shardId = i; + snapshotExecutor.execute(new AbstractRunnable() { + @Override + protected void doRun() throws Exception { + final BlobContainer shardContainer = shardContainer(indexId, shardId); + final Set originalShardBlobs = shardContainer.listBlobs(OperationPurpose.SNAPSHOT).keySet(); + final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots; + final long newGen; + if (useShardGenerations) { + newGen = -1L; + blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots( + originalShardBlobs, + shardContainer, + originalRepositoryData.shardGenerations().getShardGen(indexId, shardId) + ).v1(); + } else { + Tuple tuple = buildBlobStoreIndexShardSnapshots( + originalShardBlobs, + shardContainer + ); + newGen = tuple.v2() + 1; + blobStoreIndexShardSnapshots = tuple.v1(); + } + allShardsListener.onResponse( + deleteFromShardSnapshotMeta( + survivingSnapshots, + indexId, + shardId, + snapshotIds, + shardContainer, + originalShardBlobs, + blobStoreIndexShardSnapshots, + newGen + ) + ); + } + + @Override + public void onFailure(Exception ex) { + logger.warn( + () -> format( + "%s failed to delete shard data for shard [%s][%s]", + snapshotIds, + indexId.getName(), + shardId + ), + ex + ); + // Just passing null here to count down the listener instead of failing it, the stale data left behind + // here will be retried in the next delete or repository cleanup + allShardsListener.onResponse(null); + } + }); + } + })); + } + } + + // ----------------------------------------------------------------------------------------------------------------------------- + // Updating each shard + + /** + * Delete snapshot from shard level metadata. + * + * @param indexGeneration generation to write the new shard level level metadata to. If negative a uuid id shard generation should + * be used + */ + private ShardSnapshotMetaDeleteResult deleteFromShardSnapshotMeta( + Set survivingSnapshots, + IndexId indexId, + int shardId, + Collection snapshotIds, + BlobContainer shardContainer, + Set originalShardBlobs, + BlobStoreIndexShardSnapshots snapshots, + long indexGeneration + ) { + // Build a list of snapshots that should be preserved + final BlobStoreIndexShardSnapshots updatedSnapshots = snapshots.withRetainedSnapshots(survivingSnapshots); + ShardGeneration writtenGeneration = null; + try { + if (updatedSnapshots.snapshots().isEmpty()) { + return new ShardSnapshotMetaDeleteResult(indexId, shardId, ShardGenerations.DELETED_SHARD_GEN, originalShardBlobs); } else { - writtenGeneration = new ShardGeneration(indexGeneration); - writeShardIndexBlobAtomic(shardContainer, indexGeneration, updatedSnapshots, Collections.emptyMap()); + if (indexGeneration < 0L) { + writtenGeneration = ShardGeneration.newGeneration(); + INDEX_SHARD_SNAPSHOTS_FORMAT.write(updatedSnapshots, shardContainer, writtenGeneration.toBlobNamePart(), compress); + } else { + writtenGeneration = new ShardGeneration(indexGeneration); + writeShardIndexBlobAtomic(shardContainer, indexGeneration, updatedSnapshots, Collections.emptyMap()); + } + final Set survivingSnapshotUUIDs = survivingSnapshots.stream() + .map(SnapshotId::getUUID) + .collect(Collectors.toSet()); + return new ShardSnapshotMetaDeleteResult( + indexId, + shardId, + writtenGeneration, + unusedBlobs(originalShardBlobs, survivingSnapshotUUIDs, updatedSnapshots) + ); } - final Set survivingSnapshotUUIDs = survivingSnapshots.stream().map(SnapshotId::getUUID).collect(Collectors.toSet()); - return new ShardSnapshotMetaDeleteResult( - indexId, - shardId, - writtenGeneration, - unusedBlobs(originalShardBlobs, survivingSnapshotUUIDs, updatedSnapshots) + } catch (IOException e) { + throw new RepositoryException( + metadata.name(), + "Failed to finalize snapshot deletion " + + snapshotIds + + " with shard index [" + + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(writtenGeneration.toBlobNamePart()) + + "]", + e ); } - } catch (IOException e) { - throw new RepositoryException( - metadata.name(), - "Failed to finalize snapshot deletion " - + snapshotIds - + " with shard index [" - + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(writtenGeneration.toBlobNamePart()) - + "]", - e - ); } - } - // Unused blobs are all previous index-, data- and meta-blobs and that are not referenced by the new index- as well as all - // temporary blobs - private static List unusedBlobs( - Set originalShardBlobs, - Set survivingSnapshotUUIDs, - BlobStoreIndexShardSnapshots updatedSnapshots - ) { - return originalShardBlobs.stream() - .filter( - blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX) - || (blob.startsWith(SNAPSHOT_PREFIX) - && blob.endsWith(".dat") - && survivingSnapshotUUIDs.contains( - blob.substring(SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()) - ) == false) - || (blob.startsWith(UPLOADED_DATA_BLOB_PREFIX) && updatedSnapshots.findNameFile(canonicalName(blob)) == null) - || FsBlobContainer.isTempBlobName(blob) - ) - .toList(); - } - - // --------------------------------------------------------------------------------------------------------------------------------- - // Cleaning up dangling blobs + // Unused blobs are all previous index-, data- and meta-blobs and that are not referenced by the new index- as well as all + // temporary blobs + private static List unusedBlobs( + Set originalShardBlobs, + Set survivingSnapshotUUIDs, + BlobStoreIndexShardSnapshots updatedSnapshots + ) { + return originalShardBlobs.stream() + .filter( + blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX) + || (blob.startsWith(SNAPSHOT_PREFIX) + && blob.endsWith(".dat") + && survivingSnapshotUUIDs.contains( + blob.substring(SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()) + ) == false) + || (blob.startsWith(UPLOADED_DATA_BLOB_PREFIX) && updatedSnapshots.findNameFile(canonicalName(blob)) == null) + || FsBlobContainer.isTempBlobName(blob) + ) + .toList(); + } - /** - * Delete any dangling blobs in the repository root (i.e. {@link RepositoryData}, {@link SnapshotInfo} and {@link Metadata} blobs) - * as well as any containers for indices that are now completely unreferenced. - */ - private void cleanupUnlinkedRootAndIndicesBlobs( - Collection snapshotIds, - Map originalIndexContainers, - Map originalRootBlobs, - RepositoryData newRepositoryData, - ActionListener listener - ) { - cleanupStaleBlobs(snapshotIds, originalIndexContainers, originalRootBlobs, newRepositoryData, listener.map(ignored -> null)); - } + // --------------------------------------------------------------------------------------------------------------------------------- + // Cleaning up dangling blobs - private void cleanupUnlinkedShardLevelBlobs( - RepositoryData originalRepositoryData, - Collection snapshotIds, - Collection shardDeleteResults, - ActionListener listener - ) { - final Iterator filesToDelete = resolveFilesToDelete(originalRepositoryData, snapshotIds, shardDeleteResults); - if (filesToDelete.hasNext() == false) { - listener.onResponse(null); - return; + /** + * Delete any dangling blobs in the repository root (i.e. {@link RepositoryData}, {@link SnapshotInfo} and {@link Metadata} blobs) + * as well as any containers for indices that are now completely unreferenced. + */ + private void cleanupUnlinkedRootAndIndicesBlobs(RepositoryData newRepositoryData, ActionListener listener) { + cleanupStaleBlobs(snapshotIds, originalIndexContainers, originalRootBlobs, newRepositoryData, listener.map(ignored -> null)); } - threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> { - try { - deleteFromContainer(blobContainer(), filesToDelete); - l.onResponse(null); - } catch (Exception e) { - logger.warn(() -> format("%s Failed to delete some blobs during snapshot delete", snapshotIds), e); - throw e; + + private void cleanupUnlinkedShardLevelBlobs( + Collection shardDeleteResults, + ActionListener listener + ) { + final Iterator filesToDelete = resolveFilesToDelete(shardDeleteResults); + if (filesToDelete.hasNext() == false) { + listener.onResponse(null); + return; } - })); - } + snapshotExecutor.execute(ActionRunnable.wrap(listener, l -> { + try { + deleteFromContainer(blobContainer(), filesToDelete); + l.onResponse(null); + } catch (Exception e) { + logger.warn(() -> format("%s Failed to delete some blobs during snapshot delete", snapshotIds), e); + throw e; + } + })); + } - private Iterator resolveFilesToDelete( - RepositoryData oldRepositoryData, - Collection snapshotIds, - Collection deleteResults - ) { - final String basePath = basePath().buildAsString(); - final int basePathLen = basePath.length(); - final Map> indexMetaGenerations = oldRepositoryData.indexMetaDataToRemoveAfterRemovingSnapshots( - snapshotIds - ); - return Stream.concat(deleteResults.stream().flatMap(shardResult -> { - final String shardPath = shardPath(shardResult.indexId, shardResult.shardId).buildAsString(); - return shardResult.blobsToDelete.stream().map(blob -> shardPath + blob); - }), indexMetaGenerations.entrySet().stream().flatMap(entry -> { - final String indexContainerPath = indexPath(entry.getKey()).buildAsString(); - return entry.getValue().stream().map(id -> indexContainerPath + INDEX_METADATA_FORMAT.blobName(id)); - })).map(absolutePath -> { - assert absolutePath.startsWith(basePath); - return absolutePath.substring(basePathLen); - }).iterator(); + private Iterator resolveFilesToDelete(Collection deleteResults) { + final String basePath = basePath().buildAsString(); + final int basePathLen = basePath.length(); + final Map> indexMetaGenerations = originalRepositoryData + .indexMetaDataToRemoveAfterRemovingSnapshots(snapshotIds); + return Stream.concat(deleteResults.stream().flatMap(shardResult -> { + final String shardPath = shardPath(shardResult.indexId, shardResult.shardId).buildAsString(); + return shardResult.blobsToDelete.stream().map(blob -> shardPath + blob); + }), indexMetaGenerations.entrySet().stream().flatMap(entry -> { + final String indexContainerPath = indexPath(entry.getKey()).buildAsString(); + return entry.getValue().stream().map(id -> indexContainerPath + INDEX_METADATA_FORMAT.blobName(id)); + })).map(absolutePath -> { + assert absolutePath.startsWith(basePath); + return absolutePath.substring(basePathLen); + }).iterator(); + } } /** From 3ea97796eb9a4c150cbbdfd59278005cb1c9a613 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 11 Oct 2023 07:05:13 +0100 Subject: [PATCH 28/48] Rename args in cleanup process to match deletion process (#100620) Relates #100568 --- .../blobstore/BlobStoreRepository.java | 111 ++++++++++-------- 1 file changed, 61 insertions(+), 50 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 98d725b9d1367..1e2969b255877 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -785,16 +785,16 @@ public RepositoryStats stats() { /** * Loads {@link RepositoryData} ensuring that it is consistent with the given {@code rootBlobs} as well of the assumed generation. * - * @param repositoryStateId Expected repository generation - * @param rootBlobs Blobs at the repository root + * @param repositoryDataGeneration Expected repository generation + * @param rootBlobs Blobs at the repository root * @return RepositoryData */ - private RepositoryData safeRepositoryData(long repositoryStateId, Map rootBlobs) { + private RepositoryData safeRepositoryData(long repositoryDataGeneration, Map rootBlobs) { final long generation = latestGeneration(rootBlobs.keySet()); final long genToLoad; final RepositoryData cached; if (bestEffortConsistency) { - genToLoad = latestKnownRepoGen.accumulateAndGet(repositoryStateId, Math::max); + genToLoad = latestKnownRepoGen.accumulateAndGet(repositoryDataGeneration, Math::max); cached = null; } else { genToLoad = latestKnownRepoGen.get(); @@ -813,11 +813,11 @@ private RepositoryData safeRepositoryData(long repositoryStateId, Map resolveFilesToDelete(Collection deletedSnapshots, - Map foundIndices, - Map rootBlobs, - RepositoryData newRepoData, + Collection snapshotIds, + Map originalIndexContainers, + Map originalRootBlobs, + RepositoryData newRepositoryData, ActionListener listener ) { final var blobsDeleted = new AtomicLong(); final var bytesDeleted = new AtomicLong(); try (var listeners = new RefCountingListener(listener.map(ignored -> DeleteResult.of(blobsDeleted.get(), bytesDeleted.get())))) { - final List staleRootBlobs = staleRootBlobs(newRepoData, rootBlobs.keySet()); + final List staleRootBlobs = staleRootBlobs(newRepositoryData, originalRootBlobs.keySet()); if (staleRootBlobs.isEmpty() == false) { staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { try (ref) { - logStaleRootLevelBlobs(newRepoData.getGenId() - 1, deletedSnapshots, staleRootBlobs); + logStaleRootLevelBlobs(newRepositoryData.getGenId() - 1, snapshotIds, staleRootBlobs); deleteFromContainer(blobContainer(), staleRootBlobs.iterator()); for (final var staleRootBlob : staleRootBlobs) { - bytesDeleted.addAndGet(rootBlobs.get(staleRootBlob).length()); + bytesDeleted.addAndGet(originalRootBlobs.get(staleRootBlob).length()); } blobsDeleted.addAndGet(staleRootBlobs.size()); } catch (Exception e) { @@ -1343,23 +1342,23 @@ private void cleanupStaleBlobs( })); } - final var survivingIndexIds = newRepoData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toSet()); - for (final var indexEntry : foundIndices.entrySet()) { - final var indexSnId = indexEntry.getKey(); - if (survivingIndexIds.contains(indexSnId)) { + final var survivingIndexIds = newRepositoryData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toSet()); + for (final var indexEntry : originalIndexContainers.entrySet()) { + final var indexId = indexEntry.getKey(); + if (survivingIndexIds.contains(indexId)) { continue; } staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { try (ref) { - logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexSnId); + logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexId); final var deleteResult = indexEntry.getValue().delete(OperationPurpose.SNAPSHOT); blobsDeleted.addAndGet(deleteResult.blobsDeleted()); bytesDeleted.addAndGet(deleteResult.bytesDeleted()); - logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexSnId); + logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexId); } catch (IOException e) { logger.warn(() -> format(""" [%s] index %s is no longer part of any snapshot in the repository, \ - but failed to clean up its index folder""", metadata.name(), indexSnId), e); + but failed to clean up its index folder""", metadata.name(), indexId), e); } })); } @@ -1386,40 +1385,45 @@ private void cleanupStaleBlobs( *
  • Deleting stale indices
    • *
  • Deleting unreferenced root level blobs
    • * - * @param repositoryStateId Current repository state id - * @param repositoryMetaVersion version of the updated repository metadata to write + * @param originalRepositoryDataGeneration Current repository state id + * @param repositoryFormatIndexVersion version of the updated repository metadata to write * @param listener Listener to complete when done */ - public void cleanup(long repositoryStateId, IndexVersion repositoryMetaVersion, ActionListener listener) { + public void cleanup( + long originalRepositoryDataGeneration, + IndexVersion repositoryFormatIndexVersion, + ActionListener listener + ) { try { if (isReadOnly()) { throw new RepositoryException(metadata.name(), "cannot run cleanup on readonly repository"); } - Map rootBlobs = blobContainer().listBlobs(OperationPurpose.SNAPSHOT); - final RepositoryData repositoryData = safeRepositoryData(repositoryStateId, rootBlobs); - final Map foundIndices = blobStore().blobContainer(indicesPath()).children(OperationPurpose.SNAPSHOT); - final Set survivingIndexIds = repositoryData.getIndices() + Map originalRootBlobs = blobContainer().listBlobs(OperationPurpose.SNAPSHOT); + final RepositoryData originalRepositoryData = safeRepositoryData(originalRepositoryDataGeneration, originalRootBlobs); + final Map originalIndexContainers = blobStore().blobContainer(indicesPath()) + .children(OperationPurpose.SNAPSHOT); + final Set survivingIndexIds = originalRepositoryData.getIndices() .values() .stream() .map(IndexId::getId) .collect(Collectors.toSet()); - final List staleRootBlobs = staleRootBlobs(repositoryData, rootBlobs.keySet()); - if (survivingIndexIds.equals(foundIndices.keySet()) && staleRootBlobs.isEmpty()) { + final List staleRootBlobs = staleRootBlobs(originalRepositoryData, originalRootBlobs.keySet()); + if (survivingIndexIds.equals(originalIndexContainers.keySet()) && staleRootBlobs.isEmpty()) { // Nothing to clean up we return listener.onResponse(new RepositoryCleanupResult(DeleteResult.ZERO)); } else { // write new index-N blob to ensure concurrent operations will fail writeIndexGen( - repositoryData, - repositoryStateId, - repositoryMetaVersion, + originalRepositoryData, + originalRepositoryDataGeneration, + repositoryFormatIndexVersion, Function.identity(), listener.delegateFailureAndWrap( (l, v) -> cleanupStaleBlobs( Collections.emptyList(), - foundIndices, - rootBlobs, - repositoryData, + originalIndexContainers, + originalRootBlobs, + originalRepositoryData, l.map(RepositoryCleanupResult::new) ) ) @@ -1431,9 +1435,12 @@ public void cleanup(long repositoryStateId, IndexVersion repositoryMetaVersion, } // Finds all blobs directly under the repository root path that are not referenced by the current RepositoryData - private static List staleRootBlobs(RepositoryData repositoryData, Set rootBlobNames) { - final Set allSnapshotIds = repositoryData.getSnapshotIds().stream().map(SnapshotId::getUUID).collect(Collectors.toSet()); - return rootBlobNames.stream().filter(blob -> { + private static List staleRootBlobs(RepositoryData originalRepositoryData, Set originalRootBlobNames) { + final Set allSnapshotIds = originalRepositoryData.getSnapshotIds() + .stream() + .map(SnapshotId::getUUID) + .collect(Collectors.toSet()); + return originalRootBlobNames.stream().filter(blob -> { if (FsBlobContainer.isTempBlobName(blob)) { return true; } @@ -1452,7 +1459,7 @@ private static List staleRootBlobs(RepositoryData repositoryData, Set Long.parseLong(blob.substring(INDEX_FILE_PREFIX.length())); + return originalRepositoryData.getGenId() > Long.parseLong(blob.substring(INDEX_FILE_PREFIX.length())); } catch (NumberFormatException nfe) { // odd case of an extra file with the index- prefix that we can't identify return false; @@ -1462,17 +1469,21 @@ private static List staleRootBlobs(RepositoryData repositoryData, Set deletedSnapshots, List blobsToDelete) { + private void logStaleRootLevelBlobs( + long originalRepositoryDataGeneration, + Collection snapshotIds, + List blobsToDelete + ) { if (logger.isInfoEnabled()) { // If we're running root level cleanup as part of a snapshot delete we should not log the snapshot- and global metadata // blobs associated with the just deleted snapshots as they are expected to exist and not stale. Otherwise every snapshot // delete would also log a confusing INFO message about "stale blobs". - final Set blobNamesToIgnore = deletedSnapshots.stream() + final Set blobNamesToIgnore = snapshotIds.stream() .flatMap( snapshotId -> Stream.of( GLOBAL_METADATA_FORMAT.blobName(snapshotId.getUUID()), SNAPSHOT_FORMAT.blobName(snapshotId.getUUID()), - INDEX_FILE_PREFIX + previousGeneration + INDEX_FILE_PREFIX + originalRepositoryDataGeneration ) ) .collect(Collectors.toSet()); From 5cc90094e9b24203e92649b1470736e984f59e10 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 10 Oct 2023 23:09:08 -0700 Subject: [PATCH 29/48] Relax ValueSources check in OrdinalsGroupingOperator (#100566) ValuesSource can be Null instead of Bytes when a shard has no data for a specific field. This PR relaxes the check for ValueSources in the OrdinalsGroupingOperator. We will need to add more tests for OrdinalsGroupingOperator. Closes #100438 --- .../operator/OrdinalsGroupingOperator.java | 6 ---- .../xpack/esql/action/EsqlActionIT.java | 34 +++++++++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java index 4dab7faa2a074..7c930118903cf 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java @@ -105,12 +105,6 @@ public OrdinalsGroupingOperator( DriverContext driverContext ) { Objects.requireNonNull(aggregatorFactories); - boolean bytesValues = sources.get(0).source() instanceof ValuesSource.Bytes; - for (int i = 1; i < sources.size(); i++) { - if (sources.get(i).source() instanceof ValuesSource.Bytes != bytesValues) { - throw new IllegalStateException("ValuesSources are mismatched"); - } - } this.sources = sources; this.docChannel = docChannel; this.groupingField = groupingField; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index f10ca17d741d8..0017a8600a013 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -35,6 +35,7 @@ import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -1187,6 +1188,39 @@ public void testGroupingMultiValueByOrdinals() { } } + public void testUnsupportedTypesOrdinalGrouping() { + assertAcked( + client().admin().indices().prepareCreate("index-1").setMapping("f1", "type=keyword", "f2", "type=keyword", "v", "type=long") + ); + assertAcked( + client().admin().indices().prepareCreate("index-2").setMapping("f1", "type=object", "f2", "type=keyword", "v", "type=long") + ); + Map groups = new HashMap<>(); + int numDocs = randomIntBetween(10, 20); + for (int i = 0; i < numDocs; i++) { + String k = randomFrom("a", "b", "c"); + long v = randomIntBetween(1, 10); + groups.merge(k, v, Long::sum); + groups.merge(null, v, Long::sum); // null group + client().prepareIndex("index-1").setSource("f1", k, "v", v).get(); + client().prepareIndex("index-2").setSource("f2", k, "v", v).get(); + } + client().admin().indices().prepareRefresh("index-1", "index-2").get(); + for (String field : List.of("f1", "f2")) { + try (var resp = run("from index-1,index-2 | stats sum(v) by " + field)) { + Iterator> values = resp.values(); + Map actual = new HashMap<>(); + while (values.hasNext()) { + Iterator row = values.next(); + Long v = (Long) row.next(); + String k = (String) row.next(); + actual.put(k, v); + } + assertThat(actual, equalTo(groups)); + } + } + } + private void createNestedMappingIndex(String indexName) throws IOException { XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); From e5a1cd8cbd4e4255fc6244e6a610bf3ff66919a2 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 11 Oct 2023 09:16:03 +0200 Subject: [PATCH 30/48] Unmute and fix IdLoaderTests#testSynthesizeIdMultipleSegments again. (#100625) This time more segments were created than expected, because IWC#maxBufferedDocs was randomily set to a very small number causing more segments to be created then expected. I also run this test again with -Dtests.iters=1024 without failure. So hopefully this test will not fail again because of random test issues. Closes #100580 --- .../test/java/org/elasticsearch/index/mapper/IdLoaderTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index b22d4269c7891..6712d1c40b4ee 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -69,7 +69,6 @@ public void testSynthesizeIdSimple() throws Exception { prepareIndexReader(indexAndForceMerge(routing, docs), verify, false); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100580") public void testSynthesizeIdMultipleSegments() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); @@ -203,6 +202,7 @@ private void prepareIndexReader( IndexWriterConfig config = LuceneTestCase.newIndexWriterConfig(random(), new MockAnalyzer(random())); if (noMergePolicy) { config.setMergePolicy(NoMergePolicy.INSTANCE); + config.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); } Sort sort = new Sort( new SortField(TimeSeriesIdFieldMapper.NAME, SortField.Type.STRING, false), From 370c8266b15f76400ada22833e557dc606d3ec3e Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Wed, 11 Oct 2023 08:18:04 +0100 Subject: [PATCH 31/48] DSL waits for the tsdb time boundaries to lapse (#100470) TSDB indices are expected to receive a large amounts of writes whilst their time bounds are "active" (i.e they include `now`). This ensures TSDB doesn't execute any ingest disruptive operations (like delete, forcemerge, downsample) until the `end_time` for the TSDS backing indices has lapsed. --- docs/changelog/100470.yaml | 6 + .../lifecycle/DataStreamLifecycleService.java | 86 ++++++++--- .../DataStreamLifecycleServiceTests.java | 110 +++++++++++++ ...StreamLifecycleDownsampleDisruptionIT.java | 14 +- .../DataStreamLifecycleDownsampleIT.java | 42 ++++- .../downsample/DataStreamLifecycleDriver.java | 38 ++++- ...StreamLifecycleDownsamplingSecurityIT.java | 144 +++++++++++++----- 7 files changed, 368 insertions(+), 72 deletions(-) create mode 100644 docs/changelog/100470.yaml diff --git a/docs/changelog/100470.yaml b/docs/changelog/100470.yaml new file mode 100644 index 0000000000000..3408ae06f7fe9 --- /dev/null +++ b/docs/changelog/100470.yaml @@ -0,0 +1,6 @@ +pr: 100470 +summary: DSL waits for the tsdb time boundaries to lapse +area: Data streams +type: bug +issues: + - 99696 diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 5d85a199c4e3d..d1ea1b589b5a5 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -63,7 +63,9 @@ import org.elasticsearch.datastreams.lifecycle.downsampling.DeleteSourceAndAddDownsampleToDS; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.snapshots.SnapshotInProgressException; import org.elasticsearch.threadpool.ThreadPool; @@ -71,6 +73,7 @@ import java.io.Closeable; import java.time.Clock; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -140,7 +143,7 @@ public class DataStreamLifecycleService implements ClusterStateListener, Closeab private final ThreadPool threadPool; final ResultDeduplicator transportActionsDeduplicator; final ResultDeduplicator clusterStateChangesDeduplicator; - private final LongSupplier nowSupplier; + private LongSupplier nowSupplier; private final Clock clock; private final DataStreamLifecycleErrorStore errorStore; private volatile boolean isMaster = false; @@ -304,11 +307,24 @@ void run(ClusterState state) { } } } - Set indicesBeingRemoved; + + Set indicesToExcludeForRemainingRun = new HashSet<>(); + // the following indices should not be considered for the remainder of this service run: + // 1) the write index as it's still getting writes and we'll have to roll it over when the conditions are met + // 2) tsds indices that are still within their time bounds (i.e. now < time_series.end_time) - we don't want these indices to be + // deleted, forcemerged, or downsampled as they're still expected to receive large amounts of writes + indicesToExcludeForRemainingRun.add(currentRunWriteIndex); + indicesToExcludeForRemainingRun.addAll( + timeSeriesIndicesStillWithinTimeBounds( + state.metadata(), + getTargetIndices(dataStream, indicesToExcludeForRemainingRun, state.metadata()::index), + nowSupplier + ) + ); + try { - indicesBeingRemoved = maybeExecuteRetention(state, dataStream); + indicesToExcludeForRemainingRun.addAll(maybeExecuteRetention(state, dataStream, indicesToExcludeForRemainingRun)); } catch (Exception e) { - indicesBeingRemoved = Set.of(); // individual index errors would be reported via the API action listener for every delete call // we could potentially record errors at a data stream level and expose it via the _data_stream API? logger.error( @@ -321,13 +337,6 @@ void run(ClusterState state) { ); } - // the following indices should not be considered for the remainder of this service run: - // 1) the write index as it's still getting writes and we'll have to roll it over when the conditions are met - // 2) we exclude any indices that we're in the process of deleting because they'll be gone soon anyway - Set indicesToExcludeForRemainingRun = new HashSet<>(); - indicesToExcludeForRemainingRun.add(currentRunWriteIndex); - indicesToExcludeForRemainingRun.addAll(indicesBeingRemoved); - try { indicesToExcludeForRemainingRun.addAll( maybeExecuteForceMerge(state, getTargetIndices(dataStream, indicesToExcludeForRemainingRun, state.metadata()::index)) @@ -372,6 +381,30 @@ void run(ClusterState state) { ); } + // visible for testing + static Set timeSeriesIndicesStillWithinTimeBounds(Metadata metadata, List targetIndices, LongSupplier nowSupplier) { + Set tsIndicesWithinBounds = new HashSet<>(); + for (Index index : targetIndices) { + IndexMetadata backingIndex = metadata.index(index); + assert backingIndex != null : "the data stream backing indices must exist"; + if (IndexSettings.MODE.get(backingIndex.getSettings()) == IndexMode.TIME_SERIES) { + Instant configuredEndTime = IndexSettings.TIME_SERIES_END_TIME.get(backingIndex.getSettings()); + assert configuredEndTime != null + : "a time series index must have an end time configured but [" + index.getName() + "] does not"; + if (nowSupplier.getAsLong() <= configuredEndTime.toEpochMilli()) { + logger.trace( + "Data stream lifecycle will not perform any operations in this run on time series index [{}] because " + + "its configured [{}] end time has not lapsed", + index.getName(), + configuredEndTime + ); + tsIndicesWithinBounds.add(index); + } + } + } + return tsIndicesWithinBounds; + } + /** * Data stream lifecycle supports configuring multiple rounds of downsampling for each managed index. When attempting to execute * downsampling we iterate through the ordered rounds of downsampling that match an index (ordered ascending according to the `after` @@ -716,11 +749,13 @@ private void maybeExecuteRollover(ClusterState state, DataStream dataStream) { /** * This method sends requests to delete any indices in the datastream that exceed its retention policy. It returns the set of indices * it has sent delete requests for. - * @param state The cluster state from which to get index metadata - * @param dataStream The datastream + * + * @param state The cluster state from which to get index metadata + * @param dataStream The datastream + * @param indicesToExcludeForRemainingRun Indices to exclude from retention even if it would be time for them to be deleted * @return The set of indices that delete requests have been sent for */ - private Set maybeExecuteRetention(ClusterState state, DataStream dataStream) { + private Set maybeExecuteRetention(ClusterState state, DataStream dataStream, Set indicesToExcludeForRemainingRun) { TimeValue retention = getRetentionConfiguration(dataStream); Set indicesToBeRemoved = new HashSet<>(); if (retention != null) { @@ -728,14 +763,16 @@ private Set maybeExecuteRetention(ClusterState state, DataStream dataStre List backingIndicesOlderThanRetention = dataStream.getIndicesPastRetention(metadata::index, nowSupplier); for (Index index : backingIndicesOlderThanRetention) { - indicesToBeRemoved.add(index); - IndexMetadata backingIndex = metadata.index(index); - assert backingIndex != null : "the data stream backing indices must exist"; - - // there's an opportunity here to batch the delete requests (i.e. delete 100 indices / request) - // let's start simple and reevaluate - String indexName = backingIndex.getIndex().getName(); - deleteIndexOnce(indexName, "the lapsed [" + retention + "] retention period"); + if (indicesToExcludeForRemainingRun.contains(index) == false) { + indicesToBeRemoved.add(index); + IndexMetadata backingIndex = metadata.index(index); + assert backingIndex != null : "the data stream backing indices must exist"; + + // there's an opportunity here to batch the delete requests (i.e. delete 100 indices / request) + // let's start simple and reevaluate + String indexName = backingIndex.getIndex().getName(); + deleteIndexOnce(indexName, "the lapsed [" + retention + "] retention period"); + } } } return indicesToBeRemoved; @@ -1227,6 +1264,11 @@ public DataStreamLifecycleErrorStore getErrorStore() { return errorStore; } + // visible for testing + public void setNowSupplier(LongSupplier nowSupplier) { + this.nowSupplier = nowSupplier; + } + /** * This is a ClusterStateTaskListener that writes the force_merge_completed_timestamp into the cluster state. It is meant to run in * STATE_UPDATE_TASK_EXECUTOR. diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index b1679b5fa6701..f1e74a936e781 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.DataStreamLifecycle.Downsampling; import org.elasticsearch.cluster.metadata.DataStreamLifecycle.Downsampling.Round; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexGraveyard; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -59,6 +60,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; @@ -80,6 +82,7 @@ import java.time.Clock; import java.time.Instant; import java.time.ZoneId; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -105,6 +108,7 @@ import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService.TARGET_MERGE_FACTOR_VALUE; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.elasticsearch.test.ClusterServiceUtils.setState; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -246,6 +250,49 @@ public void testRetentionNotExecutedDueToAge() { assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class)); } + public void testRetentionNotExecutedForTSIndicesWithinTimeBounds() { + Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); + // These ranges are on the edge of each other temporal boundaries. + Instant start1 = currentTime.minus(6, ChronoUnit.HOURS); + Instant end1 = currentTime.minus(4, ChronoUnit.HOURS); + Instant start2 = currentTime.minus(4, ChronoUnit.HOURS); + Instant end2 = currentTime.plus(2, ChronoUnit.HOURS); + Instant start3 = currentTime.plus(2, ChronoUnit.HOURS); + Instant end3 = currentTime.plus(4, ChronoUnit.HOURS); + + String dataStreamName = "logs_my-app_prod"; + var clusterState = DataStreamTestHelper.getClusterStateWithDataStream( + dataStreamName, + List.of(Tuple.tuple(start1, end1), Tuple.tuple(start2, end2), Tuple.tuple(start3, end3)) + ); + Metadata.Builder builder = Metadata.builder(clusterState.metadata()); + DataStream dataStream = builder.dataStream(dataStreamName); + builder.put( + new DataStream( + dataStreamName, + dataStream.getIndices(), + dataStream.getGeneration() + 1, + dataStream.getMetadata(), + dataStream.isHidden(), + dataStream.isReplicated(), + dataStream.isSystem(), + dataStream.isAllowCustomRouting(), + dataStream.getIndexMode(), + DataStreamLifecycle.newBuilder().dataRetention(0L).build() + ) + ); + clusterState = ClusterState.builder(clusterState).metadata(builder).build(); + + dataStreamLifecycleService.run(clusterState); + assertThat(clientSeenRequests.size(), is(2)); // rollover the write index and one delete request for the index that's out of the + // TS time bounds + assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class)); + TransportRequest deleteIndexRequest = clientSeenRequests.get(1); + assertThat(deleteIndexRequest, instanceOf(DeleteIndexRequest.class)); + // only the first generation index should be eligible for retention + assertThat(((DeleteIndexRequest) deleteIndexRequest).indices(), is(new String[] { dataStream.getIndices().get(0).getName() })); + } + public void testIlmManagedIndicesAreSkipped() { String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); int numBackingIndices = 3; @@ -1186,6 +1233,69 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc assertThat(error, containsString("resource_already_exists_exception")); } + public void testTimeSeriesIndicesStillWithinTimeBounds() { + Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); + // These ranges are on the edge of each other temporal boundaries. + Instant start1 = currentTime.minus(6, ChronoUnit.HOURS); + Instant end1 = currentTime.minus(4, ChronoUnit.HOURS); + Instant start2 = currentTime.minus(4, ChronoUnit.HOURS); + Instant end2 = currentTime.plus(2, ChronoUnit.HOURS); + Instant start3 = currentTime.plus(2, ChronoUnit.HOURS); + Instant end3 = currentTime.plus(4, ChronoUnit.HOURS); + + String dataStreamName = "logs_my-app_prod"; + var clusterState = DataStreamTestHelper.getClusterStateWithDataStream( + dataStreamName, + List.of(Tuple.tuple(start1, end1), Tuple.tuple(start2, end2), Tuple.tuple(start3, end3)) + ); + DataStream dataStream = clusterState.getMetadata().dataStreams().get(dataStreamName); + + { + // test for an index for which `now` is outside its time bounds + Index firstGenIndex = dataStream.getIndices().get(0); + Set indices = DataStreamLifecycleService.timeSeriesIndicesStillWithinTimeBounds( + clusterState.metadata(), + // the end_time for the first generation has lapsed + List.of(firstGenIndex), + currentTime::toEpochMilli + ); + assertThat(indices.size(), is(0)); + } + + { + Set indices = DataStreamLifecycleService.timeSeriesIndicesStillWithinTimeBounds( + clusterState.metadata(), + // the end_time for the first generation has lapsed, but the other 2 generations are still within bounds + dataStream.getIndices(), + currentTime::toEpochMilli + ); + assertThat(indices.size(), is(2)); + assertThat(indices, containsInAnyOrder(dataStream.getIndices().get(1), dataStream.getIndices().get(2))); + } + + { + // non time_series indices are not within time bounds (they don't have any) + IndexMetadata indexMeta = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30)) + .settings( + Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1) + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) + .build() + ) + .build(); + + Metadata newMetadata = Metadata.builder(clusterState.metadata()).put(indexMeta, true).build(); + + Set indices = DataStreamLifecycleService.timeSeriesIndicesStillWithinTimeBounds( + newMetadata, + List.of(indexMeta.getIndex()), + currentTime::toEpochMilli + ); + assertThat(indices.size(), is(0)); + } + } + /* * Creates a test cluster state with the given indexName. If customDataStreamLifecycleMetadata is not null, it is added as the value * of the index's custom metadata named "data_stream_lifecycle". diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java index 166f41fa063ca..5bd20ce51a57d 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java @@ -35,6 +35,7 @@ import java.util.function.Consumer; import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.getBackingIndices; +import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.putTSDBIndexTemplate; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -77,8 +78,19 @@ public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception { ) ) .build(); - int indexedDocs = DataStreamLifecycleDriver.setupDataStreamAndIngestDocs(client(), dataStreamName, lifecycle, DOC_COUNT); + DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs( + client(), + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + DOC_COUNT, + "1990-09-09T18:00:00" + ); + // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with + // testing so DSL doesn't have to wait for the end_time to lapse) + putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); // DSL runs every second and it has to tail forcemerge the index (2 seconds) and mark it as read-only (2s) before it starts diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java index cf5e79982d836..c38ed182abc64 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java @@ -32,6 +32,7 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.getBackingIndices; +import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.putTSDBIndexTemplate; import static org.hamcrest.Matchers.is; public class DataStreamLifecycleDownsampleIT extends ESIntegTestCase { @@ -68,7 +69,15 @@ public void testDownsampling() throws Exception { ) .build(); - DataStreamLifecycleDriver.setupDataStreamAndIngestDocs(client(), dataStreamName, lifecycle, DOC_COUNT); + DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs( + client(), + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + DOC_COUNT, + "1990-09-09T18:00:00" + ); List backingIndices = getBackingIndices(client(), dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); @@ -85,6 +94,9 @@ public void testDownsampling() throws Exception { witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); } }); + // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with + // testing so DSL doesn't have to wait for the end_time to lapse) + putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); @@ -127,7 +139,15 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception ) ) .build(); - DataStreamLifecycleDriver.setupDataStreamAndIngestDocs(client(), dataStreamName, lifecycle, DOC_COUNT); + DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs( + client(), + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + DOC_COUNT, + "1990-09-09T18:00:00" + ); List backingIndices = getBackingIndices(client(), dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); @@ -144,7 +164,9 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); } }); - + // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with + // testing so DSL doesn't have to wait for the end_time to lapse) + putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { @@ -182,7 +204,15 @@ public void testUpdateDownsampleRound() throws Exception { ) .build(); - DataStreamLifecycleDriver.setupDataStreamAndIngestDocs(client(), dataStreamName, lifecycle, DOC_COUNT); + DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs( + client(), + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + DOC_COUNT, + "1990-09-09T18:00:00" + ); List backingIndices = getBackingIndices(client(), dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); @@ -199,7 +229,9 @@ public void testUpdateDownsampleRound() throws Exception { witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); } }); - + // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with + // testing so DSL doesn't have to wait for the end_time to lapse) + putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java index be71c546a9d4c..d704f3bf93c54 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java @@ -37,6 +37,8 @@ import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -62,10 +64,17 @@ public class DataStreamLifecycleDriver { public static final String FIELD_DIMENSION_2 = "dimension_long"; public static final String FIELD_METRIC_COUNTER = "counter"; - public static int setupDataStreamAndIngestDocs(Client client, String dataStreamName, DataStreamLifecycle lifecycle, int docCount) - throws IOException { - putTSDBIndexTemplate(client, dataStreamName + "*", lifecycle); - return indexDocuments(client, dataStreamName, docCount); + public static int setupTSDBDataStreamAndIngestDocs( + Client client, + String dataStreamName, + @Nullable String startTime, + @Nullable String endTime, + DataStreamLifecycle lifecycle, + int docCount, + String firstDocTimestamp + ) throws IOException { + putTSDBIndexTemplate(client, dataStreamName + "*", startTime, endTime, lifecycle); + return indexDocuments(client, dataStreamName, docCount, firstDocTimestamp); } public static List getBackingIndices(Client client, String dataStreamName) { @@ -76,10 +85,24 @@ public static List getBackingIndices(Client client, String dataStreamNam return getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().stream().map(Index::getName).toList(); } - private static void putTSDBIndexTemplate(Client client, String pattern, DataStreamLifecycle lifecycle) throws IOException { + public static void putTSDBIndexTemplate( + Client client, + String pattern, + @Nullable String startTime, + @Nullable String endTime, + DataStreamLifecycle lifecycle + ) throws IOException { Settings.Builder settings = indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1)); + if (Strings.hasText(startTime)) { + settings.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime); + } + + if (Strings.hasText(endTime)) { + settings.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime); + } + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties"); mapping.startObject(FIELD_TIMESTAMP).field("type", "date").endObject(); @@ -129,9 +152,10 @@ private static void putComposableIndexTemplate( client.execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); } - private static int indexDocuments(Client client, String dataStreamName, int docCount) { + private static int indexDocuments(Client client, String dataStreamName, int docCount, String firstDocTimestamp) { final Supplier sourceSupplier = () -> { - final String ts = randomDateForInterval(new DateHistogramInterval("1s"), System.currentTimeMillis()); + long startTime = LocalDateTime.parse(firstDocTimestamp).atZone(ZoneId.of("UTC")).toInstant().toEpochMilli(); + final String ts = randomDateForInterval(new DateHistogramInterval("1s"), startTime); double counterValue = DATE_FORMATTER.parseMillis(ts); final List dimensionValues = new ArrayList<>(5); for (int j = 0; j < randomIntBetween(1, 5); j++) { diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java index 8311d0f613175..cdddd0a5e5fe0 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java @@ -56,6 +56,10 @@ import org.elasticsearch.xpack.wildcard.Wildcard; import java.io.IOException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -130,18 +134,15 @@ public void testDownsamplingAuthorized() throws Exception { ) .build(); - setupDataStreamAndIngestDocs(client(), dataStreamName, lifecycle, 10_000); - waitAndAssertDownsamplingCompleted(dataStreamName); - } - - @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging") - public void testSystemDataStreamConfigurationWithDownsampling() throws Exception { - String dataStreamName = SystemDataStreamWithDownsamplingConfigurationPlugin.SYSTEM_DATA_STREAM_NAME; - indexDocuments(client(), dataStreamName, 10_000); - waitAndAssertDownsamplingCompleted(dataStreamName); - } - - private void waitAndAssertDownsamplingCompleted(String dataStreamName) throws Exception { + setupDataStreamAndIngestDocs( + client(), + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + 10_000, + "1990-09-09T18:00:00" + ); List backingIndices = getDataStreamBackingIndices(dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0).getName(); String firstRoundDownsamplingIndex = "downsample-5m-" + firstGenerationBackingIndex; @@ -158,6 +159,9 @@ private void waitAndAssertDownsamplingCompleted(String dataStreamName) throws Ex } }); + // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with + // testing so DSL doesn't have to wait for the end_time to lapse) + putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { @@ -188,6 +192,52 @@ private void waitAndAssertDownsamplingCompleted(String dataStreamName) throws Ex }, 30, TimeUnit.SECONDS); } + @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging") + public void testSystemDataStreamConfigurationWithDownsampling() throws Exception { + String dataStreamName = SystemDataStreamWithDownsamplingConfigurationPlugin.SYSTEM_DATA_STREAM_NAME; + indexDocuments(client(), dataStreamName, 10_000, Instant.now().toEpochMilli()); + List backingIndices = getDataStreamBackingIndices(dataStreamName); + String firstGenerationBackingIndex = backingIndices.get(0).getName(); + String secondRoundDownsamplingIndex = "downsample-10m-" + firstGenerationBackingIndex; + + Set witnessedDownsamplingIndices = new HashSet<>(); + clusterService().addListener(event -> { + if (event.indicesCreated().contains(secondRoundDownsamplingIndex)) { + witnessedDownsamplingIndices.add(secondRoundDownsamplingIndex); + } + }); + + DataStreamLifecycleService masterDataStreamLifecycleService = internalCluster().getCurrentMasterNodeInstance( + DataStreamLifecycleService.class + ); + try { + // we can't update the index template backing a system data stream, so we run DSL "in the future" + // this means that only one round of downsampling will execute due to an optimisation we have in DSL to execute the last + // matching round + masterDataStreamLifecycleService.setNowSupplier(() -> Instant.now().plus(50, ChronoUnit.DAYS).toEpochMilli()); + client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); + + assertBusy(() -> { + assertNoAuthzErrors(); + assertThat(witnessedDownsamplingIndices.contains(secondRoundDownsamplingIndex), is(true)); + }, 30, TimeUnit.SECONDS); + + assertBusy(() -> { + assertNoAuthzErrors(); + List dsBackingIndices = getDataStreamBackingIndices(dataStreamName); + + assertThat(dsBackingIndices.size(), is(2)); + String writeIndex = dsBackingIndices.get(1).getName(); + assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); + // the last downsampling round must remain in the data stream + assertThat(dsBackingIndices.get(0).getName(), is(secondRoundDownsamplingIndex)); + }, 30, TimeUnit.SECONDS); + } finally { + // restore a real nowSupplier so other tests running against this cluster succeed + masterDataStreamLifecycleService.setNowSupplier(() -> Instant.now().toEpochMilli()); + } + } + private Map collectErrorsFromStoreAsMap() { Iterable lifecycleServices = internalCluster().getInstances(DataStreamLifecycleService.class); Map indicesAndErrors = new HashMap<>(); @@ -221,15 +271,36 @@ private void assertNoAuthzErrors() { } } - private void setupDataStreamAndIngestDocs(Client client, String dataStreamName, DataStreamLifecycle lifecycle, int docCount) - throws IOException { - putTSDBIndexTemplate(client, dataStreamName + "*", lifecycle); - indexDocuments(client, dataStreamName, docCount); + private void setupDataStreamAndIngestDocs( + Client client, + String dataStreamName, + @Nullable String startTime, + @Nullable String endTime, + DataStreamLifecycle lifecycle, + int docCount, + String firstDocTimestamp + ) throws IOException { + putTSDBIndexTemplate(client, dataStreamName + "*", startTime, endTime, lifecycle); + long startTimestamp = LocalDateTime.parse(firstDocTimestamp).atZone(ZoneId.of("UTC")).toInstant().toEpochMilli(); + indexDocuments(client, dataStreamName, docCount, startTimestamp); } - private void putTSDBIndexTemplate(Client client, String pattern, DataStreamLifecycle lifecycle) throws IOException { + private void putTSDBIndexTemplate( + Client client, + String pattern, + @Nullable String startTime, + @Nullable String endTime, + DataStreamLifecycle lifecycle + ) throws IOException { Settings.Builder settings = indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1)); + if (Strings.hasText(startTime)) { + settings.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime); + } + + if (Strings.hasText(endTime)) { + settings.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime); + } CompressedXContent mapping = getTSDBMappings(); putComposableIndexTemplate(client, "id1", mapping, List.of(pattern), settings.build(), null, lifecycle); } @@ -275,9 +346,9 @@ private void putComposableIndexTemplate( client.execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); } - private void indexDocuments(Client client, String dataStreamName, int docCount) { + private void indexDocuments(Client client, String dataStreamName, int docCount, long startTime) { final Supplier sourceSupplier = () -> { - final String ts = randomDateForInterval(new DateHistogramInterval("1s"), System.currentTimeMillis()); + final String ts = randomDateForInterval(new DateHistogramInterval("1s"), startTime); double counterValue = DATE_FORMATTER.parseMillis(ts); final List dimensionValues = new ArrayList<>(5); for (int j = 0; j < randomIntBetween(1, 5); j++) { @@ -336,27 +407,26 @@ private void bulkIndex(Client client, String dataStreamName, Supplier getSystemDataStreamDescriptors() { - DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder() - .downsampling( - new DataStreamLifecycle.Downsampling( - List.of( - new DataStreamLifecycle.Downsampling.Round( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), - new DataStreamLifecycle.Downsampling.Round( - TimeValue.timeValueSeconds(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + public static final DataStreamLifecycle LIFECYCLE = DataStreamLifecycle.newBuilder() + .downsampling( + new DataStreamLifecycle.Downsampling( + List.of( + new DataStreamLifecycle.Downsampling.Round( + TimeValue.timeValueMillis(0), + new DownsampleConfig(new DateHistogramInterval("5m")) + ), + new DataStreamLifecycle.Downsampling.Round( + TimeValue.timeValueSeconds(10), + new DownsampleConfig(new DateHistogramInterval("10m")) ) ) ) - .build(); + ) + .build(); + static final String SYSTEM_DATA_STREAM_NAME = ".fleet-actions-results"; + @Override + public Collection getSystemDataStreamDescriptors() { Settings.Builder settings = indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1)); @@ -368,7 +438,7 @@ public Collection getSystemDataStreamDescriptors() { SystemDataStreamDescriptor.Type.EXTERNAL, new ComposableIndexTemplate( List.of(SYSTEM_DATA_STREAM_NAME), - new Template(settings.build(), getTSDBMappings(), null, lifecycle), + new Template(settings.build(), getTSDBMappings(), null, LIFECYCLE), null, null, null, From 616960c2cf4bf7179499f729d208111493213085 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Wed, 11 Oct 2023 09:32:57 +0100 Subject: [PATCH 32/48] Manually update the min CCS version to the version used by 8.11 (#100582) The build automation for 8.11 should have done this, but it didn't work at the time --- server/src/main/java/org/elasticsearch/TransportVersions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 6267fb3b86ae4..5d51a7959b5fa 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -198,7 +198,7 @@ static TransportVersion def(int id) { * Reference to the minimum transport version that can be used with CCS. * This should be the transport version used by the previous minor release. */ - public static final TransportVersion MINIMUM_CCS_VERSION = V_8_500_061; + public static final TransportVersion MINIMUM_CCS_VERSION = ML_PACKAGE_LOADER_PLATFORM_ADDED; static final NavigableMap VERSION_IDS = getAllVersionIds(TransportVersions.class); From 29e3d2829b365e011346271ceaa7d5f2746318fe Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 11 Oct 2023 11:48:04 +0300 Subject: [PATCH 33/48] ESQL: fix non-null value being returned for unsupported data types in ValueSources (#100656) * Refine (and fix) all cases where an unsupported data type field's values are returned from SourceValues. * Improve unsupported data types handling in TopN --- docs/changelog/100656.yaml | 6 ++ .../compute/lucene/ValueSources.java | 26 +++--- .../topn/DefaultSortableTopNEncoder.java | 2 +- .../compute/operator/topn/TopNEncoder.java | 5 + .../topn/UnsupportedTypesTopNEncoder.java | 45 +++++++++ .../resources/rest-api-spec/test/40_tsdb.yml | 6 ++ .../test/40_unsupported_types.yml | 93 +++++++++++++++++++ .../rest-api-spec/test/50_index_patterns.yml | 4 + .../esql/planner/LocalExecutionPlanner.java | 2 + 9 files changed, 175 insertions(+), 14 deletions(-) create mode 100644 docs/changelog/100656.yaml create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/UnsupportedTypesTopNEncoder.java diff --git a/docs/changelog/100656.yaml b/docs/changelog/100656.yaml new file mode 100644 index 0000000000000..1ee9a2ad0e47a --- /dev/null +++ b/docs/changelog/100656.yaml @@ -0,0 +1,6 @@ +pr: 100656 +summary: "ESQL: fix non-null value being returned for unsupported data types in `ValueSources`" +area: ES|QL +type: bug +issues: + - 100048 diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java index e5ce5436990b7..29a539b1e068e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java @@ -69,6 +69,18 @@ public static List sources( sources.add(new ValueSourceInfo(new NullValueSourceType(), new NullValueSource(), elementType, ctx.getIndexReader())); continue; // the field does not exist in this context } + if (asUnsupportedSource) { + sources.add( + new ValueSourceInfo( + new UnsupportedValueSourceType(fieldType.typeName()), + new UnsupportedValueSource(null), + elementType, + ctx.getIndexReader() + ) + ); + HeaderWarning.addWarning("Field [{}] cannot be retrieved, it is unsupported or not indexed; returning null", fieldName); + continue; + } if (fieldType.hasDocValues() == false) { // MatchOnlyTextFieldMapper class lives in the mapper-extras module. We use string equality @@ -99,19 +111,7 @@ public static List sources( var fieldContext = new FieldContext(fieldName, fieldData, fieldType); var vsType = fieldData.getValuesSourceType(); var vs = vsType.getField(fieldContext, null); - - if (asUnsupportedSource) { - sources.add( - new ValueSourceInfo( - new UnsupportedValueSourceType(fieldType.typeName()), - new UnsupportedValueSource(vs), - elementType, - ctx.getIndexReader() - ) - ); - } else { - sources.add(new ValueSourceInfo(vsType, vs, elementType, ctx.getIndexReader())); - } + sources.add(new ValueSourceInfo(vsType, vs, elementType, ctx.getIndexReader())); } return sources; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java index 8634d87e2932f..6ccde6b76ce13 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java @@ -23,7 +23,7 @@ public BytesRef decodeBytesRef(BytesRef bytes, BytesRef scratch) { @Override public String toString() { - return "DefaultUnsortable"; + return "DefaultSortable"; } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java index 2d8f2666ff2f2..f1fb7cb7736c5 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java @@ -41,6 +41,11 @@ public interface TopNEncoder { */ VersionTopNEncoder VERSION = new VersionTopNEncoder(); + /** + * Placeholder encoder for unsupported data types. + */ + UnsupportedTypesTopNEncoder UNSUPPORTED = new UnsupportedTypesTopNEncoder(); + void encodeLong(long value, BreakingBytesRefBuilder bytesRefBuilder); long decodeLong(BytesRef bytes); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/UnsupportedTypesTopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/UnsupportedTypesTopNEncoder.java new file mode 100644 index 0000000000000..d80d70970409e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/UnsupportedTypesTopNEncoder.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator.topn; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; + +/** + * TopNEncoder for data types that are unsupported. This is just a placeholder class, reaching the encode/decode methods here is a bug. + * + * While this class is needed to build the TopNOperator value and key extractors infrastructure, encoding/decoding is needed + * when actually sorting on a field (which shouldn't be possible for unsupported data types) using key extractors, or when encoding/decoding + * unsupported data types fields values (which should always be "null" by convention) using value extractors. + */ +class UnsupportedTypesTopNEncoder extends SortableTopNEncoder { + @Override + public int encodeBytesRef(BytesRef value, BreakingBytesRefBuilder bytesRefBuilder) { + throw new UnsupportedOperationException("Encountered a bug; trying to encode an unsupported data type value for TopN"); + } + + @Override + public BytesRef decodeBytesRef(BytesRef bytes, BytesRef scratch) { + throw new UnsupportedOperationException("Encountered a bug; trying to decode an unsupported data type value for TopN"); + } + + @Override + public String toString() { + return "UnsupportedTypesTopNEncoder"; + } + + @Override + public TopNEncoder toSortable() { + return this; + } + + @Override + public TopNEncoder toUnsortable() { + return this; + } +} diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml index 14ae1ff98d8ad..895a1718b2cbc 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml @@ -106,6 +106,8 @@ setup: --- load everything: - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" esql.query: body: query: 'from test' @@ -156,6 +158,8 @@ filter on counter: --- from doc with aggregate_metric_double: - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" esql.query: body: query: 'from test2' @@ -183,6 +187,8 @@ stats on aggregate_metric_double: --- from index pattern unsupported counter: - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" esql.query: body: query: 'FROM test*' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml index 44af9559598ab..ad0c7b516fde1 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml @@ -263,3 +263,96 @@ unsupported: - match: { columns.0.name: shape } - match: { columns.0.type: unsupported } - length: { values: 0 } + +--- +unsupported with sort: + - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + esql.query: + body: + query: 'from test | sort some_doc.bar' + + - match: { columns.0.name: aggregate_metric_double } + - match: { columns.0.type: unsupported } + - match: { columns.1.name: binary } + - match: { columns.1.type: unsupported } + - match: { columns.2.name: completion } + - match: { columns.2.type: unsupported } + - match: { columns.3.name: date_nanos } + - match: { columns.3.type: unsupported } + - match: { columns.4.name: date_range } + - match: { columns.4.type: unsupported } + - match: { columns.5.name: dense_vector } + - match: { columns.5.type: unsupported } + - match: { columns.6.name: double_range } + - match: { columns.6.type: unsupported } + - match: { columns.7.name: float_range } + - match: { columns.7.type: unsupported } + - match: { columns.8.name: geo_point } + - match: { columns.8.type: unsupported } + - match: { columns.9.name: geo_point_alias } + - match: { columns.9.type: unsupported } + - match: { columns.10.name: histogram } + - match: { columns.10.type: unsupported } + - match: { columns.11.name: integer_range } + - match: { columns.11.type: unsupported } + - match: { columns.12.name: ip_range } + - match: { columns.12.type: unsupported } + - match: { columns.13.name: long_range } + - match: { columns.13.type: unsupported } + - match: { columns.14.name: match_only_text } + - match: { columns.14.type: text } + - match: { columns.15.name: name } + - match: { columns.15.type: keyword } + - match: { columns.16.name: rank_feature } + - match: { columns.16.type: unsupported } + - match: { columns.17.name: rank_features } + - match: { columns.17.type: unsupported } + - match: { columns.18.name: search_as_you_type } + - match: { columns.18.type: unsupported } + - match: { columns.19.name: search_as_you_type._2gram } + - match: { columns.19.type: unsupported } + - match: { columns.20.name: search_as_you_type._3gram } + - match: { columns.20.type: unsupported } + - match: { columns.21.name: search_as_you_type._index_prefix } + - match: { columns.21.type: unsupported } + - match: { columns.22.name: shape } + - match: { columns.22.type: unsupported } + - match: { columns.23.name: some_doc.bar } + - match: { columns.23.type: long } + - match: { columns.24.name: some_doc.foo } + - match: { columns.24.type: keyword } + - match: { columns.25.name: text } + - match: { columns.25.type: text } + - match: { columns.26.name: token_count } + - match: { columns.26.type: integer } + + - length: { values: 1 } + - match: { values.0.0: null } + - match: { values.0.1: null } + - match: { values.0.2: null } + - match: { values.0.3: null } + - match: { values.0.4: null } + - match: { values.0.5: null } + - match: { values.0.6: null } + - match: { values.0.7: null } + - match: { values.0.8: null } + - match: { values.0.9: null } + - match: { values.0.10: null } + - match: { values.0.11: null } + - match: { values.0.12: null } + - match: { values.0.13: null } + - match: { values.0.14: "foo bar baz" } + - match: { values.0.15: Alice } + - match: { values.0.16: null } + - match: { values.0.17: null } + - match: { values.0.18: null } + - match: { values.0.19: null } + - match: { values.0.20: null } + - match: { values.0.21: null } + - match: { values.0.22: null } + - match: { values.0.23: 12 } + - match: { values.0.24: xy } + - match: { values.0.25: "foo bar" } + - match: { values.0.26: 3 } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml index 280a32aa10cd3..ff327b2592c88 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml @@ -267,6 +267,8 @@ disjoint_mappings: --- same_name_different_type: + - skip: + features: allowed_warnings_regex - do: indices.create: index: test1 @@ -307,6 +309,8 @@ same_name_different_type: - { "message": 2 } - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" esql.query: body: query: 'from test1,test2 ' diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 1c26de4a599f5..b86072e1b6da0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -380,6 +380,8 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte case "version" -> TopNEncoder.VERSION; case "boolean", "null", "byte", "short", "integer", "long", "double", "float", "half_float", "datetime", "date_period", "time_duration", "object", "nested", "scaled_float", "unsigned_long", "_doc" -> TopNEncoder.DEFAULT_SORTABLE; + // unsupported fields are encoded as BytesRef, we'll use the same encoder; all values should be null at this point + case "unsupported" -> TopNEncoder.UNSUPPORTED; default -> throw new EsqlIllegalArgumentException("No TopN sorting encoder for type " + inverse.get(channel).type()); }; } From c4e55ab14c9335c0fff343e94895fa0447e663d9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 11 Oct 2023 09:54:09 +0100 Subject: [PATCH 34/48] Fix interruption of markAllocationIdAsInSync (#100610) `IndexShard#markAllocationIdAsInSync` is interruptible because it may block the thread on a monitor waiting for the local checkpoint to advance, but we lost the ability to interrupt it on a recovery cancellation in #95270. Closes #96578 Closes #100589 --- docs/changelog/100610.yaml | 7 ++ .../indices/recovery/IndexRecoveryIT.java | 108 ++++++++++++++++++ .../recovery/RecoverySourceHandler.java | 5 +- .../org/elasticsearch/test/ESTestCase.java | 13 ++- 4 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/100610.yaml diff --git a/docs/changelog/100610.yaml b/docs/changelog/100610.yaml new file mode 100644 index 0000000000000..7423ce9225868 --- /dev/null +++ b/docs/changelog/100610.yaml @@ -0,0 +1,7 @@ +pr: 100610 +summary: Fix interruption of `markAllocationIdAsInSync` +area: Recovery +type: bug +issues: + - 96578 + - 100589 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index d3aed4a3e2bf2..f556486795c2a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -35,11 +35,15 @@ import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; @@ -70,6 +74,8 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.gateway.ReplicaShardAllocatorIT; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; @@ -85,6 +91,7 @@ import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.index.seqno.RetentionLeases; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.index.shard.GlobalCheckpointListeners; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.Store; @@ -122,7 +129,9 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -132,6 +141,7 @@ import static java.util.stream.Collectors.toList; import static org.elasticsearch.action.DocWriteResponse.Result.CREATED; import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED; +import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener; import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING; import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED; import static org.elasticsearch.node.RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING; @@ -1688,6 +1698,104 @@ public void testWaitForClusterStateToBeAppliedOnSourceNode() throws Exception { } } + public void testDeleteIndexDuringFinalization() throws Exception { + internalCluster().startMasterOnlyNode(); + final var primaryNode = internalCluster().startDataOnlyNode(); + String indexName = "test-index"; + createIndex(indexName, indexSettings(1, 0).build()); + ensureGreen(indexName); + final List indexRequests = IntStream.range(0, between(10, 500)) + .mapToObj(n -> client().prepareIndex(indexName).setSource("foo", "bar")) + .toList(); + indexRandom(randomBoolean(), true, true, indexRequests); + assertThat(indicesAdmin().prepareFlush(indexName).get().getFailedShards(), equalTo(0)); + + final var replicaNode = internalCluster().startDataOnlyNode(); + + final SubscribableListener recoveryCompleteListener = new SubscribableListener<>(); + final PlainActionFuture deleteListener = new PlainActionFuture<>(); + + final var threadPool = internalCluster().clusterService().threadPool(); + + final var indexId = internalCluster().clusterService().state().routingTable().index(indexName).getIndex(); + final var primaryIndexShard = internalCluster().getInstance(IndicesService.class, primaryNode) + .indexServiceSafe(indexId) + .getShard(0); + final var globalCheckpointBeforeRecovery = primaryIndexShard.getLastSyncedGlobalCheckpoint(); + + final var replicaNodeTransportService = asInstanceOf( + MockTransportService.class, + internalCluster().getInstance(TransportService.class, replicaNode) + ); + replicaNodeTransportService.addRequestHandlingBehavior( + PeerRecoveryTargetService.Actions.TRANSLOG_OPS, + (handler, request, channel, task) -> handler.messageReceived( + request, + new TestTransportChannel(ActionTestUtils.assertNoFailureListener(response -> { + // Process the TRANSLOG_OPS response on the replica (avoiding failing it due to a concurrent delete) but + // before sending the response back send another document to the primary, advancing the GCP to prevent the replica + // being marked as in-sync (NB below we delay the replica write until after the index is deleted) + client().prepareIndex(indexName).setSource("foo", "baz").execute(ActionListener.noop()); + + primaryIndexShard.addGlobalCheckpointListener( + globalCheckpointBeforeRecovery + 1, + new GlobalCheckpointListeners.GlobalCheckpointListener() { + @Override + public Executor executor() { + return EsExecutors.DIRECT_EXECUTOR_SERVICE; + } + + @Override + public void accept(long globalCheckpoint, Exception e) { + assertNull(e); + + // Now the GCP has advanced the replica won't be marked in-sync so respond to the TRANSLOG_OPS request + // to start recovery finalization + try { + channel.sendResponse(response); + } catch (IOException ex) { + fail(ex); + } + + // Wait a short while for finalization to block on advancing the replica's GCP and then delete the index + threadPool.schedule( + () -> client().admin().indices().prepareDelete(indexName).execute(deleteListener), + TimeValue.timeValueMillis(100), + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + } + }, + TimeValue.timeValueSeconds(10) + ); + })), + task + ) + ); + + // delay the delivery of the replica write until the end of the test so the replica never becomes in-sync + replicaNodeTransportService.addRequestHandlingBehavior( + BulkAction.NAME + "[s][r]", + (handler, request, channel, task) -> recoveryCompleteListener.addListener( + assertNoFailureListener(ignored -> handler.messageReceived(request, channel, task)) + ) + ); + + // Create the replica to trigger the whole process + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings(indexName) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) + ); + + // Wait for the index to be deleted + assertTrue(deleteListener.get(20, TimeUnit.SECONDS).isAcknowledged()); + + final var peerRecoverySourceService = internalCluster().getInstance(PeerRecoverySourceService.class, primaryNode); + assertBusy(() -> assertEquals(0, peerRecoverySourceService.numberOfOngoingRecoveries())); + recoveryCompleteListener.onResponse(null); + } + private void assertGlobalCheckpointIsStableAndSyncedInAllNodes(String indexName, List nodes, int shard) throws Exception { assertThat(nodes, is(not(empty()))); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index fc5df1a4aa282..81bc226102f62 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.util.CancellableThreads; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; @@ -426,7 +425,7 @@ public void onFailure(Exception e) { } static void runUnderPrimaryPermit( - CheckedRunnable action, + Runnable action, IndexShard primary, CancellableThreads cancellableThreads, ActionListener listener @@ -1260,7 +1259,7 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis */ final SubscribableListener markInSyncStep = new SubscribableListener<>(); runUnderPrimaryPermit( - () -> shard.markAllocationIdAsInSync(request.targetAllocationId(), targetLocalCheckpoint), + () -> cancellableThreads.execute(() -> shard.markAllocationIdAsInSync(request.targetAllocationId(), targetLocalCheckpoint)), shard, cancellableThreads, markInSyncStep diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 540ef4cf1027b..9ccfbd2e25ca6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -114,6 +114,7 @@ import org.elasticsearch.xcontent.XContentParser.Token; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -2036,7 +2037,17 @@ protected static boolean isTurkishLocale() { || Locale.getDefault().getLanguage().equals(new Locale("az").getLanguage()); } - public static void fail(Throwable t, String msg, Object... args) { + public static T fail(Throwable t, String msg, Object... args) { throw new AssertionError(org.elasticsearch.common.Strings.format(msg, args), t); } + + public static T fail(Throwable t) { + return fail(t, "unexpected"); + } + + @SuppressWarnings("unchecked") + public static T asInstanceOf(Class clazz, Object o) { + assertThat(o, Matchers.instanceOf(clazz)); + return (T) o; + } } From 5dc7cccf1368061d27689f0335d3cfa17ab1f57c Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 11 Oct 2023 12:00:52 +0300 Subject: [PATCH 35/48] Fix remaining logs tests (#100407) In this PR we convert the remaining 3 yaml logs tests to java rest tests. This should remove all the flaky tests from the data stream yaml suite that's why we are reenabling it. Closes: https://github.com/elastic/elasticsearch/issues/99911 Closes: https://github.com/elastic/elasticsearch/issues/97795 --- .../datastreams/EcsLogsDataStreamIT.java | 433 ++++++++++++++++++ .../datastreams/LogsDataStreamIT.java | 408 ++++++++++++++++- .../resources/ecs-logs/es-agent-ecs-log.json | 118 +++++ .../DataStreamsClientYamlTestSuiteIT.java | 2 - .../data_stream/230_logs_message_pipeline.yml | 114 ----- .../data_stream/240_logs_ecs_mappings.yml | 406 ---------------- .../data_stream/250_logs_no_subobjects.yml | 218 --------- .../test/data_stream/lifecycle/20_basic.yml | 3 + 8 files changed, 946 insertions(+), 756 deletions(-) create mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java create mode 100644 modules/data-streams/src/javaRestTest/resources/ecs-logs/es-agent-ecs-log.json delete mode 100644 modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/230_logs_message_pipeline.yml delete mode 100644 modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml delete mode 100644 modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java new file mode 100644 index 0000000000000..7de4ed2f2843c --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java @@ -0,0 +1,433 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.core.PathUtils; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.datastreams.LogsDataStreamIT.createDataStream; +import static org.elasticsearch.datastreams.LogsDataStreamIT.getMappingProperties; +import static org.elasticsearch.datastreams.LogsDataStreamIT.getValueFromPath; +import static org.elasticsearch.datastreams.LogsDataStreamIT.getWriteBackingIndex; +import static org.elasticsearch.datastreams.LogsDataStreamIT.indexDoc; +import static org.elasticsearch.datastreams.LogsDataStreamIT.searchDocs; +import static org.elasticsearch.datastreams.LogsDataStreamIT.waitForLogs; +import static org.hamcrest.Matchers.is; + +public class EcsLogsDataStreamIT extends DisabledSecurityDataStreamTestCase { + + private static final String DATA_STREAM_NAME = "logs-generic-default"; + private RestClient client; + private String backingIndex; + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + + { + Request request = new Request("PUT", "/_ingest/pipeline/logs@custom"); + request.setJsonEntity(""" + { + "processors": [ + { + "pipeline" : { + "name": "logs@json-message", + "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" + } + } + ] + } + """); + assertOK(client.performRequest(request)); + } + createDataStream(client, DATA_STREAM_NAME); + backingIndex = getWriteBackingIndex(client, DATA_STREAM_NAME); + } + + @After + public void cleanUp() throws IOException { + adminClient().performRequest(new Request("DELETE", "_data_stream/*")); + } + + @SuppressWarnings("unchecked") + public void testElasticAgentLogEcsMappings() throws Exception { + { + Path path = PathUtils.get(Thread.currentThread().getContextClassLoader().getResource("ecs-logs/es-agent-ecs-log.json").toURI()); + String agentLog = Files.readString(path); + indexDoc(client, DATA_STREAM_NAME, agentLog); + List results = searchDocs(client, DATA_STREAM_NAME, """ + { + "query": { + "term": { + "test": { + "value": "elastic-agent-log" + } + } + }, + "fields": ["message"] + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + Map fields = ((Map>) results.get(0)).get("fields"); + + // timestamp from deserialized JSON message field should win + assertThat(source.get("@timestamp"), is("2023-05-16T13:49:40.374Z")); + assertThat( + ((Map>) source.get("kubernetes")).get("pod").get("name"), + is("elastic-agent-managed-daemonset-jwktj") + ); + // expecting the extracted message from within the original JSON-formatted message + assertThat(((List) fields.get("message")).get(0), is("Non-zero metrics in the last 30s")); + + Map properties = getMappingProperties(client, backingIndex); + assertThat(getValueFromPath(properties, List.of("@timestamp", "type")), is("date")); + assertThat(getValueFromPath(properties, List.of("message", "type")), is("match_only_text")); + assertThat( + getValueFromPath(properties, List.of("kubernetes", "properties", "pod", "properties", "name", "type")), + is("keyword") + ); + assertThat(getValueFromPath(properties, List.of("kubernetes", "properties", "pod", "properties", "ip", "type")), is("ip")); + assertThat(getValueFromPath(properties, List.of("kubernetes", "properties", "pod", "properties", "test_ip", "type")), is("ip")); + assertThat( + getValueFromPath( + properties, + List.of("kubernetes", "properties", "labels", "properties", "pod-template-generation", "type") + ), + is("keyword") + ); + assertThat(getValueFromPath(properties, List.of("log", "properties", "file", "properties", "path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("log", "properties", "file", "properties", "path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("host", "properties", "os", "properties", "name", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("host", "properties", "os", "properties", "name", "fields", "text", "type")), + is("match_only_text") + ); + } + } + + @SuppressWarnings("unchecked") + public void testGeneralMockupEcsMappings() throws Exception { + { + indexDoc(client, DATA_STREAM_NAME, """ + { + "start_timestamp": "not a date", + "start-timestamp": "not a date", + "timestamp.us": 1688550340718000, + "test": "mockup-ecs-log", + "registry": { + "data": { + "strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"] + } + }, + "process": { + "title": "ssh", + "executable": "/usr/bin/ssh", + "name": "ssh", + "command_line": "/usr/bin/ssh -l user 10.0.0.16", + "working_directory": "/home/ekoren", + "io": { + "text": "test" + } + }, + "url": { + "path": "/page", + "full": "https://mydomain.com/app/page", + "original": "https://mydomain.com/app/original" + }, + "email": { + "message_id": "81ce15$8r2j59@mail01.example.com" + }, + "parent": { + "url": { + "path": "/page", + "full": "https://mydomain.com/app/page", + "original": "https://mydomain.com/app/original" + }, + "body": { + "content": "Some content" + }, + "file": { + "path": "/path/to/my/file", + "target_path": "/path/to/my/file" + }, + "code_signature.timestamp": "2023-07-05", + "registry.data.strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"] + }, + "error": { + "stack_trace": "co.elastic.test.TestClass error:\\n at co.elastic.test.BaseTestClass", + "message": "Error occurred" + }, + "file": { + "path": "/path/to/my/file", + "target_path": "/path/to/my/file" + }, + "os": { + "full": "Mac OS Mojave" + }, + "user_agent": { + "original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15" + }, + "user": { + "full_name": "John Doe" + }, + "vulnerability": { + "score": { + "base": 5.5, + "temporal": 5.5, + "version": "2.0" + }, + "textual_score": "bad" + }, + "host": { + "cpu": { + "usage": 0.68 + } + }, + "geo": { + "location": { + "lon": -73.614830, + "lat": 45.505918 + } + }, + "data_stream": { + "dataset": "nginx.access", + "namespace": "production", + "custom": "whatever" + }, + "structured_data": { + "key1": "value1", + "key2": ["value2", "value3"] + }, + "exports": { + "key": "value" + }, + "top_level_imports": { + "key": "value" + }, + "nested": { + "imports": { + "key": "value" + } + }, + "numeric_as_string": "42", + "socket": { + "ip": "127.0.0.1", + "remote_ip": "187.8.8.8" + } + } + """); + List results = searchDocs(client, DATA_STREAM_NAME, """ + { + "query": { + "term": { + "test": { + "value": "mockup-ecs-log" + } + } + }, + "fields": ["start-timestamp", "start_timestamp"], + "script_fields": { + "data_stream_type": { + "script": { + "source": "doc['data_stream.type'].value" + } + } + } + } + """); + assertThat(results.size(), is(1)); + Map fields = ((Map>) results.get(0)).get("fields"); + List ignored = ((Map>) results.get(0)).get("_ignored"); + Map ignoredFieldValues = ((Map>) results.get(0)).get("ignored_field_values"); + + // the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type + assertThat(ignored.size(), is(2)); + assertThat(ignored.get(0), is("start_timestamp")); + List startTimestampValues = (List) ignoredFieldValues.get("start_timestamp"); + assertThat(startTimestampValues.size(), is(1)); + assertThat(startTimestampValues.get(0), is("not a date")); + // "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" + assertThat(fields.get("start-timestamp"), is(List.of("not a date"))); + // verify that data_stream.type has the correct constant_keyword value + assertThat(fields.get("data_stream_type"), is(List.of("logs"))); + assertThat(ignored.get(1), is("vulnerability.textual_score")); + + Map properties = getMappingProperties(client, backingIndex); + assertThat(getValueFromPath(properties, List.of("error", "properties", "message", "type")), is("match_only_text")); + assertThat( + getValueFromPath(properties, List.of("registry", "properties", "data", "properties", "strings", "type")), + is("wildcard") + ); + assertThat( + getValueFromPath( + properties, + List.of("parent", "properties", "registry", "properties", "data", "properties", "strings", "type") + ), + is("wildcard") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "io", "properties", "text", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("email", "properties", "message_id", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url", "properties", "path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url", "properties", "full", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url", "properties", "full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "full", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "full", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("url", "properties", "original", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("url", "properties", "original", "fields", "text", "type")), + is("match_only_text") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "original", "type")), + is("wildcard") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "original", "fields", "text", "type")), + is("match_only_text") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "body", "properties", "content", "type")), + is("wildcard") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "body", "properties", "content", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "command_line", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "command_line", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("error", "properties", "stack_trace", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("error", "properties", "stack_trace", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("file", "properties", "path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("file", "properties", "path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("parent", "properties", "file", "properties", "path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "file", "properties", "path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("file", "properties", "target_path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("file", "properties", "target_path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "file", "properties", "target_path", "type")), + is("keyword") + ); + assertThat( + getValueFromPath( + properties, + List.of("parent", "properties", "file", "properties", "target_path", "fields", "text", "type") + ), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("os", "properties", "full", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("os", "properties", "full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("user_agent", "properties", "original", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("user_agent", "properties", "original", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "title", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "title", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "executable", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "executable", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "name", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "name", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "working_directory", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "working_directory", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("user", "properties", "full_name", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("user", "properties", "full_name", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("start_timestamp", "type")), is("date")); + // testing the default mapping of string input fields to keyword if not matching any pattern + assertThat(getValueFromPath(properties, List.of("start-timestamp", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("timestamp", "properties", "us", "type")), is("long")); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "code_signature", "properties", "timestamp", "type")), + is("date") + ); + assertThat( + getValueFromPath(properties, List.of("vulnerability", "properties", "score", "properties", "base", "type")), + is("float") + ); + assertThat( + getValueFromPath(properties, List.of("vulnerability", "properties", "score", "properties", "temporal", "type")), + is("float") + ); + assertThat( + getValueFromPath(properties, List.of("vulnerability", "properties", "score", "properties", "version", "type")), + is("keyword") + ); + assertThat(getValueFromPath(properties, List.of("vulnerability", "properties", "textual_score", "type")), is("float")); + assertThat( + getValueFromPath(properties, List.of("host", "properties", "cpu", "properties", "usage", "type")), + is("scaled_float") + ); + assertThat( + getValueFromPath(properties, List.of("host", "properties", "cpu", "properties", "usage", "scaling_factor")), + is(1000.0) + ); + assertThat(getValueFromPath(properties, List.of("geo", "properties", "location", "type")), is("geo_point")); + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "dataset", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "namespace", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "type", "type")), is("constant_keyword")); + // not one of the three data_stream fields that are explicitly mapped to constant_keyword + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "custom", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("structured_data", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("exports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("top_level_imports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("nested", "properties", "imports", "type")), is("flattened")); + // verifying the default mapping for strings into keyword, overriding the automatic numeric string detection + assertThat(getValueFromPath(properties, List.of("numeric_as_string", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("socket", "properties", "ip", "type")), is("ip")); + assertThat(getValueFromPath(properties, List.of("socket", "properties", "remote_ip", "type")), is("ip")); + } + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 5bb9c8b340ee9..cc8695b9e0e5b 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -21,6 +21,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesRegex; +import static org.hamcrest.Matchers.nullValue; public class LogsDataStreamIT extends DisabledSecurityDataStreamTestCase { @@ -45,8 +46,8 @@ public void testDefaultLogsSettingAndMapping() throws Exception { // Extend the mapping and verify putMapping(client, backingIndex); Map mappingProperties = getMappingProperties(client, backingIndex); - assertThat(((Map) mappingProperties.get("@timestamp")).get("ignore_malformed"), equalTo(false)); - assertThat(((Map) mappingProperties.get("numeric_field")).get("type"), equalTo("integer")); + assertThat(getValueFromPath(mappingProperties, List.of("@timestamp", "ignore_malformed")), equalTo(false)); + assertThat(getValueFromPath(mappingProperties, List.of("numeric_field", "type")), equalTo("integer")); // Insert valid doc and verify successful indexing { @@ -149,11 +150,8 @@ public void testCustomMapping() throws Exception { // Verify that the new field from the custom component template is applied putMapping(client, backingIndex); Map mappingProperties = getMappingProperties(client, backingIndex); - assertThat(((Map) mappingProperties.get("numeric_field")).get("type"), equalTo("integer")); - assertThat( - ((Map) mappingProperties.get("socket")).get("properties"), - equalTo(Map.of("ip", Map.of("type", "keyword"))) - ); + assertThat(getValueFromPath(mappingProperties, List.of("numeric_field", "type")), equalTo("integer")); + assertThat(getValueFromPath(mappingProperties, List.of("socket", "properties", "ip", "type")), is("keyword")); // Insert valid doc and verify successful indexing { @@ -227,7 +225,7 @@ public void testLogsDefaultPipeline() throws Exception { // Verify mapping from custom logs Map mappingProperties = getMappingProperties(client, backingIndex); - assertThat(((Map) mappingProperties.get("@timestamp")).get("type"), equalTo("date")); + assertThat(getValueFromPath(mappingProperties, List.of("@timestamp", "type")), equalTo("date")); // no timestamp - testing default pipeline's @timestamp set processor { @@ -284,7 +282,358 @@ public void testLogsDefaultPipeline() throws Exception { } } - private static void waitForLogs(RestClient client) throws Exception { + @SuppressWarnings("unchecked") + public void testLogsMessagePipeline() throws Exception { + RestClient client = client(); + waitForLogs(client); + + { + Request request = new Request("PUT", "/_ingest/pipeline/logs@custom"); + request.setJsonEntity(""" + { + "processors": [ + { + "pipeline" : { + "name": "logs@json-message", + "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" + } + } + ] + } + """); + assertOK(client.performRequest(request)); + } + + String dataStreamName = "logs-generic-default"; + createDataStream(client, dataStreamName); + + { + indexDoc(client, dataStreamName, """ + { + "@timestamp":"2023-05-09T16:48:34.135Z", + "message":"json", + "log.level": "INFO", + "ecs.version": "1.6.0", + "service.name":"my-app", + "event.dataset":"my-app.RollingFile", + "process.thread.name":"main", + "log.logger":"root.pkg.MyApp" + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "message": { + "value": "json" + } + } + }, + "fields": ["message"] + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + Map fields = ((Map>) results.get(0)).get("fields"); + + // root field parsed from JSON should win + assertThat(source.get("@timestamp"), is("2023-05-09T16:48:34.135Z")); + assertThat(source.get("message"), is("json")); + assertThat(((List) fields.get("message")).get(0), is("json")); + + // successful access to subfields verifies that dot expansion is part of the pipeline + assertThat(source.get("log.level"), is("INFO")); + assertThat(source.get("ecs.version"), is("1.6.0")); + assertThat(source.get("service.name"), is("my-app")); + assertThat(source.get("event.dataset"), is("my-app.RollingFile")); + assertThat(source.get("process.thread.name"), is("main")); + assertThat(source.get("log.logger"), is("root.pkg.MyApp")); + // _tmp_json_message should be removed by the pipeline + assertThat(source.get("_tmp_json_message"), is(nullValue())); + } + + // test malformed-JSON parsing - parsing error should be ignored and the document should be indexed with original message + { + indexDoc(client, dataStreamName, """ + { + "@timestamp":"2023-05-10", + "test":"malformed_json", + "message": "{\\"@timestamp\\":\\"2023-05-09T16:48:34.135Z\\", \\"message\\":\\"malformed_json\\"}}" + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "test": { + "value": "malformed_json" + } + } + } + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + + // root field parsed from JSON should win + assertThat(source.get("@timestamp"), is("2023-05-10")); + assertThat(source.get("message"), is("{\"@timestamp\":\"2023-05-09T16:48:34.135Z\", \"message\":\"malformed_json\"}}")); + assertThat(source.get("_tmp_json_message"), is(nullValue())); + } + + // test non-string message field + { + indexDoc(client, dataStreamName, """ + { + "message": 42, + "test": "numeric_message" + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "test": { + "value": "numeric_message" + } + } + }, + "fields": ["message"] + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + Map fields = ((Map>) results.get(0)).get("fields"); + + assertThat(source.get("message"), is(42)); + assertThat(((List) fields.get("message")).get(0), is("42")); + } + } + + @SuppressWarnings("unchecked") + public void testNoSubobjects() throws Exception { + RestClient client = client(); + waitForLogs(client); + { + Request request = new Request("POST", "/_component_template/logs-test-subobjects-mappings"); + request.setJsonEntity(""" + { + "template": { + "settings": { + "mapping": { + "ignore_malformed": true + } + }, + "mappings": { + "subobjects": false, + "date_detection": false, + "properties": { + "data_stream.type": { + "type": "constant_keyword", + "value": "logs" + }, + "data_stream.dataset": { + "type": "constant_keyword" + }, + "data_stream.namespace": { + "type": "constant_keyword" + } + } + } + } + } + """); + assertOK(client.performRequest(request)); + } + { + Request request = new Request("POST", "/_index_template/logs-ecs-test-template"); + request.setJsonEntity(""" + { + "priority": 200, + "data_stream": {}, + "index_patterns": ["logs-*-*"], + "composed_of": ["logs-test-subobjects-mappings", "ecs@dynamic_templates"] + } + """); + assertOK(client.performRequest(request)); + } + String dataStream = "logs-ecs-test-subobjects"; + createDataStream(client, dataStream); + String backingIndexName = getWriteBackingIndex(client, dataStream); + + indexDoc(client, dataStream, """ + { + "@timestamp": "2023-06-12", + "start_timestamp": "2023-06-08", + "location" : "POINT (-71.34 41.12)", + "test": "flattened", + "test.start_timestamp": "not a date", + "test.start-timestamp": "not a date", + "registry.data.strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"], + "process.title": "ssh", + "process.executable": "/usr/bin/ssh", + "process.name": "ssh", + "process.command_line": "/usr/bin/ssh -l user 10.0.0.16", + "process.working_directory": "/home/ekoren", + "process.io.text": "test", + "url.path": "/page", + "url.full": "https://mydomain.com/app/page", + "url.original": "https://mydomain.com/app/original", + "email.message_id": "81ce15$8r2j59@mail01.example.com", + "parent.url.path": "/page", + "parent.url.full": "https://mydomain.com/app/page", + "parent.url.original": "https://mydomain.com/app/original", + "parent.body.content": "Some content", + "parent.file.path": "/path/to/my/file", + "parent.file.target_path": "/path/to/my/file", + "parent.registry.data.strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"], + "error.stack_trace": "co.elastic.test.TestClass error:\\n at co.elastic.test.BaseTestClass", + "error.message": "Error occurred", + "file.path": "/path/to/my/file", + "file.target_path": "/path/to/my/file", + "os.full": "Mac OS Mojave", + "user_agent.original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15", + "user.full_name": "John Doe", + "vulnerability.score.base": 5.5, + "vulnerability.score.temporal": 5.5, + "vulnerability.score.version": "2.0", + "vulnerability.textual_score": "bad", + "host.cpu.usage": 0.68, + "geo.location": [-73.614830, 45.505918], + "data_stream.dataset": "nginx.access", + "data_stream.namespace": "production", + "data_stream.custom": "whatever", + "structured_data": {"key1": "value1", "key2": ["value2", "value3"]}, + "exports": {"key": "value"}, + "top_level_imports": {"key": "value"}, + "nested.imports": {"key": "value"}, + "numeric_as_string": "42", + "socket.ip": "127.0.0.1", + "socket.remote_ip": "187.8.8.8" + } + """); + List hits = searchDocs(client, dataStream, """ + { + "query": { + "term": { + "test": { + "value": "flattened" + } + } + }, + "fields": [ + "data_stream.type", + "location", + "geo.location", + "test.start-timestamp", + "test.start_timestamp", + "vulnerability.textual_score" + ] + } + """); + assertThat(hits.size(), is(1)); + Map fields = ((Map>) hits.get(0)).get("fields"); + List ignored = ((Map>) hits.get(0)).get("_ignored"); + Map> ignoredFieldValues = ((Map>>) hits.get(0)).get("ignored_field_values"); + + // verify that data_stream.type has the correct constant_keyword value + assertThat(fields.get("data_stream.type"), is(List.of("logs"))); + // verify geo_point subfields evaluation + assertThat(((List>) fields.get("location")).get(0).get("type"), is("Point")); + List coordinates = ((List>>) fields.get("location")).get(0).get("coordinates"); + assertThat(coordinates.size(), is(2)); + assertThat(coordinates.get(0), equalTo(-71.34)); + assertThat(coordinates.get(1), equalTo(41.12)); + List geoLocation = (List) fields.get("geo.location"); + assertThat(((Map) geoLocation.get(0)).get("type"), is("Point")); + coordinates = ((Map>) geoLocation.get(0)).get("coordinates"); + assertThat(coordinates.size(), is(2)); + assertThat(coordinates.get(0), equalTo(-73.614830)); + assertThat(coordinates.get(1), equalTo(45.505918)); + // "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" + assertThat(fields.get("test.start-timestamp"), is(List.of("not a date"))); + assertThat(ignored.size(), is(2)); + assertThat(ignored.get(0), is("vulnerability.textual_score")); + // the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type + assertThat(ignored.get(1), is("test.start_timestamp")); + assertThat(ignoredFieldValues.get("test.start_timestamp").size(), is(1)); + assertThat(ignoredFieldValues.get("test.start_timestamp"), is(List.of("not a date"))); + assertThat(ignoredFieldValues.get("vulnerability.textual_score").size(), is(1)); + assertThat(ignoredFieldValues.get("vulnerability.textual_score").get(0), is("bad")); + + Map properties = getMappingProperties(client, backingIndexName); + assertThat(getValueFromPath(properties, List.of("error.message", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("registry.data.strings", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.registry.data.strings", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("process.io.text", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("email.message_id", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.url.path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.full", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.url.full", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.url.full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("url.original", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.original", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.url.original", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.url.original", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.body.content", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.body.content", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.command_line", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("process.command_line", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("error.stack_trace", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("error.stack_trace", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("file.path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("file.path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.file.path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("parent.file.path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("file.target_path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("file.target_path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.file.target_path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("parent.file.target_path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("os.full", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("os.full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("user_agent.original", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("user_agent.original", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.title", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.title", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.executable", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.executable", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.name", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.name", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.working_directory", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.working_directory", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("user.full_name", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("user.full_name", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("start_timestamp", "type")), is("date")); + assertThat(getValueFromPath(properties, List.of("test.start_timestamp", "type")), is("date")); + // testing the default mapping of string input fields to keyword if not matching any pattern + assertThat(getValueFromPath(properties, List.of("test.start-timestamp", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("vulnerability.score.base", "type")), is("float")); + assertThat(getValueFromPath(properties, List.of("vulnerability.score.temporal", "type")), is("float")); + assertThat(getValueFromPath(properties, List.of("vulnerability.score.version", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("vulnerability.textual_score", "type")), is("float")); + assertThat(getValueFromPath(properties, List.of("host.cpu.usage", "type")), is("scaled_float")); + assertThat(getValueFromPath(properties, List.of("host.cpu.usage", "scaling_factor")), is(1000.0)); + assertThat(getValueFromPath(properties, List.of("location", "type")), is("geo_point")); + assertThat(getValueFromPath(properties, List.of("geo.location", "type")), is("geo_point")); + assertThat(getValueFromPath(properties, List.of("data_stream.dataset", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream.namespace", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream.type", "type")), is("constant_keyword")); + // not one of the three data_stream fields that are explicitly mapped to constant_keyword + assertThat(getValueFromPath(properties, List.of("data_stream.custom", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("structured_data", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("exports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("top_level_imports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("nested.imports", "type")), is("flattened")); + // verifying the default mapping for strings into keyword, overriding the automatic numeric string detection + assertThat(getValueFromPath(properties, List.of("numeric_as_string", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("socket.ip", "type")), is("ip")); + assertThat(getValueFromPath(properties, List.of("socket.remote_ip", "type")), is("ip")); + + } + + static void waitForLogs(RestClient client) throws Exception { assertBusy(() -> { try { Request request = new Request("GET", "_index_template/logs"); @@ -295,13 +644,13 @@ private static void waitForLogs(RestClient client) throws Exception { }); } - private static void createDataStream(RestClient client, String name) throws IOException { + static void createDataStream(RestClient client, String name) throws IOException { Request request = new Request("PUT", "_data_stream/" + name); assertOK(client.performRequest(request)); } @SuppressWarnings("unchecked") - private static String getWriteBackingIndex(RestClient client, String name) throws IOException { + static String getWriteBackingIndex(RestClient client, String name) throws IOException { Request request = new Request("GET", "_data_stream/" + name); List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); Map dataStream = (Map) dataStreams.get(0); @@ -310,12 +659,12 @@ private static String getWriteBackingIndex(RestClient client, String name) throw } @SuppressWarnings("unchecked") - private static Map getSettings(RestClient client, String indexName) throws IOException { + static Map getSettings(RestClient client, String indexName) throws IOException { Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings"); return ((Map>) entityAsMap(client.performRequest(request)).get(indexName)).get("settings"); } - private static void putMapping(RestClient client, String indexName) throws IOException { + static void putMapping(RestClient client, String indexName) throws IOException { Request request = new Request("PUT", "/" + indexName + "/_mapping"); request.setJsonEntity(""" { @@ -330,24 +679,51 @@ private static void putMapping(RestClient client, String indexName) throws IOExc } @SuppressWarnings("unchecked") - private static Map getMappingProperties(RestClient client, String indexName) throws IOException { + static Map getMappingProperties(RestClient client, String indexName) throws IOException { Request request = new Request("GET", "/" + indexName + "/_mapping"); Map map = (Map) entityAsMap(client.performRequest(request)).get(indexName); Map mappings = (Map) map.get("mappings"); return (Map) mappings.get("properties"); } - private static void indexDoc(RestClient client, String dataStreamName, String doc) throws IOException { + static void indexDoc(RestClient client, String dataStreamName, String doc) throws IOException { Request request = new Request("POST", "/" + dataStreamName + "/_doc?refresh=true"); request.setJsonEntity(doc); assertOK(client.performRequest(request)); } @SuppressWarnings("unchecked") - private static List searchDocs(RestClient client, String dataStreamName, String query) throws IOException { + static List searchDocs(RestClient client, String dataStreamName, String query) throws IOException { Request request = new Request("GET", "/" + dataStreamName + "/_search"); request.setJsonEntity(query); Map hits = (Map) entityAsMap(client.performRequest(request)).get("hits"); return (List) hits.get("hits"); } + + @SuppressWarnings("unchecked") + static Object getValueFromPath(Map map, List path) { + Map current = map; + for (int i = 0; i < path.size(); i++) { + Object value = current.get(path.get(i)); + if (i == path.size() - 1) { + return value; + } + if (value == null) { + throw new IllegalStateException("Path " + String.join(".", path) + " was not found in " + map); + } + if (value instanceof Map next) { + current = (Map) next; + } else { + throw new IllegalStateException( + "Failed to reach the end of the path " + + String.join(".", path) + + " last reachable field was " + + path.get(i) + + " in " + + map + ); + } + } + return current; + } } diff --git a/modules/data-streams/src/javaRestTest/resources/ecs-logs/es-agent-ecs-log.json b/modules/data-streams/src/javaRestTest/resources/ecs-logs/es-agent-ecs-log.json new file mode 100644 index 0000000000000..29ae669e1290d --- /dev/null +++ b/modules/data-streams/src/javaRestTest/resources/ecs-logs/es-agent-ecs-log.json @@ -0,0 +1,118 @@ +{ + "@timestamp": "2023-05-16T13:49:40.377Z", + "test": "elastic-agent-log", + "container": { + "image": { + "name": "docker.elastic.co/beats/elastic-agent:8.9.0-SNAPSHOT" + }, + "runtime": "containerd", + "id": "bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d" + }, + "kubernetes": { + "container": { + "name": "elastic-agent" + }, + "node": { + "uid": "0f4dd3b8-0b29-418e-ad7a-ebc55bc279ff", + "hostname": "multi-v1.27.1-worker", + "name": "multi-v1.27.1-worker", + "labels": { + "kubernetes_io/hostname": "multi-v1.27.1-worker", + "beta_kubernetes_io/os": "linux", + "kubernetes_io/arch": "arm64", + "kubernetes_io/os": "linux", + "beta_kubernetes_io/arch": "arm64" + } + }, + "pod": { + "uid": "c91d1354-27cf-40f3-a2d6-e2b75aa96bf2", + "ip": "172.18.0.4", + "test_ip": "172.18.0.5", + "name": "elastic-agent-managed-daemonset-jwktj" + }, + "namespace": "kube-system", + "namespace_uid": "63294aeb-b23f-429d-827c-e793ccf91024", + "daemonset": { + "name": "elastic-agent-managed-daemonset" + }, + "namespace_labels": { + "kubernetes_io/metadata_name": "kube-system" + }, + "labels": { + "controller-revision-hash": "7ff74fcd4b", + "pod-template-generation": "1", + "k8s-app": "elastic-agent" + } + }, + "agent": { + "name": "multi-v1.27.1-worker", + "id": "230358e2-6c5d-4675-9069-04feaddad64b", + "ephemeral_id": "e0934bfb-7e35-4bcc-a935-803643841213", + "type": "filebeat", + "version": "8.9.0" + }, + "log": { + "file": { + "path": "/var/log/containers/elastic-agent-managed-daemonset-jwktj_kube-system_elastic-agent-bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d.log" + }, + "offset": 635247 + }, + "elastic_agent": { + "id": "230358e2-6c5d-4675-9069-04feaddad64b", + "version": "8.9.0", + "snapshot": true + }, + "message": "{\"log.level\":\"info\",\"@timestamp\":\"2023-05-16T13:49:40.374Z\",\"message\":\"Non-zero metrics in the last 30s\",\"component\":{\"binary\":\"metricbeat\",\"dataset\":\"elastic_agent.metricbeat\",\"id\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\",\"type\":\"kubernetes/metrics\"},\"log\":{\"source\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\"},\"log.logger\":\"monitoring\",\"log.origin\":{\"file.line\":187,\"file.name\":\"log/log.go\"},\"service.name\":\"metricbeat\",\"ecs.version\":\"1.6.0\"}", + "orchestrator": { + "cluster": { + "name": "multi-v1.27.1", + "url": "multi-v1.27.1-control-plane:6443" + } + }, + "input": { + "type": "filestream" + }, + "ecs": { + "version": "8.0.0" + }, + "stream": "stderr", + "data_stream": { + "namespace": "default", + "dataset": "kubernetes.container_logs" + }, + "host": { + "hostname": "multi-v1.27.1-worker", + "os": { + "kernel": "5.15.49-linuxkit", + "codename": "focal", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "20.04.6 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "ip": [ + "10.244.2.1", + "10.244.2.1", + "172.18.0.4", + "fc00:f853:ccd:e793::4", + "fe80::42:acff:fe12:4", + "172.21.0.9" + ], + "containerized": false, + "name": "multi-v1.27.1-worker", + "id": "b2c527655d7746328f0686e25d3c413a", + "mac": [ + "02-42-AC-12-00-04", + "02-42-AC-15-00-09", + "32-7E-AA-73-39-04", + "EA-F3-80-1D-88-E3" + ], + "architecture": "aarch64" + }, + "event": { + "agent_id_status": "verified", + "ingested": "2023-05-16T13:49:47Z", + "dataset": "kubernetes.container_logs" + } +} \ No newline at end of file diff --git a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java index 43438bfe9e5fb..fa7b4ca1a80c0 100644 --- a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java +++ b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java @@ -9,7 +9,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -20,7 +19,6 @@ import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.junit.ClassRule; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99764") public class DataStreamsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { public DataStreamsClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/230_logs_message_pipeline.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/230_logs_message_pipeline.yml deleted file mode 100644 index 6fd6f24a4ea14..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/230_logs_message_pipeline.yml +++ /dev/null @@ -1,114 +0,0 @@ ---- -Test log message JSON-parsing pipeline: - - do: - ingest.put_pipeline: - # opting in to use the JSON parsing pipeline for message field - id: "logs@custom" - body: > - { - "processors": [ - { - "pipeline" : { - "name": "logs@json-message", - "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" - } - } - ] - } - - - do: - indices.create_data_stream: - name: logs-generic-default - - is_true: acknowledged - - - do: - index: - index: logs-generic-default - refresh: true - body: - '@timestamp': '2023-05-10' - message: |- - { - "@timestamp":"2023-05-09T16:48:34.135Z", - "message":"json", - "log.level": "INFO", - "ecs.version": "1.6.0", - "service.name":"my-app", - "event.dataset":"my-app.RollingFile", - "process.thread.name":"main", - "log.logger":"root.pkg.MyApp" - } - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - message: - value: 'json' - fields: - - field: 'message' - - length: { hits.hits: 1 } - # root field parsed from JSON should win - - match: { hits.hits.0._source.@timestamp: '2023-05-09T16:48:34.135Z' } - - match: { hits.hits.0._source.message: 'json' } - - match: { hits.hits.0.fields.message.0: 'json' } - # successful access to subfields verifies that dot expansion is part of the pipeline - - match: { hits.hits.0._source.log.level: 'INFO' } - - match: { hits.hits.0._source.ecs.version: '1.6.0' } - - match: { hits.hits.0._source.service.name: 'my-app' } - - match: { hits.hits.0._source.event.dataset: 'my-app.RollingFile' } - - match: { hits.hits.0._source.process.thread.name: 'main' } - - match: { hits.hits.0._source.log.logger: 'root.pkg.MyApp' } - # _tmp_json_message should be removed by the pipeline - - match: { hits.hits.0._source._tmp_json_message: null } - - # test malformed-JSON parsing - parsing error should be ignored and the document should be indexed with original message - - do: - index: - index: logs-generic-default - refresh: true - body: - '@timestamp': '2023-05-10' - test: 'malformed_json' - message: '{"@timestamp":"2023-05-09T16:48:34.135Z", "message":"malformed_json"}}' - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'malformed_json' - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.@timestamp: '2023-05-10' } - - match: { hits.hits.0._source.message: '{"@timestamp":"2023-05-09T16:48:34.135Z", "message":"malformed_json"}}' } - - match: { hits.hits.0._source._tmp_json_message: null } - - # test non-string message field - - do: - index: - index: logs-generic-default - refresh: true - body: - test: 'numeric_message' - message: 42 - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'numeric_message' - fields: - - field: 'message' - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.message: 42 } - - match: { hits.hits.0.fields.message.0: '42' } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml deleted file mode 100644 index 538e362ed9ec0..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml +++ /dev/null @@ -1,406 +0,0 @@ -setup: - - do: - ingest.put_pipeline: - # opting in to use the JSON parsing pipeline for message field - id: "logs@custom" - body: > - { - "processors": [ - { - "pipeline" : { - "name": "logs@json-message", - "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" - } - } - ] - } - - - do: - indices.create_data_stream: - name: logs-generic-default - ---- -Test Elastic Agent log ECS mappings: - - skip: - version: all - reason: https://github.com/elastic/elasticsearch/issues/97795 - - do: - indices.get_data_stream: - name: logs-generic-default - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - index: - index: logs-generic-default - refresh: true - body: > - { - "@timestamp": "2023-05-16T13:49:40.377Z", - "test": "elastic-agent-log", - "container": { - "image": { - "name": "docker.elastic.co/beats/elastic-agent:8.9.0-SNAPSHOT" - }, - "runtime": "containerd", - "id": "bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d" - }, - "kubernetes": { - "container": { - "name": "elastic-agent" - }, - "node": { - "uid": "0f4dd3b8-0b29-418e-ad7a-ebc55bc279ff", - "hostname": "multi-v1.27.1-worker", - "name": "multi-v1.27.1-worker", - "labels": { - "kubernetes_io/hostname": "multi-v1.27.1-worker", - "beta_kubernetes_io/os": "linux", - "kubernetes_io/arch": "arm64", - "kubernetes_io/os": "linux", - "beta_kubernetes_io/arch": "arm64" - } - }, - "pod": { - "uid": "c91d1354-27cf-40f3-a2d6-e2b75aa96bf2", - "ip": "172.18.0.4", - "test_ip": "172.18.0.5", - "name": "elastic-agent-managed-daemonset-jwktj" - }, - "namespace": "kube-system", - "namespace_uid": "63294aeb-b23f-429d-827c-e793ccf91024", - "daemonset": { - "name": "elastic-agent-managed-daemonset" - }, - "namespace_labels": { - "kubernetes_io/metadata_name": "kube-system" - }, - "labels": { - "controller-revision-hash": "7ff74fcd4b", - "pod-template-generation": "1", - "k8s-app": "elastic-agent" - } - }, - "agent": { - "name": "multi-v1.27.1-worker", - "id": "230358e2-6c5d-4675-9069-04feaddad64b", - "ephemeral_id": "e0934bfb-7e35-4bcc-a935-803643841213", - "type": "filebeat", - "version": "8.9.0" - }, - "log": { - "file": { - "path": "/var/log/containers/elastic-agent-managed-daemonset-jwktj_kube-system_elastic-agent-bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d.log" - }, - "offset": 635247 - }, - "elastic_agent": { - "id": "230358e2-6c5d-4675-9069-04feaddad64b", - "version": "8.9.0", - "snapshot": true - }, - "message": "{\"log.level\":\"info\",\"@timestamp\":\"2023-05-16T13:49:40.374Z\",\"message\":\"Non-zero metrics in the last 30s\",\"component\":{\"binary\":\"metricbeat\",\"dataset\":\"elastic_agent.metricbeat\",\"id\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\",\"type\":\"kubernetes/metrics\"},\"log\":{\"source\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\"},\"log.logger\":\"monitoring\",\"log.origin\":{\"file.line\":187,\"file.name\":\"log/log.go\"},\"service.name\":\"metricbeat\",\"ecs.version\":\"1.6.0\"}", - "orchestrator": { - "cluster": { - "name": "multi-v1.27.1", - "url": "multi-v1.27.1-control-plane:6443" - } - }, - "input": { - "type": "filestream" - }, - "ecs": { - "version": "8.0.0" - }, - "stream": "stderr", - "data_stream": { - "namespace": "default", - "dataset": "kubernetes.container_logs" - }, - "host": { - "hostname": "multi-v1.27.1-worker", - "os": { - "kernel": "5.15.49-linuxkit", - "codename": "focal", - "name": "Ubuntu", - "type": "linux", - "family": "debian", - "version": "20.04.6 LTS (Focal Fossa)", - "platform": "ubuntu" - }, - "ip": [ - "10.244.2.1", - "10.244.2.1", - "172.18.0.4", - "fc00:f853:ccd:e793::4", - "fe80::42:acff:fe12:4", - "172.21.0.9" - ], - "containerized": false, - "name": "multi-v1.27.1-worker", - "id": "b2c527655d7746328f0686e25d3c413a", - "mac": [ - "02-42-AC-12-00-04", - "02-42-AC-15-00-09", - "32-7E-AA-73-39-04", - "EA-F3-80-1D-88-E3" - ], - "architecture": "aarch64" - }, - "event": { - "agent_id_status": "verified", - "ingested": "2023-05-16T13:49:47Z", - "dataset": "kubernetes.container_logs" - } - } - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'elastic-agent-log' - fields: - - field: 'message' - - length: { hits.hits: 1 } - # timestamp from deserialized JSON message field should win - - match: { hits.hits.0._source.@timestamp: '2023-05-16T13:49:40.374Z' } - - match: { hits.hits.0._source.kubernetes.pod.name: 'elastic-agent-managed-daemonset-jwktj' } - # expecting the extracted message from within the original JSON-formatted message - - match: { hits.hits.0.fields.message.0: 'Non-zero metrics in the last 30s' } - - - do: - indices.get_mapping: - index: logs-generic-default - - match: { .$idx0name.mappings.properties.@timestamp.type: "date" } - - match: { .$idx0name.mappings.properties.message.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.pod.properties.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.pod.properties.ip.type: "ip" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.pod.properties.test_ip.type: "ip" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.labels.properties.pod-template-generation.type: "keyword" } - - match: { .$idx0name.mappings.properties.log.properties.file.properties.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.log.properties.file.properties.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.host.properties.os.properties.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.host.properties.os.properties.name.fields.text.type: "match_only_text" } - ---- -Test general mockup ECS mappings: - - do: - indices.get_data_stream: - name: logs-generic-default - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - index: - index: logs-generic-default - refresh: true - body: > - { - "start_timestamp": "not a date", - "start-timestamp": "not a date", - "timestamp.us": 1688550340718000, - "test": "mockup-ecs-log", - "registry": { - "data": { - "strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"] - } - }, - "process": { - "title": "ssh", - "executable": "/usr/bin/ssh", - "name": "ssh", - "command_line": "/usr/bin/ssh -l user 10.0.0.16", - "working_directory": "/home/ekoren", - "io": { - "text": "test" - } - }, - "url": { - "path": "/page", - "full": "https://mydomain.com/app/page", - "original": "https://mydomain.com/app/original" - }, - "email": { - "message_id": "81ce15$8r2j59@mail01.example.com" - }, - "parent": { - "url": { - "path": "/page", - "full": "https://mydomain.com/app/page", - "original": "https://mydomain.com/app/original" - }, - "body": { - "content": "Some content" - }, - "file": { - "path": "/path/to/my/file", - "target_path": "/path/to/my/file" - }, - "code_signature.timestamp": "2023-07-05", - "registry.data.strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"] - }, - "error": { - "stack_trace": "co.elastic.test.TestClass error:\n at co.elastic.test.BaseTestClass", - "message": "Error occurred" - }, - "file": { - "path": "/path/to/my/file", - "target_path": "/path/to/my/file" - }, - "os": { - "full": "Mac OS Mojave" - }, - "user_agent": { - "original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15" - }, - "user": { - "full_name": "John Doe" - }, - "vulnerability": { - "score": { - "base": 5.5, - "temporal": 5.5, - "version": "2.0" - }, - "textual_score": "bad" - }, - "host": { - "cpu": { - "usage": 0.68 - } - }, - "geo": { - "location": { - "lon": -73.614830, - "lat": 45.505918 - } - }, - "data_stream": { - "dataset": "nginx.access", - "namespace": "production", - "custom": "whatever" - }, - "structured_data": { - "key1": "value1", - "key2": ["value2", "value3"] - }, - "exports": { - "key": "value" - }, - "top_level_imports": { - "key": "value" - }, - "nested": { - "imports": { - "key": "value" - } - }, - "numeric_as_string": "42", - "socket": { - "ip": "127.0.0.1", - "remote_ip": "187.8.8.8" - } - } - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'mockup-ecs-log' - fields: - - field: 'start_timestamp' - - field: 'start-timestamp' - script_fields: - data_stream_type: - script: - source: "doc['data_stream.type'].value" - - length: { hits.hits: 1 } - # the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type - - length: { hits.hits.0._ignored: 2 } - - match: { hits.hits.0._ignored.0: 'start_timestamp' } - - length: { hits.hits.0.ignored_field_values.start_timestamp: 1 } - - match: { hits.hits.0.ignored_field_values.start_timestamp.0: 'not a date' } - # "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" - - match: { hits.hits.0.fields.start-timestamp.0: 'not a date' } - # verify that data_stream.type has the correct constant_keyword value - - match: { hits.hits.0.fields.data_stream_type.0: 'logs' } - - match: { hits.hits.0._ignored.1: 'vulnerability.textual_score' } - - - do: - indices.get_mapping: - index: logs-generic-default - - match: { .$idx0name.mappings.properties.error.properties.message.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.registry.properties.data.properties.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.registry.properties.data.properties.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process.properties.io.properties.text.type: "wildcard" } - - match: { .$idx0name.mappings.properties.email.properties.message_id.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.url.properties.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.body.properties.content.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.body.properties.content.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.command_line.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process.properties.command_line.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.error.properties.stack_trace.type: "wildcard" } - - match: { .$idx0name.mappings.properties.error.properties.stack_trace.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file.properties.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file.properties.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file.properties.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file.properties.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.os.properties.full.type: "keyword" } - - match: { .$idx0name.mappings.properties.os.properties.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user_agent.properties.original.type: "keyword" } - - match: { .$idx0name.mappings.properties.user_agent.properties.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.title.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.title.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.executable.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.executable.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.working_directory.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.working_directory.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user.properties.full_name.type: "keyword" } - - match: { .$idx0name.mappings.properties.user.properties.full_name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.start_timestamp.type: "date" } - # testing the default mapping of string input fields to keyword if not matching any pattern - - match: { .$idx0name.mappings.properties.start-timestamp.type: "keyword" } - - match: { .$idx0name.mappings.properties.timestamp.properties.us.type: "long" } - - match: { .$idx0name.mappings.properties.parent.properties.code_signature.properties.timestamp.type: "date" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.score.properties.base.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.score.properties.temporal.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.score.properties.version.type: "keyword" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.textual_score.type: "float" } - - match: { .$idx0name.mappings.properties.host.properties.cpu.properties.usage.type: "scaled_float" } - - match: { .$idx0name.mappings.properties.host.properties.cpu.properties.usage.scaling_factor: 1000 } - - match: { .$idx0name.mappings.properties.geo.properties.location.type: "geo_point" } - - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" } - # not one of the three data_stream fields that are explicitly mapped to constant_keyword - - match: { .$idx0name.mappings.properties.data_stream.properties.custom.type: "keyword" } - - match: { .$idx0name.mappings.properties.structured_data.type: "flattened" } - - match: { .$idx0name.mappings.properties.exports.type: "flattened" } - - match: { .$idx0name.mappings.properties.top_level_imports.type: "flattened" } - - match: { .$idx0name.mappings.properties.nested.properties.imports.type: "flattened" } - # verifying the default mapping for strings into keyword, overriding the automatic numeric string detection - - match: { .$idx0name.mappings.properties.numeric_as_string.type: "keyword" } - - match: { .$idx0name.mappings.properties.socket.properties.ip.type: "ip" } - - match: { .$idx0name.mappings.properties.socket.properties.remote_ip.type: "ip" } - diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml deleted file mode 100644 index 607693e9f9955..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml +++ /dev/null @@ -1,218 +0,0 @@ ---- -Test flattened document with subobjects-false: -# NOTE: this doesn't work. In order to run this test set "subobjects: false" through logs-mappings.json - - skip: - features: allowed_warnings - - - do: - cluster.put_component_template: - name: logs-test-subobjects-mappings - body: - template: - settings: - mapping: - ignore_malformed: true - mappings: - subobjects: false - date_detection: false - properties: - data_stream.type: - type: constant_keyword - value: logs - data_stream.dataset: - type: constant_keyword - data_stream.namespace: - type: constant_keyword - - - do: - allowed_warnings: - - "index template [logs-ecs-test-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-ecs-test-template] will take precedence during new index creation" - indices.put_index_template: - name: logs-ecs-test-template - body: - priority: 200 - data_stream: {} - index_patterns: - - logs-*-* - composed_of: - - logs-test-subobjects-mappings - - ecs@dynamic_templates - - - do: - indices.create_data_stream: - name: logs-ecs-test-subobjects - - is_true: acknowledged - - - do: - indices.get_data_stream: - name: logs-ecs-test-subobjects - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - index: - index: logs-ecs-test-subobjects - refresh: true - body: > - { - "@timestamp": "2023-06-12", - "start_timestamp": "2023-06-08", - "location" : "POINT (-71.34 41.12)", - "test": "flattened", - "test.start_timestamp": "not a date", - "test.start-timestamp": "not a date", - "registry.data.strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"], - "process.title": "ssh", - "process.executable": "/usr/bin/ssh", - "process.name": "ssh", - "process.command_line": "/usr/bin/ssh -l user 10.0.0.16", - "process.working_directory": "/home/ekoren", - "process.io.text": "test", - "url.path": "/page", - "url.full": "https://mydomain.com/app/page", - "url.original": "https://mydomain.com/app/original", - "email.message_id": "81ce15$8r2j59@mail01.example.com", - "parent.url.path": "/page", - "parent.url.full": "https://mydomain.com/app/page", - "parent.url.original": "https://mydomain.com/app/original", - "parent.body.content": "Some content", - "parent.file.path": "/path/to/my/file", - "parent.file.target_path": "/path/to/my/file", - "parent.registry.data.strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"], - "error.stack_trace": "co.elastic.test.TestClass error:\n at co.elastic.test.BaseTestClass", - "error.message": "Error occurred", - "file.path": "/path/to/my/file", - "file.target_path": "/path/to/my/file", - "os.full": "Mac OS Mojave", - "user_agent.original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15", - "user.full_name": "John Doe", - "vulnerability.score.base": 5.5, - "vulnerability.score.temporal": 5.5, - "vulnerability.score.version": "2.0", - "vulnerability.textual_score": "bad", - "host.cpu.usage": 0.68, - "geo.location": [-73.614830, 45.505918], - "data_stream.dataset": "nginx.access", - "data_stream.namespace": "production", - "data_stream.custom": "whatever", - "structured_data": {"key1": "value1", "key2": ["value2", "value3"]}, - "exports": {"key": "value"}, - "top_level_imports": {"key": "value"}, - "nested.imports": {"key": "value"}, - "numeric_as_string": "42", - "socket.ip": "127.0.0.1", - "socket.remote_ip": "187.8.8.8" - } - - match: {result: "created"} - - - do: - search: - index: logs-ecs-test-subobjects - body: - query: - term: - test: - value: 'flattened' - fields: - - field: 'data_stream.type' - - field: 'location' - - field: 'geo.location' - - field: 'test.start-timestamp' - - field: 'test.start_timestamp' - - field: 'vulnerability.textual_score' - - length: { hits.hits: 1 } - # verify that data_stream.type has the correct constant_keyword value - - match: { hits.hits.0.fields.data_stream\.type.0: 'logs' } - # verify geo_point subfields evaluation - - match: { hits.hits.0.fields.location.0.type: 'Point' } - - length: { hits.hits.0.fields.location.0.coordinates: 2 } - - match: { hits.hits.0.fields.location.0.coordinates.0: -71.34 } - - match: { hits.hits.0.fields.location.0.coordinates.1: 41.12 } - - match: { hits.hits.0.fields.geo\.location.0.type: 'Point' } - - length: { hits.hits.0.fields.geo\.location.0.coordinates: 2 } - - match: { hits.hits.0.fields.geo\.location.0.coordinates.0: -73.614830 } - - match: { hits.hits.0.fields.geo\.location.0.coordinates.1: 45.505918 } - # "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" - # TODO: uncomment once https://github.com/elastic/elasticsearch/issues/96700 gets resolved - # - match: { hits.hits.0.fields.test\.start-timestamp.0: 'not a date' } - - length: { hits.hits.0._ignored: 2 } - - match: { hits.hits.0._ignored.0: 'vulnerability.textual_score' } - # the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type - - match: { hits.hits.0._ignored.1: 'test.start_timestamp' } - - length: { hits.hits.0.ignored_field_values.test\.start_timestamp: 1 } - # TODO: uncomment once https://github.com/elastic/elasticsearch/issues/96700 gets resolved - # - match: { hits.hits.0.ignored_field_values.test\.start_timestamp.0: 'not a date' } - - length: { hits.hits.0.ignored_field_values.vulnerability\.textual_score: 1 } - - match: { hits.hits.0.ignored_field_values.vulnerability\.textual_score.0: 'bad' } - - - do: - indices.get_mapping: - index: logs-ecs-test-subobjects - - match: { .$idx0name.mappings.properties.error\.message.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.registry\.data\.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.registry\.data\.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process\.io\.text.type: "wildcard" } - - match: { .$idx0name.mappings.properties.email\.message_id.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.url\.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.url\.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.url\.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.url\.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.url\.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.url\.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.body\.content.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.body\.content.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.command_line.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process\.command_line.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.error\.stack_trace.type: "wildcard" } - - match: { .$idx0name.mappings.properties.error\.stack_trace.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file\.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file\.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.file\.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent\.file\.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file\.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file\.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.file\.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent\.file\.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.os\.full.type: "keyword" } - - match: { .$idx0name.mappings.properties.os\.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user_agent\.original.type: "keyword" } - - match: { .$idx0name.mappings.properties.user_agent\.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.title.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.title.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.executable.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.executable.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.working_directory.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.working_directory.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user\.full_name.type: "keyword" } - - match: { .$idx0name.mappings.properties.user\.full_name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.start_timestamp.type: "date" } - - match: { .$idx0name.mappings.properties.test\.start_timestamp.type: "date" } - # testing the default mapping of string input fields to keyword if not matching any pattern - - match: { .$idx0name.mappings.properties.test\.start-timestamp.type: "keyword" } - - match: { .$idx0name.mappings.properties.vulnerability\.score\.base.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability\.score\.temporal.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability\.score\.version.type: "keyword" } - - match: { .$idx0name.mappings.properties.vulnerability\.textual_score.type: "float" } - - match: { .$idx0name.mappings.properties.host\.cpu\.usage.type: "scaled_float" } - - match: { .$idx0name.mappings.properties.host\.cpu\.usage.scaling_factor: 1000 } - - match: { .$idx0name.mappings.properties.location.type: "geo_point" } - - match: { .$idx0name.mappings.properties.geo\.location.type: "geo_point" } - - match: { .$idx0name.mappings.properties.data_stream\.dataset.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream\.namespace.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream\.type.type: "constant_keyword" } - # not one of the three data_stream fields that are explicitly mapped to constant_keyword - - match: { .$idx0name.mappings.properties.data_stream\.custom.type: "keyword" } - - match: { .$idx0name.mappings.properties.structured_data.type: "flattened" } - - match: { .$idx0name.mappings.properties.exports.type: "flattened" } - - match: { .$idx0name.mappings.properties.top_level_imports.type: "flattened" } - - match: { .$idx0name.mappings.properties.nested\.imports.type: "flattened" } - # verifying the default mapping for strings into keyword, overriding the automatic numeric string detection - - match: { .$idx0name.mappings.properties.numeric_as_string.type: "keyword" } - - match: { .$idx0name.mappings.properties.socket\.ip.type: "ip" } - - match: { .$idx0name.mappings.properties.socket\.remote_ip.type: "ip" } - diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml index 296c692fa2d49..1ea39087211dd 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml @@ -51,6 +51,9 @@ setup: --- "Get data stream with default lifecycle": + - skip: + version: all + reason: https://github.com/elastic/elasticsearch/pull/100187 - do: indices.get_data_lifecycle: From 24037d6ed682f55f94cb1417f5d6fa8489cffa11 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 11 Oct 2023 14:33:01 +0300 Subject: [PATCH 36/48] Exclude synthetic source test for TSDB from mixedClusterTests (#100592) * Don't print synthetic source in mapping for bwc tests * Move comment. * Don't print synthetic source in mapping for bwc tests #2 * Don't print synthetic source in mapping for bwc tests #2 * Revert "Don't print synthetic source in mapping for bwc tests #2" This reverts commit 034262c5d22229aa6e8a0b7e754fd806a521cfc4. * Revert "Don't print synthetic source in mapping for bwc tests #2" This reverts commit 44e815635e2565c0b042cfe558a7451226c89488. * Revert "Don't print synthetic source in mapping for bwc tests (#100572)" This reverts commit 9322ab9b9163f70c9bf832f1b0a1985121393cfe. * Exclude synthetic source test from mixedClusterTests * Update comment. --- qa/mixed-cluster/build.gradle | 6 ++++ .../index/mapper/SourceFieldMapper.java | 34 +++++++++---------- .../index/mapper/SourceFieldMapperTests.java | 8 ----- .../query/SearchExecutionContextTests.java | 2 +- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 08d64e2b9353b..13256179b0a2b 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -41,6 +41,12 @@ excludeList.add('aggregations/filter/Standard queries get cached') excludeList.add('aggregations/filter/Terms lookup gets cached') excludeList.add('aggregations/filters_bucket/cache hits') +// The test checks that tsdb mappings report source as synthetic. +// It is supposed to be skipped (not needed) for versions before +// 8.10 but mixed cluster tests may not respect that - see the +// comment above. +excludeList.add('tsdb/20_mapping/Synthetic source') + BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> if (bwcVersion != VersionProperties.getElasticsearchVersion()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index aeab22a6f5f35..c5d5dbec1ef15 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -101,7 +101,20 @@ public static class Builder extends MetadataFieldMapper.Builder { (previous, current, conflicts) -> (previous.value() == current.value()) || (previous.value() && current.value() == false) ); - private final Parameter mode; + /* + * The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic + * source mode. + */ + private final Parameter mode = new Parameter<>( + "mode", + true, + () -> null, + (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), + m -> toType(m).enabled.explicit() ? null : toType(m).mode, + (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), + v -> v.toString().toLowerCase(Locale.ROOT) + ).setMergeValidator((previous, current, conflicts) -> (previous == current) || current != Mode.STORED) + .setSerializerCheck((includeDefaults, isConfigured, value) -> value != null); // don't emit if `enabled` is configured private final Parameter> includes = Parameter.stringArrayParam( "includes", false, @@ -115,22 +128,9 @@ public static class Builder extends MetadataFieldMapper.Builder { private final IndexMode indexMode; - public Builder(IndexMode indexMode, IndexVersion indexVersion) { + public Builder(IndexMode indexMode) { super(Defaults.NAME); this.indexMode = indexMode; - this.mode = new Parameter<>( - "mode", - true, - // The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic source mode. - () -> getIndexMode() == IndexMode.TIME_SERIES && indexVersion.between(IndexVersion.V_8_7_0, IndexVersion.V_8_10_0) - ? Mode.SYNTHETIC - : null, - (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), - m -> toType(m).enabled.explicit() ? null : toType(m).mode, - (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), - v -> v.toString().toLowerCase(Locale.ROOT) - ).setMergeValidator((previous, current, conflicts) -> (previous == current) || current != Mode.STORED) - .setSerializerCheck((includeDefaults, isConfigured, value) -> value != null); // don't emit if `enabled` is configured } public Builder setSynthetic() { @@ -188,7 +188,7 @@ private IndexMode getIndexMode() { c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersion.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT : DEFAULT, - c -> new Builder(c.getIndexSettings().getMode(), c.getIndexSettings().getIndexVersionCreated()) + c -> new Builder(c.getIndexSettings().getMode()) ); static final class SourceFieldType extends MappedFieldType { @@ -313,7 +313,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode, IndexVersion.current()).init(this); + return new Builder(indexMode).init(this); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 433ebc467483d..f683cb60c87c3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -12,8 +12,6 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; @@ -240,10 +238,4 @@ public void testSyntheticSourceInTimeSeries() throws IOException { assertTrue(mapper.sourceMapper().isSynthetic()); assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString()); } - - public void testSyntheticSourceInTimeSeriesBwc() throws IOException { - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(IndexMode.TIME_SERIES, IndexVersion.V_8_8_0).build(); - assertTrue(sourceMapper.isSynthetic()); - assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", sourceMapper.toString()); - } } diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 9df1dc24c2793..6d671a258c26a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -381,7 +381,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, IndexVersion.current()).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); From 4c200aede09983e7c6e3f59973042d5e24231386 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 11 Oct 2023 13:16:39 +0100 Subject: [PATCH 37/48] [ML] Adjust AutodetectMemoryLimitIT/testManyDistinctOverFields (#100667) Adjust the AutodetectMemoryLimitIT/testManyDistinctOverFields integration test to account for changed memory consumption due to upgrade of Boost --- .../xpack/ml/integration/AutodetectMemoryLimitIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java index 5405852173a62..4b0783dda84cc 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java @@ -206,7 +206,7 @@ public void testManyDistinctOverFields() throws Exception { int user = 0; while (timestamp < now) { List data = new ArrayList<>(); - for (int i = 0; i < 10000; i++) { + for (int i = 0; i < 20000; i++) { // It's important that the values used here are either always represented in less than 16 UTF-8 bytes or // always represented in more than 22 UTF-8 bytes. Otherwise platform differences in when the small string // optimisation is used will make the results of this test very different for the different platforms. From 9fb550be44067ac9b484cb73af5a04521b6d0122 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 11 Oct 2023 14:24:32 +0200 Subject: [PATCH 38/48] WellKnownBinary#toWKB should not throw an IOException (#100669) The only reason this method is throwing an exception is because the method ByteArrayOutputStream#close() is declaring it although it is a noop. Therefore it can be safely ignored. Thanks @romseygeek for bringing into attention. --- .../geometry/utils/WellKnownBinary.java | 6 +++- .../geometry/utils/WKBTests.java | 33 +++++++++---------- .../mapper/LegacyGeoShapeFieldMapper.java | 2 +- .../index/mapper/GeoShapeFieldMapper.java | 3 +- .../GeoShapeWithDocValuesFieldMapper.java | 2 +- .../index/mapper/PointFieldMapper.java | 2 +- .../index/mapper/ShapeFieldMapper.java | 2 +- 7 files changed, 26 insertions(+), 24 deletions(-) diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java index 9ded2106b2500..526a621674f6b 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java @@ -24,6 +24,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; @@ -40,10 +41,13 @@ private WellKnownBinary() {} /** * Converts the given {@link Geometry} to WKB with the provided {@link ByteOrder} */ - public static byte[] toWKB(Geometry geometry, ByteOrder byteOrder) throws IOException { + public static byte[] toWKB(Geometry geometry, ByteOrder byteOrder) { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { toWKB(geometry, outputStream, ByteBuffer.allocate(8).order(byteOrder)); return outputStream.toByteArray(); + } catch (IOException ioe) { + // Should never happen as the only method throwing IOException is ByteArrayOutputStream#close and it is a NOOP + throw new UncheckedIOException(ioe); } } diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/utils/WKBTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/WKBTests.java index 9ede3d9db8126..5369475e4ed4f 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/utils/WKBTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/WKBTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.test.ESTestCase; -import java.io.IOException; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; @@ -35,47 +34,47 @@ public void testEmptyPoint() { assertEquals("Empty POINT cannot be represented in WKB", ex.getMessage()); } - public void testPoint() throws IOException { + public void testPoint() { Point point = GeometryTestUtils.randomPoint(randomBoolean()); assertWKB(point); } - public void testEmptyMultiPoint() throws IOException { + public void testEmptyMultiPoint() { MultiPoint multiPoint = MultiPoint.EMPTY; assertWKB(multiPoint); } - public void testMultiPoint() throws IOException { + public void testMultiPoint() { MultiPoint multiPoint = GeometryTestUtils.randomMultiPoint(randomBoolean()); assertWKB(multiPoint); } - public void testEmptyLine() throws IOException { + public void testEmptyLine() { Line line = Line.EMPTY; assertWKB(line); } - public void testLine() throws IOException { + public void testLine() { Line line = GeometryTestUtils.randomLine(randomBoolean()); assertWKB(line); } - public void tesEmptyMultiLine() throws IOException { + public void tesEmptyMultiLine() { MultiLine multiLine = MultiLine.EMPTY; assertWKB(multiLine); } - public void testMultiLine() throws IOException { + public void testMultiLine() { MultiLine multiLine = GeometryTestUtils.randomMultiLine(randomBoolean()); assertWKB(multiLine); } - public void testEmptyPolygon() throws IOException { + public void testEmptyPolygon() { Polygon polygon = Polygon.EMPTY; assertWKB(polygon); } - public void testPolygon() throws IOException { + public void testPolygon() { final boolean hasZ = randomBoolean(); Polygon polygon = GeometryTestUtils.randomPolygon(hasZ); if (randomBoolean()) { @@ -89,22 +88,22 @@ public void testPolygon() throws IOException { assertWKB(polygon); } - public void testEmptyMultiPolygon() throws IOException { + public void testEmptyMultiPolygon() { MultiPolygon multiPolygon = MultiPolygon.EMPTY; assertWKB(multiPolygon); } - public void testMultiPolygon() throws IOException { + public void testMultiPolygon() { MultiPolygon multiPolygon = GeometryTestUtils.randomMultiPolygon(randomBoolean()); assertWKB(multiPolygon); } - public void testEmptyGeometryCollection() throws IOException { + public void testEmptyGeometryCollection() { GeometryCollection collection = GeometryCollection.EMPTY; assertWKB(collection); } - public void testGeometryCollection() throws IOException { + public void testGeometryCollection() { GeometryCollection collection = GeometryTestUtils.randomGeometryCollection(randomBoolean()); assertWKB(collection); } @@ -115,7 +114,7 @@ public void testEmptyCircle() { assertEquals("Empty CIRCLE cannot be represented in WKB", ex.getMessage()); } - public void testCircle() throws IOException { + public void testCircle() { Circle circle = GeometryTestUtils.randomCircle(randomBoolean()); assertWKB(circle); } @@ -129,7 +128,7 @@ public void testEmptyRectangle() { assertEquals("Empty ENVELOPE cannot be represented in WKB", ex.getMessage()); } - public void testRectangle() throws IOException { + public void testRectangle() { Rectangle rectangle = GeometryTestUtils.randomRectangle(); assertWKB(rectangle); } @@ -138,7 +137,7 @@ private ByteOrder randomByteOrder() { return randomBoolean() ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; } - private void assertWKB(Geometry geometry) throws IOException { + private void assertWKB(Geometry geometry) { final boolean hasZ = geometry.hasZ(); final ByteOrder byteOrder = randomByteOrder(); final byte[] b = WellKnownBinary.toWKB(geometry, byteOrder); diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java index 46860ff38b8ca..51cc7541a9a4d 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java @@ -583,7 +583,7 @@ public String strategy() { } @Override - protected void index(DocumentParserContext context, ShapeBuilder shapeBuilder) throws IOException { + protected void index(DocumentParserContext context, ShapeBuilder shapeBuilder) { if (shapeBuilder == null) { return; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 7ae410a1a9dcb..ad287e1c6b005 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -22,7 +22,6 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; -import java.io.IOException; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -191,7 +190,7 @@ public FieldMapper.Builder getMergeBuilder() { } @Override - protected void index(DocumentParserContext context, Geometry geometry) throws IOException { + protected void index(DocumentParserContext context, Geometry geometry) { if (geometry == null) { return; } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 55929a1c1b83e..13fb4246a5b3a 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -340,7 +340,7 @@ public GeoShapeWithDocValuesFieldMapper( } @Override - protected void index(DocumentParserContext context, Geometry geometry) throws IOException { + protected void index(DocumentParserContext context, Geometry geometry) { // TODO: Make common with the index method ShapeFieldMapper if (geometry == null) { return; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java index 378b78111ab19..f5cc7280aa8bb 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java @@ -150,7 +150,7 @@ public PointFieldMapper( } @Override - protected void index(DocumentParserContext context, CartesianPoint point) throws IOException { + protected void index(DocumentParserContext context, CartesianPoint point) { if (fieldType().isIndexed()) { context.doc().add(new XYPointField(fieldType().name(), (float) point.getX(), (float) point.getY())); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index 127a4fd1050cd..838fd56cfc11a 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -204,7 +204,7 @@ public ShapeFieldMapper( } @Override - protected void index(DocumentParserContext context, Geometry geometry) throws IOException { + protected void index(DocumentParserContext context, Geometry geometry) { // TODO: Make common with the index method GeoShapeWithDocValuesFieldMapper if (geometry == null) { return; From 14263a78e88c0197cbccf1c94aafda32898b7c30 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Wed, 11 Oct 2023 08:49:18 -0400 Subject: [PATCH 39/48] Remove uses of Version from Plugin CLI commands (#100298) The Plugin CLI can generally treat strings opaquely. We had some logic comparing earlier versions, but what we really care about with most of our plugins is whether or not they were built with the current version of Elasticsearch, not whether they were built before or after. (This question will be trickier with stable plugins, but none of that code is in the CLI.) The CLI classes can be cleaned up even more once Version is removed from PluginDescriptor. Some of the tests can't use opaque strings for versions until PluginDescriptor can handle them. * Remove Version from Install and List plugin actions * Remove Version from SyncPluginsAction --- .../plugins/cli/InstallPluginAction.java | 37 +++++++++++------- .../plugins/cli/ListPluginsCommand.java | 12 ++++-- .../plugins/cli/SyncPluginsAction.java | 12 +++--- .../plugins/cli/InstallPluginActionTests.java | 39 ++++++++++++------- .../plugins/cli/ListPluginsCommandTests.java | 2 +- .../plugins/cli/SyncPluginsActionTests.java | 15 ++++--- 6 files changed, 74 insertions(+), 43 deletions(-) diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java index d32cbd8dd1736..c7bee4a6c172d 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java @@ -23,7 +23,6 @@ import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator; import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentVerifierBuilderProvider; import org.elasticsearch.Build; -import org.elasticsearch.Version; import org.elasticsearch.bootstrap.PluginPolicyInfo; import org.elasticsearch.bootstrap.PolicyUtil; import org.elasticsearch.cli.ExitCodes; @@ -84,6 +83,8 @@ import java.util.Set; import java.util.Timer; import java.util.TimerTask; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.ZipEntry; @@ -303,7 +304,7 @@ private Path download(InstallablePlugin plugin, Path tmpDir) throws Exception { // else carry on to regular download } - final String url = getElasticUrl(getStagingHash(), Version.CURRENT, isSnapshot(), pluginId, Platforms.PLATFORM_NAME); + final String url = getElasticUrl(getStagingHash(), isSnapshot(), pluginId, Platforms.PLATFORM_NAME); terminal.println(logPrefix + "Downloading " + pluginId + " from elastic"); return downloadAndValidate(url, tmpDir, true); } @@ -341,7 +342,7 @@ private Path getPluginArchivePath(String pluginId, String pluginArchiveDir) thro if (Files.isDirectory(path) == false) { throw new UserException(ExitCodes.CONFIG, "Location in ES_PLUGIN_ARCHIVE_DIR is not a directory"); } - return PathUtils.get(pluginArchiveDir, pluginId + "-" + Version.CURRENT + (isSnapshot() ? "-SNAPSHOT" : "") + ".zip"); + return PathUtils.get(pluginArchiveDir, pluginId + "-" + Build.current().qualifiedVersion() + ".zip"); } // pkg private so tests can override @@ -356,13 +357,8 @@ boolean isSnapshot() { /** * Returns the url for an official elasticsearch plugin. */ - private String getElasticUrl( - final String stagingHash, - final Version version, - final boolean isSnapshot, - final String pluginId, - final String platform - ) throws IOException, UserException { + private String getElasticUrl(final String stagingHash, final boolean isSnapshot, final String pluginId, final String platform) + throws IOException, UserException { final String baseUrl; if (isSnapshot && stagingHash == null) { throw new UserException( @@ -370,11 +366,21 @@ private String getElasticUrl( "attempted to install release build of official plugin on snapshot build of Elasticsearch" ); } + // assumption: we will only be publishing plugins to snapshot or staging when they're versioned + String semanticVersion = getSemanticVersion(Build.current().version()); + if (semanticVersion == null) { + throw new UserException( + ExitCodes.CONFIG, + "attempted to download a plugin for a non-semantically-versioned build of Elasticsearch: [" + + Build.current().version() + + "]" + ); + } if (stagingHash != null) { if (isSnapshot) { - baseUrl = nonReleaseUrl("snapshots", version, stagingHash, pluginId); + baseUrl = nonReleaseUrl("snapshots", semanticVersion, stagingHash, pluginId); } else { - baseUrl = nonReleaseUrl("staging", version, stagingHash, pluginId); + baseUrl = nonReleaseUrl("staging", semanticVersion, stagingHash, pluginId); } } else { baseUrl = String.format(Locale.ROOT, "https://artifacts.elastic.co/downloads/elasticsearch-plugins/%s", pluginId); @@ -393,7 +399,7 @@ private String getElasticUrl( return String.format(Locale.ROOT, "%s/%s-%s.zip", baseUrl, pluginId, Build.current().qualifiedVersion()); } - private static String nonReleaseUrl(final String hostname, final Version version, final String stagingHash, final String pluginId) { + private static String nonReleaseUrl(final String hostname, final String version, final String stagingHash, final String pluginId) { return String.format( Locale.ROOT, "https://%s.elastic.co/%s-%s/downloads/elasticsearch-plugins/%s", @@ -1088,4 +1094,9 @@ private static void setFileAttributes(final Path path, final Set getPluginsToUpgrade( throw new RuntimeException("Couldn't find a PluginInfo for [" + eachPluginId + "], which should be impossible"); }); - if (info.getElasticsearchVersion().before(Version.CURRENT)) { + if (info.getElasticsearchVersion().toString().equals(Build.current().version()) == false) { this.terminal.println( Terminal.Verbosity.VERBOSE, String.format( Locale.ROOT, - "Official plugin [%s] is out-of-date (%s versus %s), upgrading", + "Official plugin [%s] is out-of-sync (%s versus %s), upgrading", eachPluginId, info.getElasticsearchVersion(), - Version.CURRENT + Build.current().version() ) ); return true; @@ -278,14 +278,14 @@ private List getExistingPlugins() throws PluginSyncException { // Check for a version mismatch, unless it's an official plugin since we can upgrade them. if (InstallPluginAction.OFFICIAL_PLUGINS.contains(info.getName()) - && info.getElasticsearchVersion().equals(Version.CURRENT) == false) { + && info.getElasticsearchVersion().toString().equals(Build.current().version()) == false) { this.terminal.errorPrintln( String.format( Locale.ROOT, "WARNING: plugin [%s] was built for Elasticsearch version %s but version %s is required", info.getName(), info.getElasticsearchVersion(), - Version.CURRENT + Build.current().version() ) ); } diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java index 2a66ed3cf4349..2da05d87f831f 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java @@ -32,7 +32,6 @@ import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyDecryptorBuilder; import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyEncryptorBuilder; import org.elasticsearch.Build; -import org.elasticsearch.Version; import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.ProcessInfo; @@ -111,6 +110,7 @@ import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -298,7 +298,7 @@ private static String[] pluginProperties(String name, String[] additionalProps, "version", "1.0", "elasticsearch.version", - Version.CURRENT.toString(), + InstallPluginAction.getSemanticVersion(Build.current().version()), "java.version", System.getProperty("java.specification.version") @@ -724,7 +724,7 @@ public void testPluginPermissions() throws Exception { final Path platformBinDir = platformNameDir.resolve("bin"); Files.createDirectories(platformBinDir); - Files.createFile(tempPluginDir.resolve("fake-" + Version.CURRENT.toString() + ".jar")); + Files.createFile(tempPluginDir.resolve("fake-" + Build.current().version() + ".jar")); Files.createFile(platformBinDir.resolve("fake_executable")); Files.createDirectory(resourcesDir); Files.createFile(resourcesDir.resolve("resource")); @@ -740,7 +740,7 @@ public void testPluginPermissions() throws Exception { final Path platformName = platform.resolve("linux-x86_64"); final Path bin = platformName.resolve("bin"); assert755(fake); - assert644(fake.resolve("fake-" + Version.CURRENT + ".jar")); + assert644(fake.resolve("fake-" + Build.current().version() + ".jar")); assert755(resources); assert644(resources.resolve("resource")); assert755(platform); @@ -1110,8 +1110,8 @@ public void testOfficialPluginSnapshot() throws Exception { String url = String.format( Locale.ROOT, "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip", - Version.CURRENT, - Build.current().qualifiedVersion() + InstallPluginAction.getSemanticVersion(Build.current().version()), + Build.current().version() ); assertInstallPluginFromUrl("analysis-icu", url, "abc123", true); } @@ -1120,8 +1120,8 @@ public void testInstallReleaseBuildOfPluginOnSnapshotBuild() { String url = String.format( Locale.ROOT, "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip", - Version.CURRENT, - Build.current().qualifiedVersion() + InstallPluginAction.getSemanticVersion(Build.current().version()), + Build.current().version() ); // attempting to install a release build of a plugin (no staging ID) on a snapshot build should throw a user exception final UserException e = expectThrows( @@ -1137,9 +1137,9 @@ public void testInstallReleaseBuildOfPluginOnSnapshotBuild() { public void testOfficialPluginStaging() throws Exception { String url = "https://staging.elastic.co/" - + Version.CURRENT + + InstallPluginAction.getSemanticVersion(Build.current().version()) + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" - + Build.current().qualifiedVersion() + + Build.current().version() + ".zip"; assertInstallPluginFromUrl("analysis-icu", url, "abc123", false); } @@ -1148,7 +1148,7 @@ public void testOfficialPlatformPlugin() throws Exception { String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Platforms.PLATFORM_NAME + "-" - + Build.current().qualifiedVersion() + + Build.current().version() + ".zip"; assertInstallPluginFromUrl("analysis-icu", url, null, false); } @@ -1157,16 +1157,16 @@ public void testOfficialPlatformPluginSnapshot() throws Exception { String url = String.format( Locale.ROOT, "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s-%s.zip", - Version.CURRENT, + InstallPluginAction.getSemanticVersion(Build.current().version()), Platforms.PLATFORM_NAME, - Build.current().qualifiedVersion() + Build.current().version() ); assertInstallPluginFromUrl("analysis-icu", url, "abc123", true); } public void testOfficialPlatformPluginStaging() throws Exception { String url = "https://staging.elastic.co/" - + Version.CURRENT + + InstallPluginAction.getSemanticVersion(Build.current().version()) + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Platforms.PLATFORM_NAME + "-" @@ -1580,6 +1580,17 @@ public void testStablePluginWithoutNamedComponentsFile() throws Exception { assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON()); } + public void testGetSemanticVersion() { + assertThat(InstallPluginAction.getSemanticVersion("1.2.3"), equalTo("1.2.3")); + assertThat(InstallPluginAction.getSemanticVersion("123.456.789"), equalTo("123.456.789")); + assertThat(InstallPluginAction.getSemanticVersion("1.2.3-SNAPSHOT"), equalTo("1.2.3")); + assertThat(InstallPluginAction.getSemanticVersion("1.2.3foobar"), equalTo("1.2.3")); + assertThat(InstallPluginAction.getSemanticVersion("1.2.3.4"), equalTo("1.2.3")); + assertThat(InstallPluginAction.getSemanticVersion("1.2"), nullValue()); + assertThat(InstallPluginAction.getSemanticVersion("foo"), nullValue()); + assertThat(InstallPluginAction.getSemanticVersion("foo-1.2.3"), nullValue()); + } + private Map> namedComponentsMap() { Map> result = new LinkedHashMap<>(); Map extensibles = new LinkedHashMap<>(); diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java index e1577f7d101be..b225bc441794a 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java @@ -215,7 +215,7 @@ public void testExistingIncompatiblePlugin() throws Exception { "version", "1.0", "elasticsearch.version", - Version.fromString("1.0.0").toString(), + "1.0.0", "java.version", System.getProperty("java.specification.version"), "classname", diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java index 2c200df2a7d56..9802b4039bb7b 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.plugins.cli; import org.apache.lucene.tests.util.LuceneTestCase; -import org.elasticsearch.Version; +import org.elasticsearch.Build; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.UserException; import org.elasticsearch.common.settings.Settings; @@ -17,6 +17,7 @@ import org.elasticsearch.plugins.PluginTestUtil; import org.elasticsearch.plugins.cli.SyncPluginsAction.PluginChanges; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; import org.hamcrest.Matchers; import org.junit.Before; import org.mockito.InOrder; @@ -26,6 +27,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Objects; import java.util.Optional; import static org.hamcrest.Matchers.containsString; @@ -129,7 +131,7 @@ public void test_getPluginChanges_withPluginToInstall_returnsPluginToInstall() t * since we can't automatically upgrade it. */ public void test_getPluginChanges_withPluginToUpgrade_returnsNoChanges() throws Exception { - createPlugin("my-plugin", Version.CURRENT.previousMajor()); + createPlugin("my-plugin", VersionUtils.getPreviousVersion().toString()); config.setPlugins(List.of(new InstallablePlugin("my-plugin"))); final PluginChanges pluginChanges = action.getPluginChanges(config, Optional.empty()); @@ -142,7 +144,7 @@ public void test_getPluginChanges_withPluginToUpgrade_returnsNoChanges() throws * but needs to be upgraded, then we calculate that the plugin needs to be upgraded. */ public void test_getPluginChanges_withOfficialPluginToUpgrade_returnsPluginToUpgrade() throws Exception { - createPlugin("analysis-icu", Version.CURRENT.previousMajor()); + createPlugin("analysis-icu", VersionUtils.getPreviousVersion().toString()); config.setPlugins(List.of(new InstallablePlugin("analysis-icu"))); final PluginChanges pluginChanges = action.getPluginChanges(config, Optional.empty()); @@ -329,10 +331,11 @@ public void test_performSync_withPluginsToUpgrade_callsUpgradeAction() throws Ex } private void createPlugin(String name) throws IOException { - createPlugin(name, Version.CURRENT); + String semanticVersion = InstallPluginAction.getSemanticVersion(Build.current().version()); + createPlugin(name, Objects.nonNull(semanticVersion) ? semanticVersion : Build.current().version()); } - private void createPlugin(String name, Version version) throws IOException { + private void createPlugin(String name, String version) throws IOException { PluginTestUtil.writePluginProperties( env.pluginsFile().resolve(name), "description", @@ -342,7 +345,7 @@ private void createPlugin(String name, Version version) throws IOException { "version", "1.0", "elasticsearch.version", - version.toString(), + version, "java.version", System.getProperty("java.specification.version"), "classname", From 43f4ff3034f8118a2c1b15fea90320a8203fbcb0 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 11 Oct 2023 14:21:12 +0100 Subject: [PATCH 40/48] Encapsulate snapshot repo cleanup (#100657) Repository cleanup is basically the same as deleting an empty set of snapshots, and in particular it needs the same context as a snapshots deletion. This commit moves the cleanup process and its dependent methods within `SnapshotsDeletion` and removes a great deal of unnecessary argument-passing and other duplication. Relates #100568 --- .../TransportCleanupRepositoryAction.java | 2 + .../blobstore/BlobStoreRepository.java | 371 +++++++++--------- 2 files changed, 180 insertions(+), 193 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index bd9382aeaa758..1a626fe4dce31 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -213,6 +213,8 @@ public void onFailure(Exception e) { public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { startedCleanup = true; logger.debug("Initialized repository cleanup in cluster state for [{}][{}]", repositoryName, repositoryStateId); + // We fork here just to call SnapshotsService#minCompatibleVersion (which may be to expensive to run directly) but + // BlobStoreRepository#cleanup forks again straight away. TODO reduce the forking here. threadPool.executor(ThreadPool.Names.SNAPSHOT) .execute( ActionRunnable.wrap( diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 1e2969b255877..39d11e9d9a4f3 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -849,6 +849,33 @@ public void onFailure(Exception e) { }); } + /** + * Runs cleanup actions on the repository. Increments the repository state id by one before executing any modifications on the + * repository. + * TODO: Add shard level cleanups + * TODO: Add unreferenced index metadata cleanup + *
      + *
    • Deleting stale indices
      • + *
    • Deleting unreferenced root level blobs
      • + *
    + * + * @param repositoryDataGeneration Generation of {@link RepositoryData} at start of process + * @param repositoryFormatIndexVersion Repository format version + * @param listener Listener to complete when done + */ + public void cleanup( + long repositoryDataGeneration, + IndexVersion repositoryFormatIndexVersion, + ActionListener listener + ) { + createSnapshotsDeletion( + List.of(), + repositoryDataGeneration, + repositoryFormatIndexVersion, + listener.delegateFailureAndWrap((delegate, snapshotsDeletion) -> snapshotsDeletion.runCleanup(delegate)) + ); + } + private void createSnapshotsDeletion( Collection snapshotIds, long repositoryDataGeneration, @@ -890,7 +917,7 @@ private void createSnapshotsDeletion( class SnapshotsDeletion { /** - * The IDs of the snapshots to delete. + * The IDs of the snapshots to delete. This collection is empty if the deletion is a repository cleanup. */ private final Collection snapshotIds; @@ -976,7 +1003,7 @@ private record ShardSnapshotMetaDeleteResult( // --------------------------------------------------------------------------------------------------------------------------------- // The overall flow of execution - private void runDelete(SnapshotDeleteListener listener) { + void runDelete(SnapshotDeleteListener listener) { if (useShardGenerations) { // First write the new shard state metadata (with the removed snapshot) and compute deletion targets final ListenableFuture> writeShardMetaDataAndComputeDeletesStep = @@ -1009,7 +1036,7 @@ private void runDelete(SnapshotDeleteListener listener) { listener.onRepositoryDataWritten(newRepositoryData); // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion try (var refs = new RefCountingRunnable(listener::onDone)) { - cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener()); + cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener().map(ignored -> null)); cleanupUnlinkedShardLevelBlobs(writeShardMetaDataAndComputeDeletesStep.result(), refs.acquireListener()); } }, listener::onFailure)); @@ -1026,7 +1053,7 @@ private void runDelete(SnapshotDeleteListener listener) { listener.onDone(); })) { // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion - cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener()); + cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener().map(ignored -> null)); // writeIndexGen finishes on master-service thread so must fork here. snapshotExecutor.execute( @@ -1043,6 +1070,34 @@ private void runDelete(SnapshotDeleteListener listener) { } } + void runCleanup(ActionListener listener) { + final Set survivingIndexIds = originalRepositoryData.getIndices() + .values() + .stream() + .map(IndexId::getId) + .collect(Collectors.toSet()); + final List staleRootBlobs = staleRootBlobs(originalRepositoryData, originalRootBlobs.keySet()); + if (survivingIndexIds.equals(originalIndexContainers.keySet()) && staleRootBlobs.isEmpty()) { + // Nothing to clean up we return + listener.onResponse(new RepositoryCleanupResult(DeleteResult.ZERO)); + } else { + // write new index-N blob to ensure concurrent operations will fail + writeIndexGen( + originalRepositoryData, + originalRepositoryDataGeneration, + repositoryFormatIndexVersion, + Function.identity(), + listener.delegateFailureAndWrap( + // TODO should we pass newRepositoryData to cleanupStaleBlobs()? + (l, newRepositoryData) -> cleanupUnlinkedRootAndIndicesBlobs( + originalRepositoryData, + l.map(RepositoryCleanupResult::new) + ) + ) + ); + } + } + // --------------------------------------------------------------------------------------------------------------------------------- // Updating the shard-level metadata and accumulating results @@ -1251,14 +1306,6 @@ private static List unusedBlobs( // --------------------------------------------------------------------------------------------------------------------------------- // Cleaning up dangling blobs - /** - * Delete any dangling blobs in the repository root (i.e. {@link RepositoryData}, {@link SnapshotInfo} and {@link Metadata} blobs) - * as well as any containers for indices that are now completely unreferenced. - */ - private void cleanupUnlinkedRootAndIndicesBlobs(RepositoryData newRepositoryData, ActionListener listener) { - cleanupStaleBlobs(snapshotIds, originalIndexContainers, originalRootBlobs, newRepositoryData, listener.map(ignored -> null)); - } - private void cleanupUnlinkedShardLevelBlobs( Collection shardDeleteResults, ActionListener listener @@ -1295,201 +1342,139 @@ private Iterator resolveFilesToDelete(Collection snapshotIds, - Map originalIndexContainers, - Map originalRootBlobs, - RepositoryData newRepositoryData, - ActionListener listener - ) { - final var blobsDeleted = new AtomicLong(); - final var bytesDeleted = new AtomicLong(); - try (var listeners = new RefCountingListener(listener.map(ignored -> DeleteResult.of(blobsDeleted.get(), bytesDeleted.get())))) { - - final List staleRootBlobs = staleRootBlobs(newRepositoryData, originalRootBlobs.keySet()); - if (staleRootBlobs.isEmpty() == false) { - staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { - try (ref) { - logStaleRootLevelBlobs(newRepositoryData.getGenId() - 1, snapshotIds, staleRootBlobs); - deleteFromContainer(blobContainer(), staleRootBlobs.iterator()); - for (final var staleRootBlob : staleRootBlobs) { - bytesDeleted.addAndGet(originalRootBlobs.get(staleRootBlob).length()); - } - blobsDeleted.addAndGet(staleRootBlobs.size()); - } catch (Exception e) { - logger.warn( - () -> format( - "[%s] The following blobs are no longer part of any snapshot [%s] but failed to remove them", - metadata.name(), - staleRootBlobs - ), - e - ); - } - })); - } + /** + * Cleans up stale blobs directly under the repository root as well as all indices paths that aren't referenced by any existing + * snapshots. This method is only to be called directly after a new {@link RepositoryData} was written to the repository. + * + * @param newRepositoryData new repository data that was just written + * @param listener listener to invoke with the combined {@link DeleteResult} of all blobs removed in this operation + */ + private void cleanupUnlinkedRootAndIndicesBlobs(RepositoryData newRepositoryData, ActionListener listener) { + final var blobsDeleted = new AtomicLong(); + final var bytesDeleted = new AtomicLong(); + try ( + var listeners = new RefCountingListener(listener.map(ignored -> DeleteResult.of(blobsDeleted.get(), bytesDeleted.get()))) + ) { - final var survivingIndexIds = newRepositoryData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toSet()); - for (final var indexEntry : originalIndexContainers.entrySet()) { - final var indexId = indexEntry.getKey(); - if (survivingIndexIds.contains(indexId)) { - continue; + final List staleRootBlobs = staleRootBlobs(newRepositoryData, originalRootBlobs.keySet()); + if (staleRootBlobs.isEmpty() == false) { + staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { + try (ref) { + logStaleRootLevelBlobs(newRepositoryData.getGenId() - 1, snapshotIds, staleRootBlobs); + deleteFromContainer(blobContainer(), staleRootBlobs.iterator()); + for (final var staleRootBlob : staleRootBlobs) { + bytesDeleted.addAndGet(originalRootBlobs.get(staleRootBlob).length()); + } + blobsDeleted.addAndGet(staleRootBlobs.size()); + } catch (Exception e) { + logger.warn( + () -> format( + "[%s] The following blobs are no longer part of any snapshot [%s] but failed to remove them", + metadata.name(), + staleRootBlobs + ), + e + ); + } + })); } - staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { - try (ref) { - logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexId); - final var deleteResult = indexEntry.getValue().delete(OperationPurpose.SNAPSHOT); - blobsDeleted.addAndGet(deleteResult.blobsDeleted()); - bytesDeleted.addAndGet(deleteResult.bytesDeleted()); - logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexId); - } catch (IOException e) { - logger.warn(() -> format(""" - [%s] index %s is no longer part of any snapshot in the repository, \ - but failed to clean up its index folder""", metadata.name(), indexId), e); + + final var survivingIndexIds = newRepositoryData.getIndices() + .values() + .stream() + .map(IndexId::getId) + .collect(Collectors.toSet()); + for (final var indexEntry : originalIndexContainers.entrySet()) { + final var indexId = indexEntry.getKey(); + if (survivingIndexIds.contains(indexId)) { + continue; } - })); + staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { + try (ref) { + logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexId); + final var deleteResult = indexEntry.getValue().delete(OperationPurpose.SNAPSHOT); + blobsDeleted.addAndGet(deleteResult.blobsDeleted()); + bytesDeleted.addAndGet(deleteResult.bytesDeleted()); + logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexId); + } catch (IOException e) { + logger.warn(() -> format(""" + [%s] index %s is no longer part of any snapshot in the repository, \ + but failed to clean up its index folder""", metadata.name(), indexId), e); + } + })); + } } - } - // If we did the cleanup of stale indices purely using a throttled executor then there would be no backpressure to prevent us from - // falling arbitrarily far behind. But nor do we want to dedicate all the SNAPSHOT threads to stale index cleanups because that - // would slow down other snapshot operations in situations that do not need backpressure. - // - // The solution is to dedicate one SNAPSHOT thread to doing the cleanups eagerly, alongside the throttled executor which spreads - // the rest of the work across the other threads if they are free. If the eager cleanup loop doesn't finish before the next one - // starts then we dedicate another SNAPSHOT thread to the deletions, and so on, until eventually either we catch up or the SNAPSHOT - // pool is fully occupied with blob deletions, which pushes back on other snapshot operations. + // If we did the cleanup of stale indices purely using a throttled executor then there would be no backpressure to prevent us + // from falling arbitrarily far behind. But nor do we want to dedicate all the SNAPSHOT threads to stale index cleanups because + // that would slow down other snapshot operations in situations that do not need backpressure. + // + // The solution is to dedicate one SNAPSHOT thread to doing the cleanups eagerly, alongside the throttled executor which spreads + // the rest of the work across the other threads if they are free. If the eager cleanup loop doesn't finish before the next one + // starts then we dedicate another SNAPSHOT thread to the deletions, and so on, until eventually either we catch up or the + // SNAPSHOT pool is fully occupied with blob deletions, which pushes back on other snapshot operations. - staleBlobDeleteRunner.runSyncTasksEagerly(threadPool.executor(ThreadPool.Names.SNAPSHOT)); - } + staleBlobDeleteRunner.runSyncTasksEagerly(snapshotExecutor); + } - /** - * Runs cleanup actions on the repository. Increments the repository state id by one before executing any modifications on the - * repository. - * TODO: Add shard level cleanups - * TODO: Add unreferenced index metadata cleanup - *
      - *
    • Deleting stale indices
      • - *
    • Deleting unreferenced root level blobs
      • - *
    - * @param originalRepositoryDataGeneration Current repository state id - * @param repositoryFormatIndexVersion version of the updated repository metadata to write - * @param listener Listener to complete when done - */ - public void cleanup( - long originalRepositoryDataGeneration, - IndexVersion repositoryFormatIndexVersion, - ActionListener listener - ) { - try { - if (isReadOnly()) { - throw new RepositoryException(metadata.name(), "cannot run cleanup on readonly repository"); - } - Map originalRootBlobs = blobContainer().listBlobs(OperationPurpose.SNAPSHOT); - final RepositoryData originalRepositoryData = safeRepositoryData(originalRepositoryDataGeneration, originalRootBlobs); - final Map originalIndexContainers = blobStore().blobContainer(indicesPath()) - .children(OperationPurpose.SNAPSHOT); - final Set survivingIndexIds = originalRepositoryData.getIndices() - .values() + // Finds all blobs directly under the repository root path that are not referenced by the current RepositoryData + private static List staleRootBlobs(RepositoryData newRepositoryData, Set originalRootBlobNames) { + final Set allSnapshotIds = newRepositoryData.getSnapshotIds() .stream() - .map(IndexId::getId) + .map(SnapshotId::getUUID) .collect(Collectors.toSet()); - final List staleRootBlobs = staleRootBlobs(originalRepositoryData, originalRootBlobs.keySet()); - if (survivingIndexIds.equals(originalIndexContainers.keySet()) && staleRootBlobs.isEmpty()) { - // Nothing to clean up we return - listener.onResponse(new RepositoryCleanupResult(DeleteResult.ZERO)); - } else { - // write new index-N blob to ensure concurrent operations will fail - writeIndexGen( - originalRepositoryData, - originalRepositoryDataGeneration, - repositoryFormatIndexVersion, - Function.identity(), - listener.delegateFailureAndWrap( - (l, v) -> cleanupStaleBlobs( - Collections.emptyList(), - originalIndexContainers, - originalRootBlobs, - originalRepositoryData, - l.map(RepositoryCleanupResult::new) - ) - ) - ); - } - } catch (Exception e) { - listener.onFailure(e); - } - } - - // Finds all blobs directly under the repository root path that are not referenced by the current RepositoryData - private static List staleRootBlobs(RepositoryData originalRepositoryData, Set originalRootBlobNames) { - final Set allSnapshotIds = originalRepositoryData.getSnapshotIds() - .stream() - .map(SnapshotId::getUUID) - .collect(Collectors.toSet()); - return originalRootBlobNames.stream().filter(blob -> { - if (FsBlobContainer.isTempBlobName(blob)) { - return true; - } - if (blob.endsWith(".dat")) { - final String foundUUID; - if (blob.startsWith(SNAPSHOT_PREFIX)) { - foundUUID = blob.substring(SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()); - assert SNAPSHOT_FORMAT.blobName(foundUUID).equals(blob); - } else if (blob.startsWith(METADATA_PREFIX)) { - foundUUID = blob.substring(METADATA_PREFIX.length(), blob.length() - ".dat".length()); - assert GLOBAL_METADATA_FORMAT.blobName(foundUUID).equals(blob); - } else { - return false; + return originalRootBlobNames.stream().filter(blob -> { + if (FsBlobContainer.isTempBlobName(blob)) { + return true; } - return allSnapshotIds.contains(foundUUID) == false; - } else if (blob.startsWith(INDEX_FILE_PREFIX)) { - // TODO: Include the current generation here once we remove keeping index-(N-1) around from #writeIndexGen - try { - return originalRepositoryData.getGenId() > Long.parseLong(blob.substring(INDEX_FILE_PREFIX.length())); - } catch (NumberFormatException nfe) { - // odd case of an extra file with the index- prefix that we can't identify - return false; + if (blob.endsWith(".dat")) { + final String foundUUID; + if (blob.startsWith(SNAPSHOT_PREFIX)) { + foundUUID = blob.substring(SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()); + assert SNAPSHOT_FORMAT.blobName(foundUUID).equals(blob); + } else if (blob.startsWith(METADATA_PREFIX)) { + foundUUID = blob.substring(METADATA_PREFIX.length(), blob.length() - ".dat".length()); + assert GLOBAL_METADATA_FORMAT.blobName(foundUUID).equals(blob); + } else { + return false; + } + return allSnapshotIds.contains(foundUUID) == false; + } else if (blob.startsWith(INDEX_FILE_PREFIX)) { + // TODO: Include the current generation here once we remove keeping index-(N-1) around from #writeIndexGen + try { + return newRepositoryData.getGenId() > Long.parseLong(blob.substring(INDEX_FILE_PREFIX.length())); + } catch (NumberFormatException nfe) { + // odd case of an extra file with the index- prefix that we can't identify + return false; + } } - } - return false; - }).toList(); - } + return false; + }).toList(); + } - private void logStaleRootLevelBlobs( - long originalRepositoryDataGeneration, - Collection snapshotIds, - List blobsToDelete - ) { - if (logger.isInfoEnabled()) { - // If we're running root level cleanup as part of a snapshot delete we should not log the snapshot- and global metadata - // blobs associated with the just deleted snapshots as they are expected to exist and not stale. Otherwise every snapshot - // delete would also log a confusing INFO message about "stale blobs". - final Set blobNamesToIgnore = snapshotIds.stream() - .flatMap( - snapshotId -> Stream.of( - GLOBAL_METADATA_FORMAT.blobName(snapshotId.getUUID()), - SNAPSHOT_FORMAT.blobName(snapshotId.getUUID()), - INDEX_FILE_PREFIX + originalRepositoryDataGeneration + private void logStaleRootLevelBlobs( + long newestStaleRepositoryDataGeneration, + Collection snapshotIds, + List blobsToDelete + ) { + if (logger.isInfoEnabled()) { + // If we're running root level cleanup as part of a snapshot delete we should not log the snapshot- and global metadata + // blobs associated with the just deleted snapshots as they are expected to exist and not stale. Otherwise every snapshot + // delete would also log a confusing INFO message about "stale blobs". + final Set blobNamesToIgnore = snapshotIds.stream() + .flatMap( + snapshotId -> Stream.of( + GLOBAL_METADATA_FORMAT.blobName(snapshotId.getUUID()), + SNAPSHOT_FORMAT.blobName(snapshotId.getUUID()), + INDEX_FILE_PREFIX + newestStaleRepositoryDataGeneration + ) ) - ) - .collect(Collectors.toSet()); - final List blobsToLog = blobsToDelete.stream().filter(b -> blobNamesToIgnore.contains(b) == false).toList(); - if (blobsToLog.isEmpty() == false) { - logger.info("[{}] Found stale root level blobs {}. Cleaning them up", metadata.name(), blobsToLog); + .collect(Collectors.toSet()); + final List blobsToLog = blobsToDelete.stream().filter(b -> blobNamesToIgnore.contains(b) == false).toList(); + if (blobsToLog.isEmpty() == false) { + logger.info("[{}] Found stale root level blobs {}. Cleaning them up", metadata.name(), blobsToLog); + } } } } From 64047234dcb6f8e22a0094e3d6025328077ed5b1 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 11 Oct 2023 06:37:03 -0700 Subject: [PATCH 41/48] Updating testing and logging for dense_vector dynamic dims (#100546) This adds a test for dynamic dims update mapping merges. Also, this adds logging to help investigate a periodically failing test. related to: https://github.com/elastic/elasticsearch/issues/100502 --- .../60_dense_vector_dynamic_mapping.yml | 24 +++++++++++++++++++ .../vectors/DenseVectorFieldMapperTests.java | 22 +++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml index d2c02fcbff38e..4ef700f807c13 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml @@ -3,6 +3,30 @@ setup: version: ' - 8.10.99' reason: 'Dynamic mapping of floats to dense_vector was added in 8.11' + # Additional logging for issue: https://github.com/elastic/elasticsearch/issues/100502 + - do: + cluster.put_settings: + body: > + { + "persistent": { + "logger.org.elasticsearch.index": "TRACE" + } + } + +--- +teardown: + - skip: + version: ' - 8.10.99' + reason: 'Dynamic mapping of floats to dense_vector was added in 8.11' + + - do: + cluster.put_settings: + body: > + { + "persistent": { + "logger.org.elasticsearch.index": null + } + } --- "Fields with float arrays below the threshold still map as float": diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 2899dab6ff303..183c0083c7da1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -20,7 +20,9 @@ import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.codec.PerFieldMapperCodec; @@ -231,6 +233,26 @@ public void testDims() { } } + public void testMergeDims() throws IOException { + XContentBuilder mapping = mapping(b -> { + b.startObject("field"); + b.field("type", "dense_vector"); + b.endObject(); + }); + MapperService mapperService = createMapperService(mapping); + + mapping = mapping(b -> { + b.startObject("field"); + b.field("type", "dense_vector").field("dims", 4).field("similarity", "cosine").field("index", true); + b.endObject(); + }); + merge(mapperService, mapping); + assertEquals( + XContentHelper.convertToMap(BytesReference.bytes(mapping), false, mapping.contentType()).v2(), + XContentHelper.convertToMap(mapperService.documentMapper().mappingSource().uncompressed(), false, mapping.contentType()).v2() + ); + } + public void testDefaults() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "dense_vector").field("dims", 3))); From e411b57baf5f19159fa6de8766a2fe5718800513 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 11 Oct 2023 06:56:43 -0700 Subject: [PATCH 42/48] Harden discard logic in ExchangeBuffer (#100636) We can leave pages in the ExchangeBuffer if the noMoreInputs flag is set to true after we've checked it but before we add pages to the queue. I can reliably reproduce the testFromLimit by inserting a delay in between. This change hardens the discard logic by moving the check after we've added a Page to the queue. If the noMoreInputs flag is set to true, we will drain the pages from the queue. --- .../operator/exchange/ExchangeBuffer.java | 23 +++-- .../exchange/ExchangeBufferTests.java | 93 +++++++++++++++++++ .../xpack/esql/action/EsqlActionIT.java | 1 - 3 files changed, 106 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeBufferTests.java diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java index 930ced04636f8..df6c09ea1ff97 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java @@ -41,13 +41,12 @@ final class ExchangeBuffer { } void addPage(Page page) { + queue.add(page); + if (queueSize.incrementAndGet() == 1) { + notifyNotEmpty(); + } if (noMoreInputs) { - page.releaseBlocks(); - } else { - queue.add(page); - if (queueSize.incrementAndGet() == 1) { - notifyNotEmpty(); - } + discardPages(); } } @@ -115,13 +114,17 @@ SubscribableListener waitForReading() { } } + private void discardPages() { + Page p; + while ((p = pollPage()) != null) { + p.releaseBlocks(); + } + } + void finish(boolean drainingPages) { noMoreInputs = true; if (drainingPages) { - Page p; - while ((p = pollPage()) != null) { - p.releaseBlocks(); - } + discardPages(); } notifyNotEmpty(); if (drainingPages || queueSize.get() == 0) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeBufferTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeBufferTests.java new file mode 100644 index 0000000000000..4c975c6c07834 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeBufferTests.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator.exchange; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.compute.data.BasicBlockTests; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.MockBlockFactory; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.test.ESTestCase; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.Matchers.equalTo; + +public class ExchangeBufferTests extends ESTestCase { + + public void testDrainPages() throws Exception { + ExchangeBuffer buffer = new ExchangeBuffer(randomIntBetween(10, 1000)); + var blockFactory = blockFactory(); + CountDownLatch latch = new CountDownLatch(1); + Thread[] producers = new Thread[between(1, 4)]; + AtomicBoolean stopped = new AtomicBoolean(); + AtomicInteger addedPages = new AtomicInteger(); + for (int t = 0; t < producers.length; t++) { + producers[t] = new Thread(() -> { + try { + latch.await(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + while (stopped.get() == false && addedPages.incrementAndGet() < 10_000) { + buffer.addPage(randomPage(blockFactory)); + } + }); + producers[t].start(); + } + latch.countDown(); + try { + int minPage = between(10, 100); + int receivedPage = 0; + while (receivedPage < minPage) { + Page p = buffer.pollPage(); + if (p != null) { + p.releaseBlocks(); + ++receivedPage; + } + } + } finally { + buffer.finish(true); + stopped.set(true); + } + for (Thread t : producers) { + t.join(); + } + assertThat(buffer.size(), equalTo(0)); + blockFactory.ensureAllBlocksAreReleased(); + } + + private static MockBlockFactory blockFactory() { + BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofGb(1)).withCircuitBreaking(); + CircuitBreaker breaker = bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST); + return new MockBlockFactory(breaker, bigArrays); + } + + private static Page randomPage(BlockFactory blockFactory) { + Block block = BasicBlockTests.randomBlock( + blockFactory, + randomFrom(ElementType.LONG, ElementType.BYTES_REF, ElementType.BOOLEAN), + randomIntBetween(1, 100), + randomBoolean(), + 0, + between(1, 2), + 0, + between(1, 2) + ).block(); + return new Page(block); + } +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index 0017a8600a013..2712ef8d2f59b 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -862,7 +862,6 @@ public void testFromStatsLimit() { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99826") public void testFromLimit() { try (EsqlQueryResponse results = run("from test | keep data | limit 2")) { logger.info(results); From 18c5246f1aa4bc3a08b2bade2cb67ef3b90e649c Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 11 Oct 2023 06:56:54 -0700 Subject: [PATCH 43/48] Disallow vectors whose magnitudes will not fit in a float (#100519) While we check for a magnitude to not be `0f`, we don't verify that it actually fits within a `float` value. This commit returns a failure and rejects `float` vectors whose magnitude don't fit within a 32bit `float` value. We don't support `float64` (aka `double`) values for vector search and should fail when a user attempts to index a vector that requires storing as `double`. closes: https://github.com/elastic/elasticsearch/issues/100471 --- docs/changelog/100519.yaml | 5 +++ .../vectors/DenseVectorFieldMapper.java | 13 ++++++- .../vectors/DenseVectorFieldMapperTests.java | 34 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/100519.yaml diff --git a/docs/changelog/100519.yaml b/docs/changelog/100519.yaml new file mode 100644 index 0000000000000..086c6962b3a95 --- /dev/null +++ b/docs/changelog/100519.yaml @@ -0,0 +1,5 @@ +pr: 100519 +summary: Disallow vectors whose magnitudes will not fit in a float +area: Vector Search +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index deb178ff724bb..ee144b25f4507 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -458,6 +458,15 @@ void checkVectorMagnitude( ) { StringBuilder errorBuilder = null; + if (Float.isNaN(squaredMagnitude) || Float.isInfinite(squaredMagnitude)) { + errorBuilder = new StringBuilder( + "NaN or Infinite magnitude detected, this usually means the vector values are too extreme to fit within a float." + ); + } + if (errorBuilder != null) { + throw new IllegalArgumentException(appender.apply(errorBuilder).toString()); + } + if (similarity == VectorSimilarity.DOT_PRODUCT && Math.abs(squaredMagnitude - 1.0f) > 1e-4f) { errorBuilder = new StringBuilder( "The [" + VectorSimilarity.DOT_PRODUCT + "] similarity can only be used with unit-length vectors." @@ -886,7 +895,9 @@ public Query createKnnQuery( } elementType.checkVectorBounds(queryVector); - if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) { + if (similarity == VectorSimilarity.DOT_PRODUCT + || similarity == VectorSimilarity.COSINE + || similarity == VectorSimilarity.MAX_INNER_PRODUCT) { float squaredMagnitude = VectorUtil.dotProduct(queryVector, queryVector); elementType.checkVectorMagnitude(similarity, ElementType.errorFloatElementsAppender(queryVector), squaredMagnitude); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 183c0083c7da1..6d562f88a0100 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -413,6 +413,40 @@ public void testCosineWithZeroByteVector() throws Exception { ); } + public void testMaxInnerProductWithValidNorm() throws Exception { + DocumentMapper mapper = createDocumentMapper( + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", 3) + .field("index", true) + .field("similarity", VectorSimilarity.MAX_INNER_PRODUCT) + ) + ); + float[] vector = { -12.1f, 2.7f, -4 }; + // Shouldn't throw + mapper.parse(source(b -> b.array("field", vector))); + } + + public void testWithExtremeFloatVector() throws Exception { + for (VectorSimilarity vs : List.of(VectorSimilarity.COSINE, VectorSimilarity.DOT_PRODUCT, VectorSimilarity.COSINE)) { + DocumentMapper mapper = createDocumentMapper( + fieldMapping(b -> b.field("type", "dense_vector").field("dims", 3).field("index", true).field("similarity", vs)) + ); + float[] vector = { 0.07247924f, -4.310546E-11f, -1.7255947E30f }; + DocumentParsingException e = expectThrows( + DocumentParsingException.class, + () -> mapper.parse(source(b -> b.array("field", vector))) + ); + assertNotNull(e.getCause()); + assertThat( + e.getCause().getMessage(), + containsString( + "NaN or Infinite magnitude detected, this usually means the vector values are too extreme to fit within a float." + ) + ); + } + } + public void testInvalidParameters() { MapperParsingException e = expectThrows( MapperParsingException.class, From 769c3f319c06ab256e3265cb7d1f778ae67570b8 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 11 Oct 2023 14:58:57 +0100 Subject: [PATCH 44/48] Don't use an asserting searcher at all in MatchingDirectoryReader (#100668) Follow up to #100527 We are not testing anything to do with searching with this searcher, and so there is no point in using LuceneTestCase.newSearcher() which will wrap it with all sorts of extra checks that may access the underlying reader in ways that are not anticipated by tests. Fixes #100460 Fixes #99024 --- .../java/org/elasticsearch/index/engine/EngineTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 17f2303eb84c8..ab9d80b801863 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -1509,7 +1509,7 @@ public MatchingDirectoryReader(DirectoryReader in, Query query) throws IOExcepti @Override public LeafReader wrap(LeafReader leaf) { try { - final IndexSearcher searcher = newSearcher(leaf, false, true, false); + final IndexSearcher searcher = new IndexSearcher(leaf); searcher.setQueryCache(null); final Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE_NO_SCORES, 1.0f); final Scorer scorer = weight.scorer(leaf.getContext()); From ae8ef6f534e986c40b9fc0b423fe3c7b8337b2f3 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 11 Oct 2023 15:08:31 +0100 Subject: [PATCH 45/48] [ML] Make ELSER settings serialisation compatible with backport (#100626) #100588 introduced a patch transport version, this PR adds the same patch version and updates the serialisation logic --- .../org/elasticsearch/TransportVersions.java | 1 + .../elser/ElserMlNodeServiceSettings.java | 17 +++++-- .../ElserMlNodeServiceSettingsTests.java | 48 +++++++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 5d51a7959b5fa..6d323c4fc2ea7 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -134,6 +134,7 @@ static TransportVersion def(int id) { public static final TransportVersion NODE_INFO_REQUEST_SIMPLIFIED = def(8_510_00_0); public static final TransportVersion NESTED_KNN_VECTOR_QUERY_V = def(8_511_00_0); public static final TransportVersion ML_PACKAGE_LOADER_PLATFORM_ADDED = def(8_512_00_0); + public static final TransportVersion ELSER_SERVICE_MODEL_VERSION_ADDED_PATCH = def(8_512_00_1); public static final TransportVersion PLUGIN_DESCRIPTOR_OPTIONAL_CLASSNAME = def(8_513_00_0); public static final TransportVersion UNIVERSAL_PROFILING_LICENSE_ADDED = def(8_514_00_0); public static final TransportVersion ELSER_SERVICE_MODEL_VERSION_ADDED = def(8_515_00_0); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettings.java index 7dffbc693ca51..d1f27302f85f1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettings.java @@ -87,10 +87,21 @@ public ElserMlNodeServiceSettings(int numAllocations, int numThreads, String var public ElserMlNodeServiceSettings(StreamInput in) throws IOException { numAllocations = in.readVInt(); numThreads = in.readVInt(); - if (in.getTransportVersion().onOrAfter(TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED)) { + if (transportVersionIsCompatibleWithElserModelVersion(in.getTransportVersion())) { modelVariant = in.readString(); } else { - modelVariant = ElserMlNodeService.ELSER_V1_MODEL; + modelVariant = ElserMlNodeService.ELSER_V2_MODEL; + } + } + + static boolean transportVersionIsCompatibleWithElserModelVersion(TransportVersion transportVersion) { + var nextNonPatchVersion = TransportVersions.PLUGIN_DESCRIPTOR_OPTIONAL_CLASSNAME; + + if (transportVersion.onOrAfter(TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED)) { + return true; + } else { + return transportVersion.onOrAfter(TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED_PATCH) + && transportVersion.before(nextNonPatchVersion); } } @@ -130,7 +141,7 @@ public TransportVersion getMinimalSupportedVersion() { public void writeTo(StreamOutput out) throws IOException { out.writeVInt(numAllocations); out.writeVInt(numThreads); - if (out.getTransportVersion().onOrAfter(TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED)) { + if (transportVersionIsCompatibleWithElserModelVersion(out.getTransportVersion())) { out.writeString(modelVariant); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettingsTests.java index 35d5c0b8e9603..8b6f3f1a56ba6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettingsTests.java @@ -7,10 +7,12 @@ package org.elasticsearch.xpack.inference.services.elser; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -85,6 +87,52 @@ public void testFromMapMissingOptions() { assertThat(e.getMessage(), containsString("[service_settings] does not contain the required setting [num_allocations]")); } + public void testTransportVersionIsCompatibleWithElserModelVersion() { + assertTrue( + ElserMlNodeServiceSettings.transportVersionIsCompatibleWithElserModelVersion( + TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED + ) + ); + assertTrue( + ElserMlNodeServiceSettings.transportVersionIsCompatibleWithElserModelVersion( + TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED_PATCH + ) + ); + + assertFalse( + ElserMlNodeServiceSettings.transportVersionIsCompatibleWithElserModelVersion(TransportVersions.ML_PACKAGE_LOADER_PLATFORM_ADDED) + ); + assertFalse( + ElserMlNodeServiceSettings.transportVersionIsCompatibleWithElserModelVersion( + TransportVersions.PLUGIN_DESCRIPTOR_OPTIONAL_CLASSNAME + ) + ); + assertFalse( + ElserMlNodeServiceSettings.transportVersionIsCompatibleWithElserModelVersion( + TransportVersions.UNIVERSAL_PROFILING_LICENSE_ADDED + ) + ); + } + + public void testBwcWrite() throws IOException { + { + var settings = new ElserMlNodeServiceSettings(1, 1, ".elser_model_1"); + var copy = copyInstance(settings, TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED); + assertEquals(settings, copy); + } + { + var settings = new ElserMlNodeServiceSettings(1, 1, ".elser_model_1"); + var copy = copyInstance(settings, TransportVersions.PLUGIN_DESCRIPTOR_OPTIONAL_CLASSNAME); + assertNotEquals(settings, copy); + assertEquals(".elser_model_2", copy.getModelVariant()); + } + { + var settings = new ElserMlNodeServiceSettings(1, 1, ".elser_model_1"); + var copy = copyInstance(settings, TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED_PATCH); + assertEquals(settings, copy); + } + } + public void testFromMapInvalidSettings() { var settingsMap = new HashMap( Map.of(ElserMlNodeServiceSettings.NUM_ALLOCATIONS, 0, ElserMlNodeServiceSettings.NUM_THREADS, -1) From 7a1784c279cb66ea271e8428ad8c257e16ec1024 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Wed, 11 Oct 2023 16:28:20 +0200 Subject: [PATCH 46/48] ESQL: Paginate MV_EXPAND output (#100598) --- .../compute/operator/MvExpandOperator.java | 188 +++++++++++++--- .../operator/MvExpandOperatorStatusTests.java | 20 +- .../operator/MvExpandOperatorTests.java | 213 ++++++++++++++---- .../compute/operator/OperatorTestCase.java | 8 +- .../src/main/resources/mv_expand.csv-spec | 74 ++++++ .../esql/planner/LocalExecutionPlanner.java | 3 +- 6 files changed, 413 insertions(+), 93 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MvExpandOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MvExpandOperator.java index f6156507dffa2..c322520d8853b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MvExpandOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MvExpandOperator.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.Releasables; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -31,11 +32,12 @@ * 2 | 2 | "foo" * */ -public class MvExpandOperator extends AbstractPageMappingOperator { - public record Factory(int channel) implements OperatorFactory { +public class MvExpandOperator implements Operator { + + public record Factory(int channel, int blockSize) implements OperatorFactory { @Override public Operator get(DriverContext driverContext) { - return new MvExpandOperator(channel); + return new MvExpandOperator(channel, blockSize); } @Override @@ -46,49 +48,158 @@ public String describe() { private final int channel; + private final int pageSize; + private int noops; - public MvExpandOperator(int channel) { + private Page prev; + private boolean prevCompleted = false; + private boolean finished = false; + + private Block expandingBlock; + private Block expandedBlock; + + private int nextPositionToProcess = 0; + private int nextMvToProcess = 0; + private int nextItemOnExpanded = 0; + + /** + * Count of pages that have been processed by this operator. + */ + private int pagesIn; + private int pagesOut; + + public MvExpandOperator(int channel, int pageSize) { this.channel = channel; + this.pageSize = pageSize; + assert pageSize > 0; } @Override - protected Page process(Page page) { - Block expandingBlock = page.getBlock(channel); - Block expandedBlock = expandingBlock.expand(); + public final Page getOutput() { + if (prev == null) { + return null; + } + pagesOut++; + if (prev.getPositionCount() == 0 || expandingBlock.mayHaveMultivaluedFields() == false) { + noops++; + Page result = prev; + prev = null; + return result; + } + + try { + return process(); + } finally { + if (prevCompleted && prev != null) { + prev.releaseBlocks(); + prev = null; + } + } + } + + protected Page process() { if (expandedBlock == expandingBlock) { noops++; - return page; + prevCompleted = true; + return prev; } - if (page.getBlockCount() == 1) { + if (prev.getBlockCount() == 1) { assert channel == 0; + prevCompleted = true; return new Page(expandedBlock); } - int[] duplicateFilter = buildDuplicateExpandingFilter(expandingBlock, expandedBlock.getPositionCount()); + int[] duplicateFilter = nextDuplicateExpandingFilter(); - Block[] result = new Block[page.getBlockCount()]; + Block[] result = new Block[prev.getBlockCount()]; + int[] expandedMask = new int[duplicateFilter.length]; + for (int i = 0; i < expandedMask.length; i++) { + expandedMask[i] = i + nextItemOnExpanded; + } + nextItemOnExpanded += expandedMask.length; for (int b = 0; b < result.length; b++) { - result[b] = b == channel ? expandedBlock : page.getBlock(b).filter(duplicateFilter); + result[b] = b == channel ? expandedBlock.filter(expandedMask) : prev.getBlock(b).filter(duplicateFilter); + } + if (nextItemOnExpanded == expandedBlock.getPositionCount()) { + nextItemOnExpanded = 0; } return new Page(result); } - private int[] buildDuplicateExpandingFilter(Block expandingBlock, int newPositions) { - int[] duplicateFilter = new int[newPositions]; + private int[] nextDuplicateExpandingFilter() { + int[] duplicateFilter = new int[Math.min(pageSize, expandedBlock.getPositionCount() - nextPositionToProcess)]; int n = 0; - for (int p = 0; p < expandingBlock.getPositionCount(); p++) { - int count = expandingBlock.getValueCount(p); + while (true) { + int count = expandingBlock.getValueCount(nextPositionToProcess); int positions = count == 0 ? 1 : count; - Arrays.fill(duplicateFilter, n, n + positions, p); - n += positions; + int toAdd = Math.min(pageSize - n, positions - nextMvToProcess); + Arrays.fill(duplicateFilter, n, n + toAdd, nextPositionToProcess); + n += toAdd; + + if (n == pageSize) { + if (nextMvToProcess + toAdd == positions) { + // finished expanding this position, let's move on to next position (that will be expanded with next call) + nextMvToProcess = 0; + nextPositionToProcess++; + if (nextPositionToProcess == expandingBlock.getPositionCount()) { + nextPositionToProcess = 0; + prevCompleted = true; + } + } else { + // there are still items to expand in current position, but the duplicate filter is full, so we'll deal with them at + // next call + nextMvToProcess = nextMvToProcess + toAdd; + } + return duplicateFilter; + } + + nextMvToProcess = 0; + nextPositionToProcess++; + if (nextPositionToProcess == expandingBlock.getPositionCount()) { + nextPositionToProcess = 0; + nextMvToProcess = 0; + prevCompleted = true; + return n < pageSize ? Arrays.copyOfRange(duplicateFilter, 0, n) : duplicateFilter; + } } - return duplicateFilter; } @Override - protected AbstractPageMappingOperator.Status status(int pagesProcessed) { - return new Status(pagesProcessed, noops); + public final boolean needsInput() { + return prev == null && finished == false; + } + + @Override + public final void addInput(Page page) { + assert prev == null : "has pending input page"; + prev = page; + this.expandingBlock = prev.getBlock(channel); + this.expandedBlock = expandingBlock.expand(); + pagesIn++; + prevCompleted = false; + } + + @Override + public final void finish() { + finished = true; + } + + @Override + public final boolean isFinished() { + return finished && prev == null; + } + + @Override + public final Status status() { + return new Status(pagesIn, pagesOut, noops); + } + + @Override + public void close() { + if (prev != null) { + Releasables.closeExpectNoException(() -> prev.releaseBlocks()); + } } @Override @@ -96,35 +207,42 @@ public String toString() { return "MvExpandOperator[channel=" + channel + "]"; } - public static final class Status extends AbstractPageMappingOperator.Status { + public static final class Status implements Operator.Status { + + private final int pagesIn; + private final int pagesOut; + private final int noops; + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Operator.Status.class, "mv_expand", Status::new ); - private final int noops; - - Status(int pagesProcessed, int noops) { - super(pagesProcessed); + Status(int pagesIn, int pagesOut, int noops) { + this.pagesIn = pagesIn; + this.pagesOut = pagesOut; this.noops = noops; } Status(StreamInput in) throws IOException { - super(in); + pagesIn = in.readVInt(); + pagesOut = in.readVInt(); noops = in.readVInt(); } @Override public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); + out.writeVInt(pagesIn); + out.writeVInt(pagesOut); out.writeVInt(noops); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field("pages_processed", pagesProcessed()); + builder.field("pages_in", pagesIn); + builder.field("pages_out", pagesOut); builder.field("noops", noops); return builder.endObject(); } @@ -147,12 +265,20 @@ public boolean equals(Object o) { return false; } Status status = (Status) o; - return noops == status.noops && pagesProcessed() == status.pagesProcessed(); + return noops == status.noops && pagesIn == status.pagesIn && pagesOut == status.pagesOut; + } + + public int pagesIn() { + return pagesIn; + } + + public int pagesOut() { + return pagesOut; } @Override public int hashCode() { - return Objects.hash(noops, pagesProcessed()); + return Objects.hash(noops, pagesIn, pagesOut); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorStatusTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorStatusTests.java index fe281bbf16131..9527388a0d3cf 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorStatusTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorStatusTests.java @@ -16,12 +16,12 @@ public class MvExpandOperatorStatusTests extends AbstractWireSerializingTestCase { public static MvExpandOperator.Status simple() { - return new MvExpandOperator.Status(10, 9); + return new MvExpandOperator.Status(10, 15, 9); } public static String simpleToJson() { return """ - {"pages_processed":10,"noops":9}"""; + {"pages_in":10,"pages_out":15,"noops":9}"""; } public void testToXContent() { @@ -35,20 +35,28 @@ protected Writeable.Reader instanceReader() { @Override public MvExpandOperator.Status createTestInstance() { - return new MvExpandOperator.Status(randomNonNegativeInt(), randomNonNegativeInt()); + return new MvExpandOperator.Status(randomNonNegativeInt(), randomNonNegativeInt(), randomNonNegativeInt()); } @Override protected MvExpandOperator.Status mutateInstance(MvExpandOperator.Status instance) { - switch (between(0, 1)) { + switch (between(0, 2)) { case 0: return new MvExpandOperator.Status( - randomValueOtherThan(instance.pagesProcessed(), ESTestCase::randomNonNegativeInt), + randomValueOtherThan(instance.pagesIn(), ESTestCase::randomNonNegativeInt), + instance.pagesOut(), instance.noops() ); case 1: return new MvExpandOperator.Status( - instance.pagesProcessed(), + instance.pagesIn(), + randomValueOtherThan(instance.pagesOut(), ESTestCase::randomNonNegativeInt), + instance.noops() + ); + case 2: + return new MvExpandOperator.Status( + instance.pagesIn(), + instance.pagesOut(), randomValueOtherThan(instance.noops(), ESTestCase::randomNonNegativeInt) ); default: diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java index 69c965fc91323..f99685609ff78 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java @@ -9,17 +9,19 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.compute.data.BasicBlockTests; +import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; +import java.util.Iterator; import java.util.List; import static org.elasticsearch.compute.data.BasicBlockTests.randomBlock; import static org.elasticsearch.compute.data.BasicBlockTests.valuesAtPositions; +import static org.elasticsearch.compute.data.BlockTestUtils.deepCopyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -47,7 +49,7 @@ protected Page createPage(int positionOffset, int length) { @Override protected Operator.OperatorFactory simple(BigArrays bigArrays) { - return new MvExpandOperator.Factory(0); + return new MvExpandOperator.Factory(0, randomIntBetween(1, 1000)); } @Override @@ -60,47 +62,143 @@ protected String expectedToStringOfSimple() { return expectedDescriptionOfSimple(); } - @Override - protected void assertSimpleOutput(List input, List results) { - assertThat(results, hasSize(results.size())); - for (int i = 0; i < results.size(); i++) { - IntBlock origExpanded = input.get(i).getBlock(0); - IntBlock resultExpanded = results.get(i).getBlock(0); - int np = 0; - for (int op = 0; op < origExpanded.getPositionCount(); op++) { - if (origExpanded.isNull(op)) { - assertThat(resultExpanded.isNull(np), equalTo(true)); - assertThat(resultExpanded.getValueCount(np++), equalTo(0)); - continue; - } - List oValues = BasicBlockTests.valuesAtPositions(origExpanded, op, op + 1).get(0); - for (Object ov : oValues) { - assertThat(resultExpanded.isNull(np), equalTo(false)); - assertThat(resultExpanded.getValueCount(np), equalTo(1)); - assertThat(BasicBlockTests.valuesAtPositions(resultExpanded, np, ++np).get(0), equalTo(List.of(ov))); + class BlockListIterator implements Iterator { + private final Iterator pagesIterator; + private final int channel; + private Block currentBlock; + private int nextPosition; + + BlockListIterator(List pages, int channel) { + this.pagesIterator = pages.iterator(); + this.channel = channel; + this.currentBlock = pagesIterator.next().getBlock(channel); + this.nextPosition = 0; + } + + @Override + public boolean hasNext() { + if (currentBlock == null) { + return false; + } + + return currentBlock.getValueCount(nextPosition) == 0 + || nextPosition < currentBlock.getPositionCount() + || pagesIterator.hasNext(); + } + + @Override + public Object next() { + if (currentBlock != null && currentBlock.getValueCount(nextPosition) == 0) { + nextPosition++; + if (currentBlock.getPositionCount() == nextPosition) { + loadNextBlock(); } + return null; } + List items = valuesAtPositions(currentBlock, nextPosition, nextPosition + 1).get(0); + nextPosition++; + if (currentBlock.getPositionCount() == nextPosition) { + loadNextBlock(); + } + return items.size() == 1 ? items.get(0) : items; + } - IntBlock origDuplicated = input.get(i).getBlock(1); - IntBlock resultDuplicated = results.get(i).getBlock(1); - np = 0; - for (int op = 0; op < origDuplicated.getPositionCount(); op++) { - int copies = origExpanded.isNull(op) ? 1 : origExpanded.getValueCount(op); - for (int c = 0; c < copies; c++) { - if (origDuplicated.isNull(op)) { - assertThat(resultDuplicated.isNull(np), equalTo(true)); - assertThat(resultDuplicated.getValueCount(np++), equalTo(0)); - continue; - } - assertThat(resultDuplicated.isNull(np), equalTo(false)); - assertThat(resultDuplicated.getValueCount(np), equalTo(origDuplicated.getValueCount(op))); - assertThat( - BasicBlockTests.valuesAtPositions(resultDuplicated, np, ++np).get(0), - equalTo(BasicBlockTests.valuesAtPositions(origDuplicated, op, op + 1).get(0)) - ); + private void loadNextBlock() { + if (pagesIterator.hasNext() == false) { + currentBlock = null; + return; + } + this.currentBlock = pagesIterator.next().getBlock(channel); + nextPosition = 0; + } + } + + class BlockListIteratorExpander implements Iterator { + private final Iterator pagesIterator; + private final int channel; + private Block currentBlock; + private int nextPosition; + private int nextInPosition; + + BlockListIteratorExpander(List pages, int channel) { + this.pagesIterator = pages.iterator(); + this.channel = channel; + this.currentBlock = pagesIterator.next().getBlock(channel); + this.nextPosition = 0; + this.nextInPosition = 0; + } + + @Override + public boolean hasNext() { + if (currentBlock == null) { + return false; + } + + return currentBlock.getValueCount(nextPosition) == 0 + || nextInPosition < currentBlock.getValueCount(nextPosition) + || nextPosition < currentBlock.getPositionCount() + || pagesIterator.hasNext(); + } + + @Override + public Object next() { + if (currentBlock != null && currentBlock.getValueCount(nextPosition) == 0) { + nextPosition++; + if (currentBlock.getPositionCount() == nextPosition) { + loadNextBlock(); } + return null; + } + List items = valuesAtPositions(currentBlock, nextPosition, nextPosition + 1).get(0); + Object result = items == null ? null : items.get(nextInPosition++); + if (nextInPosition == currentBlock.getValueCount(nextPosition)) { + nextPosition++; + nextInPosition = 0; + } + if (currentBlock.getPositionCount() == nextPosition) { + loadNextBlock(); + } + return result; + } + + private void loadNextBlock() { + if (pagesIterator.hasNext() == false) { + currentBlock = null; + return; + } + this.currentBlock = pagesIterator.next().getBlock(channel); + nextPosition = 0; + nextInPosition = 0; + } + } + + @Override + protected void assertSimpleOutput(List input, List results) { + assertThat(results, hasSize(results.size())); + + var inputIter = new BlockListIteratorExpander(input, 0); + var resultIter = new BlockListIteratorExpander(results, 0); + + while (inputIter.hasNext()) { + assertThat(resultIter.hasNext(), equalTo(true)); + assertThat(resultIter.next(), equalTo(inputIter.next())); + } + assertThat(resultIter.hasNext(), equalTo(false)); + + var originalMvIter = new BlockListIterator(input, 0); + var inputIter2 = new BlockListIterator(input, 1); + var resultIter2 = new BlockListIterator(results, 1); + + while (originalMvIter.hasNext()) { + Object originalMv = originalMvIter.next(); + int originalMvSize = originalMv instanceof List l ? l.size() : 1; + assertThat(resultIter2.hasNext(), equalTo(true)); + Object inputValue = inputIter2.next(); + for (int j = 0; j < originalMvSize; j++) { + assertThat(resultIter2.next(), equalTo(inputValue)); } } + assertThat(resultIter2.hasNext(), equalTo(false)); } @Override @@ -110,7 +208,7 @@ protected ByteSizeValue smallEnoughToCircuitBreak() { } public void testNoopStatus() { - MvExpandOperator op = new MvExpandOperator(0); + MvExpandOperator op = new MvExpandOperator(0, randomIntBetween(1, 1000)); List result = drive( op, List.of(new Page(IntVector.newVectorBuilder(2).appendInt(1).appendInt(2).build().asBlock())).iterator(), @@ -118,26 +216,45 @@ public void testNoopStatus() { ); assertThat(result, hasSize(1)); assertThat(valuesAtPositions(result.get(0).getBlock(0), 0, 2), equalTo(List.of(List.of(1), List.of(2)))); - MvExpandOperator.Status status = (MvExpandOperator.Status) op.status(); - assertThat(status.pagesProcessed(), equalTo(1)); + MvExpandOperator.Status status = op.status(); + assertThat(status.pagesIn(), equalTo(1)); + assertThat(status.pagesOut(), equalTo(1)); assertThat(status.noops(), equalTo(1)); } public void testExpandStatus() { - MvExpandOperator op = new MvExpandOperator(0); + MvExpandOperator op = new MvExpandOperator(0, randomIntBetween(1, 1)); var builder = IntBlock.newBlockBuilder(2).beginPositionEntry().appendInt(1).appendInt(2).endPositionEntry(); List result = drive(op, List.of(new Page(builder.build())).iterator(), driverContext()); assertThat(result, hasSize(1)); assertThat(valuesAtPositions(result.get(0).getBlock(0), 0, 2), equalTo(List.of(List.of(1), List.of(2)))); - MvExpandOperator.Status status = (MvExpandOperator.Status) op.status(); - assertThat(status.pagesProcessed(), equalTo(1)); + MvExpandOperator.Status status = op.status(); + assertThat(status.pagesIn(), equalTo(1)); + assertThat(status.pagesOut(), equalTo(1)); assertThat(status.noops(), equalTo(0)); } - // TODO: remove this once possible - // https://github.com/elastic/elasticsearch/issues/99826 - @Override - protected boolean canLeak() { - return true; + public void testExpandWithBytesRefs() { + DriverContext context = driverContext(); + List input = CannedSourceOperator.collectPages(new AbstractBlockSourceOperator(context.blockFactory(), 8 * 1024) { + private int idx; + + @Override + protected int remaining() { + return 10000 - idx; + } + + @Override + protected Page createPage(int positionOffset, int length) { + idx += length; + return new Page( + randomBlock(context.blockFactory(), ElementType.BYTES_REF, length, true, 1, 10, 0, 0).block(), + randomBlock(context.blockFactory(), ElementType.INT, length, false, 1, 10, 0, 0).block() + ); + } + }); + List origInput = deepCopyOf(input, BlockFactory.getNonBreakingInstance()); + List results = drive(new MvExpandOperator(0, randomIntBetween(1, 1000)), input.iterator(), context); + assertSimpleOutput(origInput, results); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java index 63f601669636c..5d881f03bd07f 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java @@ -212,7 +212,7 @@ protected final void assertSimple(DriverContext context, int size) { unreleasedInputs++; } } - if ((canLeak() == false) && unreleasedInputs > 0) { + if (unreleasedInputs > 0) { throw new AssertionError("[" + unreleasedInputs + "] unreleased input blocks"); } } @@ -308,12 +308,6 @@ protected void start(Driver driver, ActionListener driverListener) { } } - // TODO: Remove this once all operators do not leak anymore - // https://github.com/elastic/elasticsearch/issues/99826 - protected boolean canLeak() { - return false; - } - public static void assertDriverContext(DriverContext driverContext) { assertTrue(driverContext.isFinished()); assertThat(driverContext.getSnapshot().releasables(), empty()); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec index 7cc11c6fab5b3..ae27e8f56f9f7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec @@ -24,3 +24,77 @@ a:integer | b:keyword | j:keyword 3 | b | "a" 3 | b | "b" ; + + +explosion +row +a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +c = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +d = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +e = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +f = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +g = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +x = 10000000000000 +| mv_expand a | mv_expand b | mv_expand c | mv_expand d | mv_expand e | mv_expand f | mv_expand g +| limit 10; + +a:integer | b:integer | c:integer | d:integer | e:integer | f:integer | g:integer | x:long +1 | 1 | 1 | 1 | 1 | 1 | 1 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 2 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 3 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 4 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 5 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 6 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 7 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 8 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 9 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 10 | 10000000000000 +; + + +explosionStats +row +a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +c = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +d = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +e = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +x = 10000000000000 +| mv_expand a | mv_expand b | mv_expand c | mv_expand d | mv_expand e +| stats sum_a = sum(a) by b +| sort b; + +//12555000 = sum(1..30) * 30 * 30 * 30 +sum_a:long | b:integer +12555000 | 1 +12555000 | 2 +12555000 | 3 +12555000 | 4 +12555000 | 5 +12555000 | 6 +12555000 | 7 +12555000 | 8 +12555000 | 9 +12555000 | 10 +12555000 | 11 +12555000 | 12 +12555000 | 13 +12555000 | 14 +12555000 | 15 +12555000 | 16 +12555000 | 17 +12555000 | 18 +12555000 | 19 +12555000 | 20 +12555000 | 21 +12555000 | 22 +12555000 | 23 +12555000 | 24 +12555000 | 25 +12555000 | 26 +12555000 | 27 +12555000 | 28 +12555000 | 29 +12555000 | 30 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index b86072e1b6da0..bdc1c948f2055 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -584,7 +584,8 @@ private PhysicalOperation planLimit(LimitExec limit, LocalExecutionPlannerContex private PhysicalOperation planMvExpand(MvExpandExec mvExpandExec, LocalExecutionPlannerContext context) { PhysicalOperation source = plan(mvExpandExec.child(), context); - return source.with(new MvExpandOperator.Factory(source.layout.get(mvExpandExec.target().id()).channel()), source.layout); + int blockSize = 5000;// TODO estimate row size and use context.pageSize() + return source.with(new MvExpandOperator.Factory(source.layout.get(mvExpandExec.target().id()).channel(), blockSize), source.layout); } /** From 9a8503678bc5841e2e3da1198af9cfce246f21a0 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Wed, 11 Oct 2023 16:54:56 +0200 Subject: [PATCH 47/48] added read and delete privilege (#100684) --- .../authz/store/KibanaOwnedReservedRoleDescriptors.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 579638f474b21..dcd7e106b2e81 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -194,7 +194,10 @@ static RoleDescriptor kibanaSystem(String name) { // Fleet telemetry queries Agent Logs indices in kibana task runner RoleDescriptor.IndicesPrivileges.builder().indices("logs-elastic_agent*").privileges("read").build(), // Fleet publishes Agent metrics in kibana task runner - RoleDescriptor.IndicesPrivileges.builder().indices("metrics-fleet_server*").privileges("auto_configure", "write").build(), + RoleDescriptor.IndicesPrivileges.builder() + .indices("metrics-fleet_server*") + .privileges("auto_configure", "read", "write", "delete") + .build(), // Legacy "Alerts as data" used in Security Solution. // Kibana user creates these indices; reads / writes to them. RoleDescriptor.IndicesPrivileges.builder().indices(ReservedRolesStore.ALERTS_LEGACY_INDEX).privileges("all").build(), From 8a6df32de66f99314de28dabf360c5cf43b2d2a9 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Wed, 11 Oct 2023 17:02:27 +0200 Subject: [PATCH 48/48] Update gradle wrapper to 8.4 (#99856) * Remove deprecated forConfigurationTime usage --- .../gradle/wrapper/gradle-wrapper.properties | 4 ++-- .../src/main/resources/minimumGradleVersion | 2 +- gradle/wrapper/gradle-wrapper.properties | 4 ++-- gradlew | 14 +++++++------- .../gradle/wrapper/gradle-wrapper.properties | 4 ++-- x-pack/plugin/eql/qa/correctness/build.gradle | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties index 6c7fa4d4653d2..01f330a93e8fa 100644 --- a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties +++ b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip +distributionSha256Sum=f2b9ed0faf8472cbe469255ae6c86eddb77076c75191741b4a462f33128dd419 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion index 223a939307878..fad03000495ca 100644 --- a/build-tools-internal/src/main/resources/minimumGradleVersion +++ b/build-tools-internal/src/main/resources/minimumGradleVersion @@ -1 +1 @@ -8.3 \ No newline at end of file +8.4 \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 6c7fa4d4653d2..01f330a93e8fa 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip +distributionSha256Sum=f2b9ed0faf8472cbe469255ae6c86eddb77076c75191741b4a462f33128dd419 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 0adc8e1a53214..1aa94a4269074 100755 --- a/gradlew +++ b/gradlew @@ -145,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -153,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -202,11 +202,11 @@ fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties index 6c7fa4d4653d2..01f330a93e8fa 100644 --- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties +++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip +distributionSha256Sum=f2b9ed0faf8472cbe469255ae6c86eddb77076c75191741b4a462f33128dd419 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/x-pack/plugin/eql/qa/correctness/build.gradle b/x-pack/plugin/eql/qa/correctness/build.gradle index 4a72f66c238e3..0008c30f260d6 100644 --- a/x-pack/plugin/eql/qa/correctness/build.gradle +++ b/x-pack/plugin/eql/qa/correctness/build.gradle @@ -14,7 +14,7 @@ dependencies { } File serviceAccountFile = providers.environmentVariable('eql_test_credentials_file') - .orElse(providers.systemProperty('eql.test.credentials.file').forUseAtConfigurationTime()) + .orElse(providers.systemProperty('eql.test.credentials.file')) .map { s -> new File(s)} .getOrNull()