diff --git a/docs/changelog/112571.yaml b/docs/changelog/112571.yaml new file mode 100644 index 0000000000000..f1be2e5c291de --- /dev/null +++ b/docs/changelog/112571.yaml @@ -0,0 +1,17 @@ +pr: 112571 +summary: Deprecate dot-prefixed indices and composable template index patterns +area: CRUD +type: deprecation +issues: [] +deprecation: + title: Deprecate dot-prefixed indices and composable template index patterns + area: CRUD + details: "Indices beginning with a dot '.' are reserved for system and internal\ + \ indices, and should not be used by and end-user. Additionally, composable index\ + \ templates that contain patterns for dot-prefixed indices should also be avoided,\ + \ as these patterns are meant for internal use only. In a future Elasticsearch\ + \ version, creation of these dot-prefixed indices will no longer be allowed." + impact: "Requests performing an action that would create an index beginning with\ + \ a dot (indexing a document, manual creation, reindex), or creating an index\ + \ template with index patterns beginning with a dot, will contain a deprecation\ + \ header warning about dot-prefixed indices in the response." diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index daf0c188cc83e..d5ce1bfc8d93d 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -40,3 +40,7 @@ if (BuildParams.isSnapshotBuild() == false) { systemProperty 'es.failure_store_feature_flag_enabled', 'true' } } + +tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.skipTest("data_stream/10_basic/Create hidden data stream", "warning does not exist for compatibility") +}) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java index f8037eba899f4..ff842d435463f 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.datastreams; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -71,10 +72,12 @@ public void testHiddenDataStreamImplicitHiddenSearch() throws IOException { // Create a template Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/hidden"); putComposableIndexTemplateRequest.setJsonEntity("{\"index_patterns\": [\".hidden\"], \"data_stream\": {\"hidden\": true}}"); + putComposableIndexTemplateRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); assertOK(client().performRequest(putComposableIndexTemplateRequest)); Request createDocRequest = new Request("POST", "/.hidden/_doc?refresh=true"); createDocRequest.setJsonEntity("{ \"@timestamp\": \"2020-10-22\", \"a\": 1 }"); + createDocRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); assertOK(client().performRequest(createDocRequest)); diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml index 7d375d14f8d3a..044ea90fec1af 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml @@ -115,8 +115,10 @@ setup: "Create hidden data stream": - requires: cluster_features: ["gte_v7.11.0"] + test_runner_features: ["warnings", "headers"] reason: "hidden data streams only available in 7.11" - do: + headers: { X-elastic-product-origin: elastic } allowed_warnings: - "index template [my-template3] has index patterns [.hidden-data-stream, hidden-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation" indices.put_index_template: diff --git a/modules/dot-prefix-validation/build.gradle b/modules/dot-prefix-validation/build.gradle new file mode 100644 index 0000000000000..b3ad3dd4aa01f --- /dev/null +++ b/modules/dot-prefix-validation/build.gradle @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-cluster-test' + +esplugin { + description 'Validation for dot-prefixed indices for non-operator users' + classname 'org.elasticsearch.validation.DotPrefixValidationPlugin' +} + +restResources { + restApi { + include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest', 'bulk', 'reindex' + } +} + +tasks.named('yamlRestTest') { + usesDefaultDistribution() +} diff --git a/modules/dot-prefix-validation/src/main/java/module-info.java b/modules/dot-prefix-validation/src/main/java/module-info.java new file mode 100644 index 0000000000000..4d0c92f255e7d --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/module-info.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module org.elasticsearch.validation { + requires org.elasticsearch.server; + requires org.elasticsearch.base; +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java new file mode 100644 index 0000000000000..ec3c22620ca46 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.admin.indices.create.AutoCreateAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; + +import java.util.Set; + +public class AutoCreateDotValidator extends DotPrefixValidator { + public AutoCreateDotValidator(ThreadContext threadContext, ClusterService clusterService) { + super(threadContext, clusterService); + } + + @Override + protected Set getIndicesFromRequest(CreateIndexRequest request) { + return Set.of(request.index()); + } + + @Override + public String actionName() { + return AutoCreateAction.NAME; + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java new file mode 100644 index 0000000000000..f39a1d09fa07c --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; + +import java.util.Set; + +public class CreateIndexDotValidator extends DotPrefixValidator { + public CreateIndexDotValidator(ThreadContext threadContext, ClusterService clusterService) { + super(threadContext, clusterService); + } + + @Override + protected Set getIndicesFromRequest(CreateIndexRequest request) { + return Set.of(request.index()); + } + + @Override + public String actionName() { + return TransportCreateIndexAction.TYPE.name(); + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java new file mode 100644 index 0000000000000..bca90147c869d --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.support.MappedActionFilter; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +public class DotPrefixValidationPlugin extends Plugin implements ActionPlugin { + private final AtomicReference> actionFilters = new AtomicReference<>(); + + public DotPrefixValidationPlugin() {} + + @Override + public Collection createComponents(PluginServices services) { + ThreadContext context = services.threadPool().getThreadContext(); + ClusterService clusterService = services.clusterService(); + + actionFilters.set( + List.of( + new CreateIndexDotValidator(context, clusterService), + new AutoCreateDotValidator(context, clusterService), + new IndexTemplateDotValidator(context, clusterService) + ) + ); + + return Set.of(); + } + + @Override + public Collection getMappedActionFilters() { + return actionFilters.get(); + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java new file mode 100644 index 0000000000000..e2c75a6401187 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilterChain; +import org.elasticsearch.action.support.MappedActionFilter; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.tasks.Task; + +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * DotPrefixValidator provides an abstract class implementing a mapped action filter. + * + * This class then implements the {@link #apply(Task, String, ActionRequest, ActionListener, ActionFilterChain)} + * method which checks for indices in the request that begin with a dot, emitting a deprecation + * warning if they do. If the request is performed by a non-external user (operator, internal product, etc.) + * as defined by {@link #isInternalRequest()} then the deprecation is emitted. Otherwise, it is skipped. + * + * The indices for consideration are returned by the abstract {@link #getIndicesFromRequest(Object)} + * method, which subclasses must implement. + * + * Some built-in index names and patterns are also elided from the check, as defined in + * {@link #IGNORED_INDEX_NAMES} and {@link #IGNORED_INDEX_PATTERNS}. + */ +public abstract class DotPrefixValidator implements MappedActionFilter { + public static final Setting VALIDATE_DOT_PREFIXES = Setting.boolSetting( + "cluster.indices.validate_dot_prefixes", + true, + Setting.Property.NodeScope + ); + + /** + * Names and patterns for indexes where no deprecation should be emitted. + * Normally we would want to transition these to either system indices, or + * to use an internal origin for the client. These are shorter-term + * workarounds until that work can be completed. + * + * .elastic-connectors-* is used by enterprise search + * .ml-* is used by ML + */ + private static Set IGNORED_INDEX_NAMES = Set.of( + ".elastic-connectors-v1", + ".elastic-connectors-sync-jobs-v1", + ".ml-state", + ".ml-anomalies-unrelated" + ); + private static Set IGNORED_INDEX_PATTERNS = Set.of(Pattern.compile("\\.ml-state-\\d+")); + + DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DotPrefixValidator.class); + + private final ThreadContext threadContext; + private final boolean isEnabled; + + public DotPrefixValidator(ThreadContext threadContext, ClusterService clusterService) { + this.threadContext = threadContext; + this.isEnabled = VALIDATE_DOT_PREFIXES.get(clusterService.getSettings()); + } + + protected abstract Set getIndicesFromRequest(RequestType request); + + @SuppressWarnings("unchecked") + @Override + public void apply( + Task task, + String action, + Request request, + ActionListener listener, + ActionFilterChain chain + ) { + Set indices = getIndicesFromRequest((RequestType) request); + if (isEnabled) { + validateIndices(indices); + } + chain.proceed(task, action, request, listener); + } + + void validateIndices(@Nullable Set indices) { + if (indices != null && isInternalRequest() == false) { + for (String index : indices) { + if (Strings.hasLength(index)) { + char c = getFirstChar(index); + if (c == '.') { + if (IGNORED_INDEX_NAMES.contains(index)) { + return; + } + if (IGNORED_INDEX_PATTERNS.stream().anyMatch(p -> p.matcher(index).matches())) { + return; + } + deprecationLogger.warn( + DeprecationCategory.INDICES, + "dot-prefix", + "Index [{}] name begins with a dot (.), which is deprecated, " + + "and will not be allowed in a future Elasticsearch version.", + index + ); + } + } + } + } + } + + private static char getFirstChar(String index) { + char c = index.charAt(0); + if (c == '<') { + // Date-math is being used for the index, we need to + // consider it by stripping the first '<' before we + // check for a dot-prefix + String strippedLeading = index.substring(1); + if (Strings.hasLength(strippedLeading)) { + c = strippedLeading.charAt(0); + } + } + return c; + } + + private boolean isInternalRequest() { + final String actionOrigin = threadContext.getTransient(ThreadContext.ACTION_ORIGIN_TRANSIENT_NAME); + final boolean isSystemContext = threadContext.isSystemContext(); + final boolean isInternalOrigin = Optional.ofNullable(actionOrigin).map(Strings::hasText).orElse(false); + final boolean hasElasticOriginHeader = Optional.ofNullable(threadContext.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER)) + .map(Strings::hasText) + .orElse(false); + return isSystemContext || isInternalOrigin || hasElasticOriginHeader; + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java new file mode 100644 index 0000000000000..dd9b0feeab388 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class IndexTemplateDotValidator extends DotPrefixValidator { + public IndexTemplateDotValidator(ThreadContext threadContext, ClusterService clusterService) { + super(threadContext, clusterService); + } + + @Override + protected Set getIndicesFromRequest(TransportPutComposableIndexTemplateAction.Request request) { + return new HashSet<>(Arrays.asList(request.indices())); + } + + @Override + public String actionName() { + return TransportPutComposableIndexTemplateAction.TYPE.name(); + } +} diff --git a/modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java b/modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..9ecf6b9ba7d83 --- /dev/null +++ b/modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.LocalClusterSpecBuilder; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; + +import static org.elasticsearch.test.cluster.FeatureFlag.FAILURE_STORE_ENABLED; + +public class DotPrefixClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public DotPrefixClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return createParameters(); + } + + private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password")); + + @Override + protected Settings restClientSettings() { + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); + } + + @ClassRule + public static ElasticsearchCluster cluster = createCluster(); + + private static ElasticsearchCluster createCluster() { + LocalClusterSpecBuilder clusterBuilder = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .feature(FAILURE_STORE_ENABLED) + .setting("xpack.security.enabled", "true") + .keystore("bootstrap.password", "x-pack-test-password") + .user("x_pack_rest_user", "x-pack-test-password"); + boolean setNodes = Boolean.parseBoolean(System.getProperty("yaml.rest.tests.set_num_nodes", "true")); + if (setNodes) { + clusterBuilder.nodes(2); + } + return clusterBuilder.build(); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + +} diff --git a/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml b/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml new file mode 100644 index 0000000000000..b160af4ee8290 --- /dev/null +++ b/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml @@ -0,0 +1,193 @@ +--- +teardown: + - do: + indices.delete: + index: .* + +--- +"Index creation with a dot-prefix is deprecated unless x-elastic-product-origin set": + - requires: + test_runner_features: ["warnings", "warnings_regex", "headers"] + + - do: + warnings: + - "Index [.myindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + indices.create: + index: .myindex + body: + settings: + index.hidden: true + + - do: + headers: { X-elastic-product-origin: kibana } + indices.create: + index: .myindex2 + body: + settings: + index.hidden: true + + - do: + warnings_regex: + - "Index \\[.*\\] name begins with a dot \\(\\.\\), which is deprecated, and will not be allowed in a future Elasticsearch version\\." + indices.create: + index: <.myindex-{now/d}> + body: + settings: + index.hidden: true + + - do: + headers: { X-elastic-product-origin: kibana } + indices.create: + index: <.myindex2-{now/d}> + body: + settings: + index.hidden: true + +--- +"Deprecated auto-creation of dot-prefixed indices": + - requires: + test_runner_features: ["warnings", "headers"] + + - do: + warnings: + - "Index [.myindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.myindex] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + index: + index: .myindex + id: "1" + body: {foo: bar} + + - do: + headers: { X-elastic-product-origin: kibana } + warnings: + - "index name [.myindex2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + index: + index: .myindex2 + id: "1" + body: {foo: bar} + + - do: + warnings: + - "Index [.myindex3] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.myindex3] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + bulk: + body: + - index: + _index: .myindex3 + - message: foo + +--- +"Deprecated auto-creation of dot-prefixed indices through pipelines": + - requires: + test_runner_features: ["warnings", "headers"] + + - do: + ingest.put_pipeline: + id: mypipeline + body: > + { + "processors": [ + { + "set" : { + "field" : "_index", + "value": "{{redirect_to}}" + } + } + ] + } + - do: + warnings: + - "Index [.other] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.other] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + index: + index: myindex + id: "1" + body: {redirect_to: ".other"} + pipeline: mypipeline + + - do: + warnings: + - "Index [.other2] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.other2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + bulk: + body: + - index: + _index: myindex + pipeline: mypipeline + - redirect_to: .other2 + + - do: + index: + index: original + id: "1" + body: { "redirect_to": ".other3" } + - do: + indices.refresh: {} + - do: + warnings: + - "Index [.other3] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.other3] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + reindex: + body: + source: + index: original + dest: + index: newindex + pipeline: mypipeline + + - do: + warnings: + - "Index [.reindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.reindex] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + reindex: + body: + source: + index: original + dest: + index: .reindex + + - do: + headers: { X-elastic-product-origin: kibana } + warnings: + - "index name [.reindex2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + reindex: + body: + source: + index: original + dest: + index: .reindex2 + + - do: + ingest.delete_pipeline: + id: mypipeline + +--- +"Deprecated index template with a dot prefix index pattern": + - requires: + test_runner_features: ["warnings", "headers"] + + - do: + warnings: + - "Index [.data-*] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + indices.put_index_template: + name: my-template + body: + index_patterns: [regular, .data-*] + data_stream: {} + + - do: + headers: { X-elastic-product-origin: kibana } + warnings: + indices.put_index_template: + name: my-template2 + body: + index_patterns: [other, .data2-*] + data_stream: {} + + - do: + indices.delete_index_template: + name: my-template + + - do: + indices.delete_index_template: + name: my-template2 diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 6da35c1ed9529..a5b7ae8d703ea 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -58,6 +58,9 @@ excludeList.add('cluster.desired_nodes/11_old_format/Test node version must have excludeList.add('cluster.desired_nodes/11_old_format/Test node version can not be null') excludeList.add('cluster.desired_nodes/20_dry_run/Test validation works for dry run updates') +// Excluded because they create dot-prefixed indices on older versions +excludeList.add('indices.resolve_index/20_resolve_system_index/*') + BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> if (bwcVersion != VersionProperties.getElasticsearchVersion()) { diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java index ddd5e3f695368..25cfbd7949611 100644 --- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java @@ -10,6 +10,7 @@ package org.elasticsearch.system.indices; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -43,13 +44,17 @@ protected Settings restClientSettings() { } public void testCreatingSystemIndex() throws Exception { - Response response = client().performRequest(new Request("PUT", "/_net_new_sys_index/_create")); + var request = new Request("PUT", "/_net_new_sys_index/_create"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); + Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @SuppressWarnings("unchecked") public void testGetFeatureUpgradedStatuses() throws Exception { - client().performRequest(new Request("PUT", "/_net_new_sys_index/_create")); + var request = new Request("PUT", "/_net_new_sys_index/_create"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); + client().performRequest(request); Response response = client().performRequest(new Request("GET", "/_migration/system_features")); assertThat(response.getStatusLine().getStatusCode(), is(200)); XContentTestUtils.JsonMapView view = XContentTestUtils.createJsonMapView(response.getEntity().getContent()); diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java index a5089a5a68263..d2c48d1e9b609 100644 --- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java @@ -12,6 +12,7 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.settings.SecureString; @@ -40,7 +41,9 @@ public void testCreatingSystemIndex() throws Exception { ); assertThat(EntityUtils.toString(e.getResponse().getEntity()), containsString("system")); - Response response = client().performRequest(new Request("PUT", "/_net_new_sys_index/_create")); + var request = new Request("PUT", "/_net_new_sys_index/_create"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); + Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -56,6 +59,7 @@ public void testIndexDoc() throws Exception { Request request = new Request("PUT", "/_net_new_sys_index/" + id); request.setJsonEntity("{}"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -74,6 +78,7 @@ public void testSearch() throws Exception { Request request = new Request("PUT", "/_net_new_sys_index/" + id); request.setJsonEntity("{}"); request.addParameter("refresh", "true"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java index a40941f503e49..6109a1cf9dc31 100644 --- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java @@ -10,6 +10,7 @@ package org.elasticsearch.system.indices; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -42,6 +43,7 @@ public void testCreatingSystemIndexWithAlias() throws Exception { { Request request = new Request("PUT", "/.internal-unmanaged-index-8"); request.setJsonEntity("{\"aliases\": {\".internal-unmanaged-alias\": {}}}"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -68,6 +70,7 @@ public void testCreatingSystemIndexWithLegacyAlias() throws Exception { { Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -79,6 +82,7 @@ public void testCreatingSystemIndexWithLegacyAlias() throws Exception { public void testCreatingSystemIndexWithIndexAliasEndpoint() throws Exception { { Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -102,6 +106,7 @@ public void testCreatingSystemIndexWithIndexAliasEndpoint() throws Exception { public void testCreatingSystemIndexWithAliasEndpoint() throws Exception { { Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -126,6 +131,7 @@ public void testCreatingSystemIndexWithAliasEndpoint() throws Exception { public void testCreatingSystemIndexWithAliasesEndpoint() throws Exception { { Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/20_hidden.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/20_hidden.yml index 619d2d5374b99..8de0cda460767 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/20_hidden.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/20_hidden.yml @@ -44,7 +44,9 @@ - requires: cluster_features: ["gte_v8.11.0"] reason: "dataset size was added in 8.11.0" + test_runner_features: ["headers"] - do: + headers: { X-elastic-product-origin: elastic } indices.create: index: .index1 body: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/20_resolve_system_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/20_resolve_system_index.yml index 0773d58758486..29e16dc084fa7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/20_resolve_system_index.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/20_resolve_system_index.yml @@ -3,9 +3,10 @@ setup: - requires: cluster_features: ["gte_v8.13.0"] reason: "resolve cluster introduced in 8.13" - test_runner_features: "warnings" + test_runner_features: ["warnings", "headers"] - do: + headers: { X-elastic-product-origin: elastic } indices.create: index: ".tasks" --- diff --git a/x-pack/plugin/security/qa/profile/src/javaRestTest/java/org/elasticsearch/xpack/security/profile/ProfileIT.java b/x-pack/plugin/security/qa/profile/src/javaRestTest/java/org/elasticsearch/xpack/security/profile/ProfileIT.java index b11c8fd39fe2c..cbf70b7735863 100644 --- a/x-pack/plugin/security/qa/profile/src/javaRestTest/java/org/elasticsearch/xpack/security/profile/ProfileIT.java +++ b/x-pack/plugin/security/qa/profile/src/javaRestTest/java/org/elasticsearch/xpack/security/profile/ProfileIT.java @@ -204,7 +204,7 @@ public void testGetProfiles() throws IOException { expectWarnings( "this request accesses system indices: [.security-profile-8], but in a future major version, " + "direct access to system indices will be prevented by default" - ) + ).toBuilder().addHeader("X-elastic-product-origin", "elastic") ); assertOK(adminClient().performRequest(indexRequest)); } @@ -515,7 +515,7 @@ public void testXpackUsageOutput() throws IOException { expectWarnings( "this request accesses system indices: [.security-profile-8], but in a future major version, " + "direct access to system indices will be prevented by default" - ) + ).toBuilder().addHeader("X-elastic-product-origin", "elastic") ); assertOK(adminClient().performRequest(indexRequest)); diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java index 38cda5bbed0f2..a10668f4b8a0a 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java @@ -9,6 +9,7 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -41,6 +42,7 @@ public void testHiddenIndexWithVisibleAlias() throws IOException { createRequest.setJsonEntity( "{\"settings\": {\"index.hidden\": true, \"number_of_replicas\": 0}, \"aliases\": {\"index_allowed\": {}}}" ); + createRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); final Response createResponse = adminClient().performRequest(createRequest); assertOK(createResponse); ensureGreen("index_allowed"); @@ -73,6 +75,7 @@ public void testHiddenIndexWithHiddenAlias() throws IOException { } } }"""); + createRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); final Response createResponse = adminClient().performRequest(createRequest); assertOK(createResponse); ensureGreen("index_allowed"); diff --git a/x-pack/plugin/stack/qa/rest/build.gradle b/x-pack/plugin/stack/qa/rest/build.gradle index a5bdb388d610f..ff2d67eda82f6 100644 --- a/x-pack/plugin/stack/qa/rest/build.gradle +++ b/x-pack/plugin/stack/qa/rest/build.gradle @@ -17,3 +17,7 @@ testClusters.configureEach { setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' } + +tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.skipTest("stack/10_basic/Test kibana reporting index auto creation", "warning does not exist for compatibility") +}) diff --git a/x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml b/x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml index 5317123783921..3b8bfa9dfb7c8 100644 --- a/x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml +++ b/x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml @@ -223,7 +223,11 @@ setup: --- "Test kibana reporting index auto creation": + - requires: + test_runner_features: ["headers"] + - do: + headers: { X-elastic-product-origin: kibana } index: index: .kibana-reporting-foo body: diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java index 477ce2127ed42..7e31b7ec0c5e4 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java @@ -103,7 +103,7 @@ public void testAliasCreatedforBWCIndexes() throws Exception { + "] starts " + "with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices " + "and system indices" - ); + ).toBuilder().addHeader("X-elastic-product-origin", "elastic").build(); Request request = new Request("PUT", "/" + TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED); String entity = "{\"settings\": " + Strings.toString(settings.build()) + "}"; request.setJsonEntity(entity); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformConfigurationIndexIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformConfigurationIndexIT.java index b2a78b1840d99..62056cc9449dc 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformConfigurationIndexIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformConfigurationIndexIT.java @@ -40,7 +40,7 @@ public void testDeleteConfigurationLeftOver() throws IOException { + TransformInternalIndexConstants.LATEST_INDEX_NAME + "], but in a future major version, direct access to system indices will " + "be prevented by default" - ); + ).toBuilder().addHeader("X-elastic-product-origin", "elastic").build(); try (XContentBuilder builder = jsonBuilder()) { builder.startObject();