Deprecate dot-prefixed indices and composable template index patterns (…

…elastic#112571)

This commit adds a module emitting a deprecation warning when a
dot-prefixed index is manually or automatically created, or when a
composable index template with an index pattern that uses a dot-prefix
is created. This pattern warns that in the future these indices will not
be allowed. In a future breaking change (10.0.0 maybe?) the deprecation
can then be changed to an exception.

These deprecations are only displayed when a non-operator user is using
the API (one that does not set the `X-elastic-product-origin` header).

docs/changelog/112571.yaml

@@ -0,0 +1,17 @@
+pr: 112571
+summary: Deprecate dot-prefixed indices and composable template index patterns
+area: CRUD
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate dot-prefixed indices and composable template index patterns
+  area: CRUD
+  details: "Indices beginning with a dot '.' are reserved for system and internal\
+    \ indices, and should not be used by and end-user. Additionally, composable index\
+    \ templates that contain patterns for dot-prefixed indices should also be avoided,\
+    \ as these patterns are meant for internal use only. In a future Elasticsearch\
+    \ version, creation of these dot-prefixed indices will no longer be allowed."
+  impact: "Requests performing an action that would create an index beginning with\
+    \ a dot (indexing a document, manual creation, reindex), or creating an index\
+    \ template with index patterns beginning with a dot, will contain a deprecation\
+    \ header warning about dot-prefixed indices in the response."

modules/data-streams/build.gradle

@@ -40,3 +40,7 @@ if (BuildParams.isSnapshotBuild() == false) {
     systemProperty 'es.failure_store_feature_flag_enabled', 'true'
   }
 }
+
+tasks.named("yamlRestCompatTestTransform").configure({ task ->
+  task.skipTest("data_stream/10_basic/Create hidden data stream", "warning does not exist for compatibility")
+})

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.datastreams;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -71,10 +72,12 @@ public void testHiddenDataStreamImplicitHiddenSearch() throws IOException {
         // Create a template
         Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/hidden");
         putComposableIndexTemplateRequest.setJsonEntity("{\"index_patterns\": [\".hidden\"], \"data_stream\": {\"hidden\": true}}");
+        putComposableIndexTemplateRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
         assertOK(client().performRequest(putComposableIndexTemplateRequest));
 
         Request createDocRequest = new Request("POST", "/.hidden/_doc?refresh=true");
         createDocRequest.setJsonEntity("{ \"@timestamp\": \"2020-10-22\", \"a\": 1 }");
+        createDocRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
 
         assertOK(client().performRequest(createDocRequest));
 

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml

@@ -115,8 +115,10 @@ setup:
 "Create hidden data stream":
   - requires:
       cluster_features: ["gte_v7.11.0"]
+      test_runner_features: ["warnings", "headers"]
       reason: "hidden data streams only available in 7.11"
   - do:
+      headers: { X-elastic-product-origin: elastic }
       allowed_warnings:
         - "index template [my-template3] has index patterns [.hidden-data-stream, hidden-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation"
       indices.put_index_template:

modules/dot-prefix-validation/build.gradle

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
+
+esplugin {
+  description 'Validation for dot-prefixed indices for non-operator users'
+  classname 'org.elasticsearch.validation.DotPrefixValidationPlugin'
+}
+
+restResources {
+  restApi {
+    include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest', 'bulk', 'reindex'
+  }
+}
+
+tasks.named('yamlRestTest') {
+  usesDefaultDistribution()
+}

modules/dot-prefix-validation/src/main/java/module-info.java

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module org.elasticsearch.validation {
+    requires org.elasticsearch.server;
+    requires org.elasticsearch.base;
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.Set;
+
+public class AutoCreateDotValidator extends DotPrefixValidator<CreateIndexRequest> {
+    public AutoCreateDotValidator(ThreadContext threadContext, ClusterService clusterService) {
+        super(threadContext, clusterService);
+    }
+
+    @Override
+    protected Set<String> getIndicesFromRequest(CreateIndexRequest request) {
+        return Set.of(request.index());
+    }
+
+    @Override
+    public String actionName() {
+        return AutoCreateAction.NAME;
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.Set;
+
+public class CreateIndexDotValidator extends DotPrefixValidator<CreateIndexRequest> {
+    public CreateIndexDotValidator(ThreadContext threadContext, ClusterService clusterService) {
+        super(threadContext, clusterService);
+    }
+
+    @Override
+    protected Set<String> getIndicesFromRequest(CreateIndexRequest request) {
+        return Set.of(request.index());
+    }
+
+    @Override
+    public String actionName() {
+        return TransportCreateIndexAction.TYPE.name();
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.support.MappedActionFilter;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class DotPrefixValidationPlugin extends Plugin implements ActionPlugin {
+    private final AtomicReference<List<MappedActionFilter>> actionFilters = new AtomicReference<>();
+
+    public DotPrefixValidationPlugin() {}
+
+    @Override
+    public Collection<?> createComponents(PluginServices services) {
+        ThreadContext context = services.threadPool().getThreadContext();
+        ClusterService clusterService = services.clusterService();
+
+        actionFilters.set(
+            List.of(
+                new CreateIndexDotValidator(context, clusterService),
+                new AutoCreateDotValidator(context, clusterService),
+                new IndexTemplateDotValidator(context, clusterService)
+            )
+        );
+
+        return Set.of();
+    }
+
+    @Override
+    public Collection<MappedActionFilter> getMappedActionFilters() {
+        return actionFilters.get();
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.ActionFilterChain;
+import org.elasticsearch.action.support.MappedActionFilter;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.tasks.Task;
+
+import java.util.Optional;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * DotPrefixValidator provides an abstract class implementing a mapped action filter.
+ *
+ * This class then implements the {@link #apply(Task, String, ActionRequest, ActionListener, ActionFilterChain)}
+ * method which checks for indices in the request that begin with a dot, emitting a deprecation
+ * warning if they do. If the request is performed by a non-external user (operator, internal product, etc.)
+ * as defined by {@link #isInternalRequest()} then the deprecation is emitted. Otherwise, it is skipped.
+ *
+ * The indices for consideration are returned by the abstract {@link #getIndicesFromRequest(Object)}
+ * method, which subclasses must implement.
+ *
+ * Some built-in index names and patterns are also elided from the check, as defined in
+ * {@link #IGNORED_INDEX_NAMES} and {@link #IGNORED_INDEX_PATTERNS}.
+ */
+public abstract class DotPrefixValidator<RequestType> implements MappedActionFilter {
+    public static final Setting<Boolean> VALIDATE_DOT_PREFIXES = Setting.boolSetting(
+        "cluster.indices.validate_dot_prefixes",
+        true,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * Names and patterns for indexes where no deprecation should be emitted.
+     * Normally we would want to transition these to either system indices, or
+     * to use an internal origin for the client. These are shorter-term
+     * workarounds until that work can be completed.
+     *
+     * .elastic-connectors-* is used by enterprise search
+     * .ml-* is used by ML
+     */
+    private static Set<String> IGNORED_INDEX_NAMES = Set.of(
+        ".elastic-connectors-v1",
+        ".elastic-connectors-sync-jobs-v1",
+        ".ml-state",
+        ".ml-anomalies-unrelated"
+    );
+    private static Set<Pattern> IGNORED_INDEX_PATTERNS = Set.of(Pattern.compile("\\.ml-state-\\d+"));
+
+    DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DotPrefixValidator.class);
+
+    private final ThreadContext threadContext;
+    private final boolean isEnabled;
+
+    public DotPrefixValidator(ThreadContext threadContext, ClusterService clusterService) {
+        this.threadContext = threadContext;
+        this.isEnabled = VALIDATE_DOT_PREFIXES.get(clusterService.getSettings());
+    }
+
+    protected abstract Set<String> getIndicesFromRequest(RequestType request);
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse> void apply(
+        Task task,
+        String action,
+        Request request,
+        ActionListener<Response> listener,
+        ActionFilterChain<Request, Response> chain
+    ) {
+        Set<String> indices = getIndicesFromRequest((RequestType) request);
+        if (isEnabled) {
+            validateIndices(indices);
+        }
+        chain.proceed(task, action, request, listener);
+    }
+
+    void validateIndices(@Nullable Set<String> indices) {
+        if (indices != null && isInternalRequest() == false) {
+            for (String index : indices) {
+                if (Strings.hasLength(index)) {
+                    char c = getFirstChar(index);
+                    if (c == '.') {
+                        if (IGNORED_INDEX_NAMES.contains(index)) {
+                            return;
+                        }
+                        if (IGNORED_INDEX_PATTERNS.stream().anyMatch(p -> p.matcher(index).matches())) {
+                            return;
+                        }
+                        deprecationLogger.warn(
+                            DeprecationCategory.INDICES,
+                            "dot-prefix",
+                            "Index [{}] name begins with a dot (.), which is deprecated, "
+                                + "and will not be allowed in a future Elasticsearch version.",
+                            index
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    private static char getFirstChar(String index) {
+        char c = index.charAt(0);
+        if (c == '<') {
+            // Date-math is being used for the index, we need to
+            // consider it by stripping the first '<' before we
+            // check for a dot-prefix
+            String strippedLeading = index.substring(1);
+            if (Strings.hasLength(strippedLeading)) {
+                c = strippedLeading.charAt(0);
+            }
+        }
+        return c;
+    }
+
+    private boolean isInternalRequest() {
+        final String actionOrigin = threadContext.getTransient(ThreadContext.ACTION_ORIGIN_TRANSIENT_NAME);
+        final boolean isSystemContext = threadContext.isSystemContext();
+        final boolean isInternalOrigin = Optional.ofNullable(actionOrigin).map(Strings::hasText).orElse(false);
+        final boolean hasElasticOriginHeader = Optional.ofNullable(threadContext.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER))
+            .map(Strings::hasText)
+            .orElse(false);
+        return isSystemContext || isInternalOrigin || hasElasticOriginHeader;
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+public class IndexTemplateDotValidator extends DotPrefixValidator<TransportPutComposableIndexTemplateAction.Request> {
+    public IndexTemplateDotValidator(ThreadContext threadContext, ClusterService clusterService) {
+        super(threadContext, clusterService);
+    }
+
+    @Override
+    protected Set<String> getIndicesFromRequest(TransportPutComposableIndexTemplateAction.Request request) {
+        return new HashSet<>(Arrays.asList(request.indices()));
+    }
+
+    @Override
+    public String actionName() {
+        return TransportPutComposableIndexTemplateAction.TYPE.name();
+    }
+}