Support complex datemath expressions in index and index alias names (e…

…lastic#100646)

* Support complex datemath expressions in index and index alias names

* Update docs/changelog/100646.yaml

* Follow PR comments

* fix

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

docs/changelog/100646.yaml

@@ -0,0 +1,5 @@
+pr: 100646
+summary: Support complex datemath expressions in index and index alias names
+area: Search
+type: bug
+issues: []

modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java

@@ -159,4 +159,24 @@ public void testReindexFromRemoteGivenSimpleDateMathIndexName() throws Interrupt
         }));
     }
 
+    public void testReindexFromRemoteGivenComplexDateMathIndexName() throws InterruptedException {
+        assertAcked(client(REMOTE_CLUSTER).admin().indices().prepareCreate("datemath-2001-01-01-14"));
+        final int docsNumber = indexDocs(client(REMOTE_CLUSTER), "datemath-2001-01-01-14");
+
+        // Remote name contains `:` symbol twice
+        final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "<datemath-{2001-01-01-13||+1h/h{yyyy-MM-dd-HH|-07:00}}>";
+        new ReindexRequestBuilder(client(LOCAL_CLUSTER), ReindexAction.INSTANCE).source(sourceIndexInRemote)
+            .destination("desc-index-001")
+            .get();
+
+        assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> {
+            SearchResponse resp = client(LOCAL_CLUSTER).prepareSearch("desc-index-001")
+                .setQuery(new MatchAllQueryBuilder())
+                .setSize(1000)
+                .get();
+            final TotalHits totalHits = resp.getHits().getTotalHits();
+            return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber;
+        }));
+    }
+
 }

modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.index.reindex.ReindexRequest;
 import org.elasticsearch.index.reindex.RemoteInfo;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.transport.RemoteClusterAware;
 
 import java.util.Arrays;
 import java.util.List;
@@ -160,6 +161,13 @@ private static SearchRequest skipRemoteIndexNames(SearchRequest source) {
     }
 
     private static boolean isRemoteExpression(String expression) {
-        return expression.contains(":");
+        // An index expression that references a remote cluster uses ":" to separate the cluster-alias from the index portion of the
+        // expression, e.g., cluster0:index-name
+        // in the same time date-math `expression` can also contain ':' symbol inside its name
+        // to distinguish between those two, given `expression` is pre-evaluated using date-math resolver
+        // after evaluation date-math `expression` should not contain ':' symbol
+        // otherwise if `expression` is legit remote name, ':' symbol remains
+        return IndexNameExpressionResolver.resolveDateMathExpression(expression)
+            .contains(String.valueOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR));
     }
 }

modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java

@@ -156,4 +156,22 @@ public void testMissingSources() {
         assertThat(response, matcher().created(0).slices(hasSize(0)));
     }
 
+    public void testReindexFromComplexDateMathIndexName() throws Exception {
+        String sourceIndexName = "datemath-2001-01-01-14";
+        String destIndexName = "<reindex-datemath-{2001-01-01-13||+1h/h{yyyy-MM-dd-HH|-07:00}}>";
+        indexRandom(
+            true,
+            client().prepareIndex(sourceIndexName).setId("1").setSource("foo", "a"),
+            client().prepareIndex(sourceIndexName).setId("2").setSource("foo", "a"),
+            client().prepareIndex(sourceIndexName).setId("3").setSource("foo", "b"),
+            client().prepareIndex(sourceIndexName).setId("4").setSource("foo", "c")
+        );
+        assertHitCount(client().prepareSearch(sourceIndexName).setSize(0).get(), 4);
+
+        // Copy all the docs
+        ReindexRequestBuilder copy = reindex().source(sourceIndexName).destination(destIndexName).refresh(true);
+        assertThat(copy.get(), matcher().created(4));
+        assertHitCount(client().prepareSearch(destIndexName).setSize(0).get(), 4);
+    }
+
 }

modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFailureTests.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.reindex;
 
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
@@ -19,6 +20,7 @@
 import java.util.concurrent.Future;
 
 import static org.elasticsearch.action.DocWriteRequest.OpType.CREATE;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
@@ -116,6 +118,25 @@ public void testResponseOnSearchFailure() throws Exception {
         assumeFalse("Wasn't able to trigger a reindex failure in " + attempt + " attempts.", true);
     }
 
+    public void testDateMathResolvesSameIndexName() throws Exception {
+        String sourceIndexName = "datemath-2001-01-01-14";
+        String destIndexName = "<datemath-{2001-01-01-13||+1h/h{yyyy-MM-dd-HH|-07:00}}>";
+        indexRandom(
+            true,
+            client().prepareIndex(sourceIndexName).setId("1").setSource("foo", "a"),
+            client().prepareIndex(sourceIndexName).setId("2").setSource("foo", "a"),
+            client().prepareIndex(sourceIndexName).setId("3").setSource("foo", "b"),
+            client().prepareIndex(sourceIndexName).setId("4").setSource("foo", "c")
+        );
+        assertHitCount(client().prepareSearch(sourceIndexName).setSize(0).get(), 4);
+
+        ActionRequestValidationException e = expectThrows(
+            ActionRequestValidationException.class,
+            () -> reindex().source(sourceIndexName).destination(destIndexName).get()
+        );
+        assertThat(e.getMessage(), containsString("reindex cannot write into an index its reading from [datemath-2001-01-01-14]"));
+    }
+
     private void indexDocs(int count) throws Exception {
         List<IndexRequestBuilder> docs = new ArrayList<>(count);
         for (int i = 0; i < count; i++) {

server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.cluster.metadata.ClusterNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -74,7 +75,10 @@ protected Map<String, List<String>> groupClusterIndices(Set<String> remoteCluste
         Map<String, List<String>> perClusterIndices = new HashMap<>();
         Set<String> clustersToRemove = new HashSet<>();
         for (String index : requestIndices) {
-            int i = index.indexOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR);
+            // ensure that `index` is a remote name and not a datemath expression which includes ':' symbol
+            // since datemath expression after evaluation should not contain ':' symbol
+            String probe = IndexNameExpressionResolver.resolveDateMathExpression(index);
+            int i = probe.indexOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR);
             if (i >= 0) {
                 if (isRemoteClusterClientEnabled == false) {
                     assert remoteClusterNames.isEmpty() : remoteClusterNames;