[ML] Check for internal index searchability as well as active primary

Currently, before performing operations that require the ML internal indices be available we check whether their primary shards are active. In stateless Elasticsearch we need to separately check whether the indices are searchable, as search and indexing shards are separate.
droberts195 · Oct 13, 2023 · 66c1a76 · 66c1a76
1 parent b8a204f
commit 66c1a76
Show file tree

Hide file tree

Showing 9 changed files with 24 additions and 5 deletions.
diff --git a/...plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheck.java b/...plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheck.java
@@ -65,6 +65,6 @@ static boolean mlConfigIndexIsAllocated(ClusterState clusterState) {
         }
 
         IndexRoutingTable routingTable = clusterState.getRoutingTable().index(configIndexOrAlias.getWriteIndex());
-        return routingTable != null && routingTable.allPrimaryShardsActive();
+        return routingTable != null && routingTable.allPrimaryShardsActive() && routingTable.readyForSearch(clusterState);
     }
 }
diff --git a/...lugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdater.java b/...lugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdater.java
@@ -58,7 +58,9 @@ public boolean isAbleToRun(ClusterState latestState) {
                 continue;
             }
             IndexRoutingTable routingTable = latestState.getRoutingTable().index(index);
-            if (routingTable == null || routingTable.allPrimaryShardsActive() == false) {
+            if (routingTable == null
+                || routingTable.allPrimaryShardsActive() == false
+                || routingTable.readyForSearch(latestState) == false) {
                 return false;
             }
         }

diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
@@ -206,7 +206,9 @@ private AssignmentFailure verifyIndicesActive() {
 
         for (String concreteIndex : concreteIndices) {
             IndexRoutingTable routingTable = clusterState.getRoutingTable().index(concreteIndex);
-            if (routingTable == null || routingTable.allPrimaryShardsActive() == false) {
+            if (routingTable == null
+                || routingTable.allPrimaryShardsActive() == false
+                || routingTable.readyForSearch(clusterState) == false) {
                 return new AssignmentFailure(
                     "cannot start datafeed ["
                         + datafeedId

diff --git a/...lugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java b/...lugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java
@@ -241,7 +241,9 @@ static boolean verifyIndicesExistAndPrimaryShardsAreActive(ClusterState clusterS
                 return false;
             }
             IndexRoutingTable routingTable = clusterState.getRoutingTable().index(index);
-            if (routingTable == null || routingTable.allPrimaryShardsActive() == false) {
+            if (routingTable == null
+                || routingTable.allPrimaryShardsActive() == false
+                || routingTable.readyForSearch(clusterState) == false) {
                 return false;
             }
         }

diff --git a/.../ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java b/.../ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java
@@ -66,7 +66,9 @@ public static List<String> verifyIndicesPrimaryShardsAreActive(
                 continue;
             }
             IndexRoutingTable routingTable = clusterState.getRoutingTable().index(index);
-            if (routingTable == null || routingTable.allPrimaryShardsActive() == false) {
+            if (routingTable == null
+                || routingTable.allPrimaryShardsActive() == false
+                || routingTable.readyForSearch(clusterState) == false) {
                 unavailableIndices.add(index);
             }
         }

diff --git a/.../ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdaterTests.java b/.../ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdaterTests.java
@@ -169,6 +169,7 @@ public void testIsAbleToRun() {
                 .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                 .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid")
         );
         metadata.put(indexMetadata);
         Index index = new Index(MlConfigIndex.indexName(), "_uuid");

diff --git a/.../ml/src/test/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsServiceTests.java b/.../ml/src/test/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsServiceTests.java
@@ -71,6 +71,7 @@ public void testVerifyIndicesExistAndPrimaryShardsAreActive() {
                         .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid")
                 );
             metadata.put(indexMetadata);
             addToRoutingTable(concreteIndex, routingTable);
@@ -91,6 +92,7 @@ public void testVerifyIndicesExistAndPrimaryShardsAreActive() {
                         .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid")
                 );
             metadata.put(indexMetadata);
             addToRoutingTable(concreteIndex, routingTable);
@@ -111,6 +113,7 @@ public void testVerifyIndicesExistAndPrimaryShardsAreActive() {
                         .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid")
                 );
             metadata.put(indexMetadata);
             addToRoutingTable(concreteIndex, routingTable);
@@ -174,6 +177,7 @@ public void testUpdateStatsUpgradeMode() {
                         .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid")
                 );
             Metadata.Builder metadata = Metadata.builder().put(indexMetadata);
 
@@ -201,6 +205,7 @@ public void testUpdateStatsUpgradeMode() {
                         .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid")
                 );
 
             // now set the upgrade mode
@@ -232,6 +237,7 @@ public void testUpdateStatsUpgradeMode() {
                         .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid")
                 );
 
             Metadata.Builder metadata = Metadata.builder()
@@ -288,6 +294,7 @@ public void testUpdateStatsResetMode() {
                         .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid")
                 );
 
             // now set the upgrade mode
@@ -319,6 +326,7 @@ public void testUpdateStatsResetMode() {
                         .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid")
                 );
 
             Metadata.Builder metadata = Metadata.builder()

diff --git a/...rc/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java b/...rc/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java
@@ -257,6 +257,7 @@ private void addIndices(Metadata.Builder metadata, RoutingTable.Builder routingT
                     .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                     .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                     .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid")
             );
             if (indexName.equals(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX)) {
                 indexMetadata.putAlias(new AliasMetadata.Builder(AnomalyDetectorsIndex.jobStateIndexWriteAlias()));

diff --git a/...rc/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java b/...rc/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java
@@ -123,6 +123,7 @@ private void addIndices(Metadata.Builder metadata, RoutingTable.Builder routingT
                     .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                     .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                     .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid")
             );
             if (indexName.equals(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX)) {
                 indexMetadata.putAlias(new AliasMetadata.Builder(AnomalyDetectorsIndex.jobStateIndexWriteAlias()));