From 59e972170a26b2e476478abc64d1e36f1a5c9fb2 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Sun, 7 Apr 2019 13:32:54 -0400 Subject: [PATCH] Ensure relocation occur in testRelocationWithConcurrentIndexing (#40801) If the relocation is throttled, the subsequent search request on the target node (i.e., with preference _only_nodes=target_node) will fail because some shards have not moved to that node yet. With this change, we will wait for the relocation happens by busily checking the routing table of the testing index on the target node. Closes #34950 --- .../java/org/elasticsearch/upgrades/RecoveryIT.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index f8246ca6dcd24..85102209e60f4 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -41,6 +41,7 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength; @@ -50,6 +51,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isIn; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -205,7 +207,6 @@ private String getNodeId(Predicate versionPredicate) throws IOException return null; } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/34950") public void testRelocationWithConcurrentIndexing() throws Exception { final String index = "relocation_with_concurrent_indexing"; switch (CLUSTER_TYPE) { @@ -239,6 +240,15 @@ public void testRelocationWithConcurrentIndexing() throws Exception { ensureNoInitializingShards(); // wait for all other shard activity to finish updateIndexSettings(index, Settings.builder().put("index.routing.allocation.include._id", newNode)); asyncIndexDocs(index, 10, 50).get(); + // ensure the relocation from old node to new node has occurred; otherwise ensureGreen can + // return true even though shards haven't moved to the new node yet (allocation was throttled). + assertBusy(() -> { + Map state = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state"))); + String xpath = "routing_table.indices." + index + ".shards.0.node"; + @SuppressWarnings("unchecked") List assignedNodes = (List) XContentMapValues.extractValue(xpath, state); + assertNotNull(state.toString(), assignedNodes); + assertThat(state.toString(), newNode, isIn(assignedNodes)); + }, 60, TimeUnit.SECONDS); ensureGreen(index); client().performRequest(new Request("POST", index + "/_refresh")); assertCount(index, "_only_nodes:" + newNode, 60);