Fixing flaky test testThreeZoneOneReplicaWithForceZoneValueAndLoadAwa…

…reness (#3646)

* Fixing flaky test org.opensearch.cluster.allocation.AwarenessAllocationIT.testThreeZoneOneReplicaWithForceZoneValueAndLoadAwareness by adding dedicated cluster manager node

Signed-off-by: Rishab Nahata <rnnahata@amazon.com>

server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java

@@ -364,18 +364,22 @@ public void testThreeZoneOneReplicaWithForceZoneValueAndLoadAwareness() throws E
             .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c")
             .put("cluster.routing.allocation.load_awareness.skew_factor", "0.0")
             .put("cluster.routing.allocation.load_awareness.provisioned_capacity", Integer.toString(nodeCountPerAZ * 3))
+            .put("cluster.routing.allocation.allow_rebalance", "indices_primaries_active")
             .build();
 
+        logger.info("--> starting a dedicated cluster manager node");
+        internalCluster().startClusterManagerOnlyNode();
+
         logger.info("--> starting 15 nodes on zones 'a' & 'b' & 'c'");
-        List<String> nodes_in_zone_a = internalCluster().startNodes(
+        List<String> nodes_in_zone_a = internalCluster().startDataOnlyNodes(
             nodeCountPerAZ,
             Settings.builder().put(commonSettings).put("node.attr.zone", "a").build()
         );
-        List<String> nodes_in_zone_b = internalCluster().startNodes(
+        List<String> nodes_in_zone_b = internalCluster().startDataOnlyNodes(
             nodeCountPerAZ,
             Settings.builder().put(commonSettings).put("node.attr.zone", "b").build()
         );
-        List<String> nodes_in_zone_c = internalCluster().startNodes(
+        List<String> nodes_in_zone_c = internalCluster().startDataOnlyNodes(
             nodeCountPerAZ,
             Settings.builder().put(commonSettings).put("node.attr.zone", "c").build()
         );
@@ -395,7 +399,7 @@ public void testThreeZoneOneReplicaWithForceZoneValueAndLoadAwareness() throws E
             .setIndices("test-1")
             .setWaitForEvents(Priority.LANGUID)
             .setWaitForGreenStatus()
-            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3))
+            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3 + 1))
             .setWaitForNoRelocatingShards(true)
             .setWaitForNoInitializingShards(true)
             .execute()
@@ -431,7 +435,7 @@ public void testThreeZoneOneReplicaWithForceZoneValueAndLoadAwareness() throws E
             .prepareHealth()
             .setIndices("test-1")
             .setWaitForEvents(Priority.LANGUID)
-            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3 - nodesToStop))
+            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3 - nodesToStop + 1))
             .setWaitForNoRelocatingShards(true)
             .setWaitForNoInitializingShards(true)
             .execute()
@@ -452,7 +456,7 @@ public void testThreeZoneOneReplicaWithForceZoneValueAndLoadAwareness() throws E
             .prepareHealth()
             .setIndices("test-1", "test-2")
             .setWaitForEvents(Priority.LANGUID)
-            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3 - nodesToStop))
+            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3 - nodesToStop + 1))
             .setWaitForNoRelocatingShards(true)
             .setWaitForNoInitializingShards(true)
             .execute()
@@ -477,7 +481,7 @@ public void testThreeZoneOneReplicaWithForceZoneValueAndLoadAwareness() throws E
             .prepareHealth()
             .setIndices("test-1", "test-2")
             .setWaitForEvents(Priority.LANGUID)
-            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3))
+            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3 + 1))
             .setWaitForGreenStatus()
             .setWaitForActiveShards(2 * numOfShards * (numOfReplica + 1))
             .setWaitForNoRelocatingShards(true)