diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index 561e4349a4890..0507caab1b017 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -133,7 +133,7 @@ public void removeFilesystemProvider() { } // Increasing watermark limit to avoid flaky test case failures. - private static final long WATERMARK_BYTES = new ByteSizeValue(1, ByteSizeUnit.MB).getBytes(); + private static final long WATERMARK_BYTES = new ByteSizeValue(10, ByteSizeUnit.KB).getBytes(); private static final String INDEX_ROUTING_ALLOCATION_NODE_SETTING = "index.routing.allocation.include._name"; @Override @@ -202,17 +202,14 @@ public void testIndexCreateBlockWhenAllNodesExceededHighWatermark() throws Excep } // Wait for all nodes to breach high disk watermark. + ensureAllNodesAboveHighWatermark(clusterInfoService); + // Validate if index create block is applied on the cluster assertBusy(() -> { - refreshDiskUsage(); - assertTrue( - StreamSupport.stream(clusterInfoService.getClusterInfo().getNodeLeastAvailableDiskUsages().values().spliterator(), false) - .allMatch(cur -> cur.value.getFreeBytes() < WATERMARK_BYTES) - ); + clusterInfoService.refresh(); + ClusterState state = client().admin().cluster().prepareState().setLocal(true).get().getState(); + assertTrue(state.blocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id())); }, 30L, TimeUnit.SECONDS); - // Validate if cluster block is applied on the cluster - ClusterState state = client().admin().cluster().prepareState().setLocal(true).get().getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id())); } public void testIndexCreateBlockNotAppliedWhenAnyNodesBelowHighWatermark() throws Exception { @@ -245,25 +242,21 @@ public void testIndexCreateBlockIsRemovedWhenAnyNodesNotExceedHighWatermark() th indexNames.add(indexName); } - // Wait for all the node to breach high disk watermark. + // Wait for all nodes to breach high disk watermark. + ensureAllNodesAboveHighWatermark(clusterInfoService); + // Validate if index create block is applied on the cluster assertBusy(() -> { - refreshDiskUsage(); - assertTrue( - StreamSupport.stream(clusterInfoService.getClusterInfo().getNodeLeastAvailableDiskUsages().values().spliterator(), false) - .allMatch(cur -> cur.value.getFreeBytes() < WATERMARK_BYTES) - ); + clusterInfoService.refresh(); + ClusterState state = client().admin().cluster().prepareState().setLocal(true).get().getState(); + assertTrue(state.blocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id())); }, 30L, TimeUnit.SECONDS); - // Validate if index create block is applied on the cluster - ClusterState state = client().admin().cluster().prepareState().setLocal(true).get().getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id())); - - // Delete indices to free space + // Delete indices to free space on nodes deleteIndices(indexNames); // Validate if index create block is removed on the cluster assertBusy(() -> { - refreshDiskUsage(); + clusterInfoService.refresh(); ClusterState state1 = client().admin().cluster().prepareState().setLocal(true).get().getState(); assertFalse(state1.blocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id())); }, 30L, TimeUnit.SECONDS); @@ -292,17 +285,14 @@ public void testIndexCreateBlockWithAReadOnlyBlock() throws Exception { .build(); client().admin().indices().prepareUpdateSettings(indexName).setSettings(readOnlySettings).get(); + // Wait for all nodes to breach high disk watermark. + ensureAllNodesAboveHighWatermark(clusterInfoService); + // Validate if index create block is applied on the cluster assertBusy(() -> { - refreshDiskUsage(); - assertTrue( - StreamSupport.stream(clusterInfoService.getClusterInfo().getNodeLeastAvailableDiskUsages().values().spliterator(), false) - .allMatch(cur -> cur.value.getFreeBytes() < WATERMARK_BYTES) - ); + clusterInfoService.refresh(); + ClusterState state = client().admin().cluster().prepareState().setLocal(true).get().getState(); + assertTrue(state.blocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id())); }, 30L, TimeUnit.SECONDS); - - // Validate index create block is applied on the cluster. - ClusterState state = client().admin().cluster().prepareState().setLocal(true).get().getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id())); } public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Exception { @@ -389,12 +379,20 @@ private void deleteIndices(final List indexNames) throws ExecutionExcept } } + private void ensureAllNodesAboveHighWatermark(final InternalClusterInfoService clusterInfoService) { + while (true) { + if (StreamSupport.stream(clusterInfoService.getClusterInfo().getNodeLeastAvailableDiskUsages().values().spliterator(), false) + .allMatch(cur -> cur.value.getFreeBytes() < WATERMARK_BYTES)) { + return; + } + } + } + private String populateNode(final String dataNodeName) throws Exception { final Path dataNodePath = internalCluster().getInstance(Environment.class, dataNodeName).dataFiles()[0]; final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); long minShardSize = createAndPopulateIndex(indexName, dataNodeName); fileSystemProvider.getTestFileStore(dataNodePath).setTotalSpace(minShardSize + WATERMARK_BYTES - 1L); - refreshDiskUsage(); return indexName; }