HDDS-9962. Intermittent timeout in TestBlockDeletion.testBlockDeletion (

apache#5886)
xichen01 · Dec 29, 2023 · 2aee970 · 2aee970
1 parent 2bc9f37
commit 2aee970
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 9 deletions.
diff --git a/...ds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java b/...ds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java
@@ -237,10 +237,12 @@ public int getBlockDeleteTXNum() {
   public void notifyStatusChanged() {
     serviceLock.lock();
     try {
-      if (scmContext.isLeaderReady() && !scmContext.isInSafeMode() &&
-          serviceStatus != ServiceStatus.RUNNING) {
-        safemodeExitMillis = clock.millis();
-        serviceStatus = ServiceStatus.RUNNING;
+      if (scmContext.isLeaderReady() && !scmContext.isInSafeMode()) {
+        if (serviceStatus != ServiceStatus.RUNNING) {
+          LOG.info("notifyStatusChanged" + ":" + ServiceStatus.RUNNING);
+          safemodeExitMillis = clock.millis();
+          serviceStatus = ServiceStatus.RUNNING;
+        }
       } else {
         serviceStatus = ServiceStatus.PAUSING;
       }

diff --git a/...g/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/...g/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -168,6 +168,7 @@ public void init() throws Exception {
         0,
         TimeUnit.MILLISECONDS);
     conf.setInt("hdds.datanode.block.delete.threads.max", 5);
+    conf.setInt("hdds.datanode.block.delete.queue.limit", 32);
     ReplicationManager.ReplicationManagerConfiguration replicationConf = conf
         .getObject(ReplicationManager.ReplicationManagerConfiguration.class);
     replicationConf.setInterval(Duration.ofSeconds(300));
@@ -293,11 +294,6 @@ public void testBlockDeletion(ReplicationConfig repConfig) throws Exception {
     Assertions.assertFalse(containerIdsWithDeletedBlocks.isEmpty());
     // Containers in the DN and SCM should have same delete transactionIds
     matchContainerTransactionIds();
-    // Containers in the DN and SCM should have same delete transactionIds
-    // after DN restart. The assertion is just to verify that the state of
-    // containerInfos in dn and scm is consistent after dn restart.
-    cluster.restartHddsDatanode(0, true);
-    matchContainerTransactionIds();
 
     // Verify transactions committed
     GenericTestUtils.waitFor(() -> {
@@ -309,6 +305,13 @@ public void testBlockDeletion(ReplicationConfig repConfig) throws Exception {
         return false;
       }
     }, 500, 10000);
+
+    // Containers in the DN and SCM should have same delete transactionIds
+    // after DN restart. The assertion is just to verify that the state of
+    // containerInfos in dn and scm is consistent after dn restart.
+    cluster.restartHddsDatanode(0, true);
+    matchContainerTransactionIds();
+
     Assertions.assertEquals(metrics.getNumBlockDeletionTransactionCreated(),
         metrics.getNumBlockDeletionTransactionCompleted());
     Assertions.assertTrue(metrics.getNumBlockDeletionCommandSent() >=