HDDS-10682. EC Reconstruction creates empty chunks at the end of bloc…

…ks with partial stripes (apache#6515)
jojochuang · Apr 12, 2024 · a5fccbc · a5fccbc
1 parent c63e2b9
commit a5fccbc
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 9 deletions.
diff --git a/...java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java b/...java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java
@@ -323,9 +323,13 @@ public void reconstructECBlockGroup(BlockLocationInfo blockLocationInfo,
             }
             // TODO: can be submitted in parallel
             for (int i = 0; i < bufs.length; i++) {
-              CompletableFuture<ContainerProtos.ContainerCommandResponseProto>
-                  future = targetBlockStreams[i].write(bufs[i]);
-              checkFailures(targetBlockStreams[i], future);
+              if (bufs[i].remaining() != 0) {
+                // If the buffer is empty, we don't need to write it as it will cause
+                // an empty chunk to be added to the end of the block.
+                CompletableFuture<ContainerProtos.ContainerCommandResponseProto>
+                    future = targetBlockStreams[i].write(bufs[i]);
+                checkFailures(targetBlockStreams[i], future);
+              }
               bufs[i].clear();
             }
             length -= readLen;

diff --git a/...ration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java b/...ration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
@@ -136,7 +136,9 @@ public class TestContainerCommandsEC {
   private static final int EC_CHUNK_SIZE = 1024 * 1024;
   private static final int STRIPE_DATA_SIZE = EC_DATA * EC_CHUNK_SIZE;
   private static final int NUM_DN = EC_DATA + EC_PARITY + 3;
-  private static byte[][] inputChunks = new byte[EC_DATA][EC_CHUNK_SIZE];
+  // Data slots are EC_DATA + 1 so we can generate enough data to have a full stripe
+  // plus one extra chunk.
+  private static byte[][] inputChunks = new byte[EC_DATA + 1][EC_CHUNK_SIZE];
 
   // Each key size will be in range [min, max), min inclusive, max exclusive
   private static final int[][] KEY_SIZE_RANGES =
@@ -621,13 +623,13 @@ void testECReconstructionCoordinatorWithPartialStripe(List<Integer> missingIndex
     testECReconstructionCoordinator(missingIndexes, 1);
   }
 
-  @Test
-  void testECReconstructParityWithPartialStripe()
-          throws Exception {
-    testECReconstructionCoordinator(ImmutableList.of(4, 5), 1);
+  @ParameterizedTest
+  @MethodSource("recoverableMissingIndexes")
+  void testECReconstructionCoordinatorWithFullAndPartialStripe(List<Integer> missingIndexes)
+      throws Exception {
+    testECReconstructionCoordinator(missingIndexes, 4);
   }
 
-
   static Stream<List<Integer>> recoverableMissingIndexes() {
     return Stream
         .concat(IntStream.rangeClosed(1, 5).mapToObj(ImmutableList::of), Stream