fixing tests

apache · May 11, 2022 · 750c4ea · 750c4ea
1 parent 56fc702
commit 750c4ea
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 6 deletions.
diff --git a/.../hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieBaseParquetWriter.java b/.../hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieBaseParquetWriter.java
@@ -38,7 +38,7 @@ public abstract class HoodieBaseParquetWriter<R> extends ParquetWriter<R> {
 
   private static final int WRITTEN_RECORDS_THRESHOLD_FOR_FILE_SIZE_CHECK = 1000;
 
-  private final AtomicLong writtenRecordCount = new AtomicLong(1);
+  private final AtomicLong writtenRecordCount = new AtomicLong(0);
   private final long maxFileSize;
   private long lastCachedDataSize = -1;
 
@@ -69,7 +69,7 @@ public boolean canWrite() {
     //      if we cache last data size check, since we account for how many records
     //      were written we can accurately project avg record size, and therefore
     //      estimate how many more records we can write before cut off
-    if (lastCachedDataSize == -1 || writtenRecordCount.get() % WRITTEN_RECORDS_THRESHOLD_FOR_FILE_SIZE_CHECK == 0) {
+    if (lastCachedDataSize == -1 || getWrittenRecordCount() % WRITTEN_RECORDS_THRESHOLD_FOR_FILE_SIZE_CHECK == 0) {
       lastCachedDataSize = getDataSize();
     }
     return lastCachedDataSize < maxFileSize;

diff --git a/.../src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java b/.../src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
@@ -379,7 +379,7 @@ public void testFileSizeUpsertRecords() throws Exception {
 
     List<HoodieRecord> records = new ArrayList<>();
     // Approx 1150 records are written for block size of 64KB
-    for (int i = 0; i < 2000; i++) {
+    for (int i = 0; i < 2050; i++) {
       String recordStr = "{\"_row_key\":\"" + UUID.randomUUID().toString()
           + "\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":" + i + "}";
       RawTripTestPayload rowChange = new RawTripTestPayload(recordStr);
@@ -402,7 +402,8 @@ public void testFileSizeUpsertRecords() throws Exception {
         counts++;
       }
     }
-    assertEquals(5, counts, "If the number of records are more than 1150, then there should be a new file");
+    // we check canWrite only once every 1000 records. and so 2 files with 1000 records and 3rd file with 50 records.
+    assertEquals(3, counts, "If the number of records are more than 1150, then there should be a new file");
   }
 
   @Test

diff --git a/...ient/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java b/...ient/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
@@ -419,7 +419,7 @@ public void testFileSizeUpsertRecords() throws Exception {
 
     List<HoodieRecord> records = new ArrayList<>();
     // Approx 1150 records are written for block size of 64KB
-    for (int i = 0; i < 2000; i++) {
+    for (int i = 0; i < 2050; i++) {
       String recordStr = "{\"_row_key\":\"" + UUID.randomUUID().toString()
           + "\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":" + i + "}";
       RawTripTestPayload rowChange = new RawTripTestPayload(recordStr);
@@ -441,7 +441,8 @@ public void testFileSizeUpsertRecords() throws Exception {
         counts++;
       }
     }
-    assertEquals(5, counts, "If the number of records are more than 1150, then there should be a new file");
+    // we check canWrite only once every 1000 records. and so 2 files with 1000 records and 3rd file with 50 records.
+    assertEquals(3, counts, "If the number of records are more than 1150, then there should be a new file");
   }
 
   @Test