diff --git a/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java b/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java index 5b7147111a3a..694e55e197c8 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -72,6 +73,11 @@ public void init() throws IOException { initMetaClient(); } + @AfterEach + public void tearDown() throws Exception { + cleanMetaClient(); + } + @Test public void testBootstrapIndex() throws IOException { testBootstrapIndexOneRound(10); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java index d8bdaca0782d..78151a64c58a 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.Path; import org.junit.Rule; import org.junit.contrib.java.lang.system.EnvironmentVariables; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -81,6 +82,11 @@ public void setUp() throws IOException { basePath = "file:" + basePath; } + @AfterEach + public void tearDown() throws Exception { + cleanMetaClient(); + } + @Test public void testMakeDataFileName() { String instantTime = HoodieActiveTimeline.formatDate(new Date()); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java b/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java index d825c294cc6b..2413c79c64c1 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java @@ -148,6 +148,7 @@ public void testFailedToGetAppendStreamFromHDFSNameNode() writer.appendBlock(new HoodieCommandBlock(header)); // The log version should be different for this new writer assertNotEquals(writer.getLogFile().getLogVersion(), logFileVersion); + writer.close(); } } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java index 3ec15d4f65d1..84ac3351df81 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -48,6 +49,12 @@ public void setupTest() throws IOException { fs = metaClient.getFs(); } + @AfterEach + public void tearDown() throws Exception { + fs.close(); + cleanMetaClient(); + } + static Stream formatProviderFn() { return Stream.of( Arguments.arguments(Option.empty()), diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java index 0defefe2ea4e..7c9345b0e608 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -60,6 +61,11 @@ public void setUp() throws Exception { backupCfgPath = new Path(metaPath, HoodieTableConfig.HOODIE_PROPERTIES_FILE_BACKUP); } + @AfterEach + public void tearDown() throws Exception { + fs.close(); + } + @Test public void testCreate() throws IOException { assertTrue(fs.exists(new Path(metaPath, HoodieTableConfig.HOODIE_PROPERTIES_FILE))); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java index 840e6ddf4ad3..5c227c67dd48 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java @@ -25,6 +25,7 @@ import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -47,6 +48,11 @@ public void init() throws IOException { initMetaClient(); } + @AfterEach + public void tearDown() throws Exception { + cleanMetaClient(); + } + @Test public void checkMetadata() { assertEquals(HoodieTestUtils.RAW_TRIPS_TEST_NAME, metaClient.getTableConfig().getTableName(), diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java index da078372b5c3..094c607fb9be 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java @@ -37,6 +37,7 @@ import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -61,6 +62,11 @@ public void setUp() throws Exception { initMetaClient(); } + @AfterEach + public void tearDown() throws Exception { + cleanMetaClient(); + } + @Test public void testGetPartitionsWithReplaceCommits() throws IOException { HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java index beca794dd636..6806c6660a8f 100755 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java @@ -30,6 +30,7 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -71,6 +72,11 @@ public void setUp() throws Exception { initMetaClient(); } + @AfterEach + public void tearDown() throws Exception { + cleanMetaClient(); + } + @Test public void testLoadingInstantsFromFiles() throws IOException { HoodieInstant instant1 = new HoodieInstant(State.REQUESTED, HoodieTimeline.COMMIT_ACTION, "1"); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFSViewWithClustering.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFSViewWithClustering.java index 559375c795a8..58bb670922e1 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFSViewWithClustering.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFSViewWithClustering.java @@ -66,6 +66,12 @@ public void setup() throws IOException { refreshFsView(); } + @AfterEach + public void tearDown() throws Exception { + closeFsView(); + cleanMetaClient(); + } + protected void refreshFsView() throws IOException { super.refreshFsView(); closeFsView(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java index c8b6372630e5..180b601628e4 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java @@ -64,6 +64,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -119,6 +120,12 @@ public void setup() throws IOException { refreshFsView(); } + @AfterEach + public void tearDown() throws Exception { + closeFsView(); + cleanMetaClient(); + } + protected void refreshFsView() throws IOException { super.refreshFsView(); closeFsView(); @@ -1400,6 +1407,7 @@ public void testReplaceWithTimeTravel() throws IOException { .filter(dfile -> dfile.getFileId().equals(fileId3)).count()); assertEquals(1, filteredView.getLatestBaseFiles(partitionPath1) .filter(dfile -> dfile.getFileId().equals(fileId4)).count()); + filteredView.close(); // ensure replacedFileGroupsBefore works with all instants List replacedOnInstant1 = fsView.getReplacedFileGroupsBeforeOrOn("1", partitionPath1).collect(Collectors.toList()); @@ -1858,6 +1866,8 @@ public void testPendingMajorAndMinorCompactionOperations() throws Exception { // Verify file system view after 4th commit which is logcompaction.requested. verifyFileSystemView(partitionPath, expectedState, fileSystemView); + + fileSystemView.close(); } private HoodieCompactionPlan getHoodieCompactionPlan(List operations) { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java index 2367ba3035a3..07cd694b3307 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java @@ -55,6 +55,7 @@ import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -104,11 +105,17 @@ public void init() throws IOException { refreshFsView(); } + @AfterEach + public void tearDown() throws Exception { + cleanMetaClient(); + } + @Test public void testEmptyPartitionsAndTimeline() throws IOException { SyncableFileSystemView view = getFileSystemView(metaClient); assertFalse(view.getLastInstant().isPresent()); partitions.forEach(p -> assertEquals(0, view.getLatestFileSlices(p).count())); + view.close(); } @Test @@ -164,6 +171,7 @@ public void testAsyncCompaction() throws IOException { // Finish Compaction instantsToFiles.putAll(testMultipleWriteSteps(view, Collections.singletonList("19"), false, "19", 2, Collections.singletonList(new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "24")))); + view.close(); } @Test @@ -227,6 +235,8 @@ public void testAsyncMajorAndMinorCompaction() throws IOException { // Finish Compaction instantsToFiles.putAll(testMultipleWriteSteps(view, Collections.singletonList("25"), false, "25", 2, Collections.singletonList(new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "25")))); + + view.close(); } @Test @@ -267,6 +277,9 @@ public void testIngestion() throws IOException { // Clean instants. testCleans(view, Arrays.asList("21", "22"), instantsToFiles, Arrays.asList("18", "19"), 0, 0); + + newView.close(); + view.close(); } @Test @@ -314,6 +327,9 @@ public void testReplaceCommits() throws IOException { // Clean instants. testCleans(view, Arrays.asList("21", "22"), instantsToFiles, Arrays.asList("18", "19"), NUM_FILE_IDS_PER_PARTITION, 1); + + newView.close(); + view.close(); } private void testMultipleReplaceSteps(Map> instantsToFiles, SyncableFileSystemView view, List instants, @@ -329,6 +345,7 @@ private void testMultipleReplaceSteps(Map> instantsToFiles, // 1 fileId is replaced for every partition, so subtract partitions.size() expectedSlicesPerPartition = expectedSlicesPerPartition + fileIdsPerPartition.size() - 1; areViewsConsistent(view, newView, expectedSlicesPerPartition * partitions.size()); + newView.close(); } catch (IOException e) { throw new HoodieIOException("unable to test replace", e); } @@ -466,6 +483,13 @@ public void testMultipleTransitions() throws IOException { v.sync(); areViewsConsistent(v, view1, partitions.size() * fileIdsPerPartition.size() * 3); }); + + view1.close(); + view2.close(); + view3.close(); + view4.close(); + view5.close(); + view6.close(); } /* @@ -527,6 +551,7 @@ private void testCleans(SyncableFileSystemView view, List newCleanerInst metaClient.reloadActiveTimeline(); SyncableFileSystemView newView = getFileSystemView(metaClient); areViewsConsistent(view, newView, expTotalFileSlicesPerPartition * partitions.size()); + newView.close(); } catch (IOException e) { throw new HoodieException(e); } @@ -582,6 +607,7 @@ private void testRestore(SyncableFileSystemView view, List newRestoreIns metaClient.reloadActiveTimeline(); SyncableFileSystemView newView = getFileSystemView(metaClient); areViewsConsistent(view, newView, expTotalFileSlicesPerPartition * partitions.size()); + newView.close(); } catch (IOException e) { throw new HoodieException(e); } @@ -703,6 +729,7 @@ private void scheduleCompaction(SyncableFileSystemView view, String instantTime metaClient.reloadActiveTimeline(); SyncableFileSystemView newView = getFileSystemView(metaClient); areViewsConsistent(view, newView, initialExpTotalFileSlices + partitions.size() * fileIdsPerPartition.size()); + newView.close(); } /** @@ -732,6 +759,7 @@ private void scheduleLogCompaction(SyncableFileSystemView view, String instantTi metaClient.reloadActiveTimeline(); SyncableFileSystemView newView = getFileSystemView(metaClient); areViewsConsistent(view, newView, initialExpTotalFileSlices); + newView.close(); } /** @@ -862,6 +890,7 @@ private Map> testMultipleWriteSteps(SyncableFileSystemView metaClient.reloadActiveTimeline(); SyncableFileSystemView newView = getFileSystemView(metaClient); areViewsConsistent(view, newView, fileIdsPerPartition.size() * partitions.size() * multiple); + newView.close(); instantToFiles.put(instant, filePaths); if (!deltaCommit) { multiple++; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java index 29fdbefced0b..c428c248b878 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.core.Logger; import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -80,6 +81,13 @@ public void setUp() { "file0001" + HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension())); } + @AfterEach + public void tearDown() throws Exception { + testFileSliceStream.close(); + testBaseFileStream.close(); + fsView.close(); + } + private void resetMocks() { reset(primary, secondary); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java index d9b3d4ecada1..df64b3c8222b 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java @@ -91,6 +91,12 @@ protected void initMetaClient() throws IOException { basePath = metaClient.getBasePath(); } + protected void cleanMetaClient() { + if (metaClient != null) { + metaClient = null; + } + } + protected void refreshFsView() throws IOException { metaClient = HoodieTableMetaClient.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build(); } diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java index 9db05b31e3e8..c9bf80633a6e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java @@ -26,6 +26,7 @@ import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.internal.schema.Types; import org.apache.hudi.internal.schema.utils.SerDeHelper; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -47,6 +48,11 @@ public void setUp() throws Exception { initMetaClient(); } + @AfterEach + public void tearDown() throws Exception { + cleanMetaClient(); + } + @Test public void testPersistAndReadHistorySchemaStr() throws IOException { timeline = new HoodieActiveTimeline(metaClient); diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java b/hudi-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java index 9ec793daa012..6656ece519f2 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java +++ b/hudi-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java @@ -54,6 +54,7 @@ public void setUp() throws IOException { @AfterEach public void tearDown() throws IOException { metaClient.getFs().delete(new Path(metaClient.getBasePath()), true); + cleanMetaClient(); } /**