fix bug (#14247)

apache · Dec 2, 2024 · 64b43e7 · 64b43e7
1 parent f776859
commit 64b43e7
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 4 deletions.
diff --git a/...-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/...-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -3053,9 +3053,6 @@ private boolean loadTsFileToUnSequence(
     PipeInsertionDataNodeListener.getInstance()
         .listenToTsFile(dataRegionId, tsFileResource, true, isGeneratedByPipe);
 
-    // help tsfile resource degrade
-    tsFileResourceManager.registerSealedTsFileResource(tsFileResource);
-
     tsFileManager.add(tsFileResource, false);
 
     return true;

diff --git a/...node/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/...node/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
@@ -171,7 +171,6 @@ public void removeAll(List<TsFileResource> tsFileResourceList, boolean sequence)
     try {
       for (TsFileResource resource : tsFileResourceList) {
         remove(resource, sequence);
-        TsFileResourceManager.getInstance().removeTsFileResource(resource);
       }
     } finally {
       writeLock("removeAll");
@@ -184,6 +183,7 @@ public void removeAll(List<TsFileResource> tsFileResourceList, boolean sequence)
    */
   public void insertToPartitionFileList(
       TsFileResource tsFileResource, long timePartition, boolean sequence, int insertPos) {
+    registerTsFileResourceToResourceManager(tsFileResource);
     writeLock("add");
     try {
       Map<Long, TsFileResourceList> selectedMap = sequence ? sequenceFiles : unsequenceFiles;
@@ -196,6 +196,7 @@ public void insertToPartitionFileList(
   }
 
   public void add(TsFileResource tsFileResource, boolean sequence) {
+    registerTsFileResourceToResourceManager(tsFileResource);
     writeLock("add");
     try {
       Map<Long, TsFileResourceList> selectedMap = sequence ? sequenceFiles : unsequenceFiles;
@@ -208,6 +209,7 @@ public void add(TsFileResource tsFileResource, boolean sequence) {
   }
 
   public void keepOrderInsert(TsFileResource tsFileResource, boolean sequence) throws IOException {
+    registerTsFileResourceToResourceManager(tsFileResource);
     writeLock("keepOrderInsert");
     try {
       Map<Long, TsFileResourceList> selectedMap = sequence ? sequenceFiles : unsequenceFiles;
@@ -230,6 +232,13 @@ public void addAll(List<TsFileResource> tsFileResourceList, boolean sequence) {
     }
   }
 
+  private void registerTsFileResourceToResourceManager(TsFileResource resource) {
+    TsFileResourceStatus status = resource.getStatus();
+    if (status != TsFileResourceStatus.UNCLOSED && status != TsFileResourceStatus.DELETED) {
+      TsFileResourceManager.getInstance().registerSealedTsFileResource(resource);
+    }
+  }
+
   /** This method is called after compaction to update memory. */
   public void replace(
       List<TsFileResource> seqFileResources,

diff --git a/...iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java b/...iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java
@@ -72,6 +72,7 @@ public void setUp()
     enableInsertionCrossSpaceCompaction =
         IoTDBDescriptor.getInstance().getConfig().isEnableCrossSpaceCompaction();
     IoTDBDescriptor.getInstance().getConfig().setEnableCrossSpaceCompaction(true);
+    TsFileResourceManager.getInstance().clear();
   }
 
   @After
@@ -80,6 +81,7 @@ public void tearDown() throws IOException, StorageEngineException {
         .getConfig()
         .setEnableCrossSpaceCompaction(enableInsertionCrossSpaceCompaction);
     super.tearDown();
+    TsFileResourceManager.getInstance().clear();
   }
 
   @Test
@@ -127,6 +129,9 @@ public void test1() throws IOException, InterruptedException {
     long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime();
     Assert.assertEquals(2, timestamp);
     Assert.assertTrue(tsFileManager.getTsFileList(false).isEmpty());
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -168,6 +173,9 @@ public void test2() throws IOException, InterruptedException {
     TsFileResource targetFile = tsFileManager.getTsFileList(true).get(1);
     long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime();
     Assert.assertEquals(2, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -209,6 +217,9 @@ public void test3() throws IOException, InterruptedException {
     TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0);
     long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime();
     Assert.assertEquals(0, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -256,6 +267,9 @@ public void test4() throws IOException, InterruptedException {
     long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime();
     Assert.assertEquals(4, timestamp);
     Assert.assertTrue(tsFileManager.getTsFileList(false).isEmpty());
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -291,6 +305,9 @@ public void test5() throws IOException, InterruptedException {
     TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0);
     long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime();
     Assert.assertEquals(2, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -340,6 +357,9 @@ public void testInsertionCompactionSchedule() throws IOException, InterruptedExc
     TsFileResource targetFile2 = tsFileManager.getTsFileList(true).get(2);
     timestamp = TsFileNameGenerator.getTsFileName(targetFile2.getTsFile().getName()).getTime();
     Assert.assertEquals(6, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -378,6 +398,9 @@ public void testInsertionCompactionScheduleWithEmptySeqSpace()
     TsFileResource targetFile3 = tsFileManager.getTsFileList(true).get(2);
     timestamp = TsFileNameGenerator.getTsFileName(targetFile3.getTsFile().getName()).getTime();
     Assert.assertEquals(4, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -423,6 +446,9 @@ public void testInsertionCompactionScheduleWithEmptySeqSpace2()
     TsFileResource targetFile2 = tsFileManager.getTsFileList(true).get(1);
     timestamp = TsFileNameGenerator.getTsFileName(targetFile2.getTsFile().getName()).getTime();
     Assert.assertEquals(2, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -473,6 +499,9 @@ public void testInsertionCompactionScheduleWithMultiTimePartitions()
     TsFileResource targetFile3 = tsFileManager.getOrCreateSequenceListByTimePartition(2808).get(0);
     timestamp = TsFileNameGenerator.getTsFileName(targetFile3.getTsFile().getName()).getTime();
     Assert.assertEquals(4, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -526,6 +555,9 @@ public void testInsertionCompactionUpdateFileMetrics() throws IOException {
     Assert.assertEquals(seqFileNumBeforeCompaction, FileMetrics.getInstance().getFileCount(true));
     Assert.assertEquals(
         unseqFileNumBeforeCompaction, FileMetrics.getInstance().getFileCount(false));
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   public TsFileResource generateSingleNonAlignedSeriesFileWithDevices(