diff --git a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java index e71d34d48e18..8bb9bac2491e 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java @@ -21,8 +21,9 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.MeasurementPath; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.exception.write.WriteProcessException; @@ -249,24 +250,22 @@ private void generateTEXT(final Object obj, final int row) { public void generateDeletion(final String device, final int number) throws IOException, IllegalPathException { - try (final ModificationFileV1 modificationFile = - new ModificationFileV1(tsFile.getAbsolutePath() + ModificationFileV1.FILE_SUFFIX)) { + try (final ModificationFile modificationFile = + new ModificationFile(tsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX)) { writer.flushAllChunkGroups(); final TreeSet timeSet = device2TimeSet.get(device); if (timeSet.isEmpty()) { return; } - final long fileOffset = tsFile.length(); final long maxTime = timeSet.last() - 1; for (int i = 0; i < number; i++) { final int endTime = random.nextInt((int) (maxTime)) + 1; final int startTime = random.nextInt(endTime); for (final IMeasurementSchema measurementSchema : device2MeasurementSchema.get(device)) { - final Deletion deletion = - new Deletion( + final ModEntry deletion = + new TreeDeletionEntry( new MeasurementPath(device, measurementSchema.getMeasurementId()), - fileOffset, startTime, endTime); modificationFile.write(deletion); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java index dd97c18a049a..76810eeddf0e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java @@ -25,9 +25,9 @@ import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.rpc.TSStatusCode; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -58,11 +58,11 @@ public class IoTDBDeletionIT { }; private String insertTemplate = - "INSERT INTO root.vehicle.d0(timestamp,s0,s1,s2,s3,s4" + ") VALUES(%d,%d,%d,%f,%s,%b)"; - private String deleteAllTemplate = "DELETE FROM root.vehicle.d0.* WHERE time <= 10000"; + "INSERT INTO root.vehicle%d.d0(timestamp,s0,s1,s2,s3,s4" + ") VALUES(%d,%d,%d,%f,%s,%b)"; + private String deleteAllTemplate = "DELETE FROM root.vehicle%d.d0.* WHERE time <= 10000"; - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUp() throws Exception { Locale.setDefault(Locale.ENGLISH); EnvFactory.getEnv() @@ -75,8 +75,8 @@ public void setUp() throws Exception { prepareSeries(); } - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } @@ -90,36 +90,36 @@ public void testUnsupportedValueFilter() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("insert into root.vehicle.d0(time,s0) values (10,310)"); - statement.execute("insert into root.vehicle.d0(time,s3) values (10,'text')"); - statement.execute("insert into root.vehicle.d0(time,s4) values (10,true)"); + statement.execute("insert into root.vehicle1.d0(time,s0) values (10,310)"); + statement.execute("insert into root.vehicle1.d0(time,s3) values (10,'text')"); + statement.execute("insert into root.vehicle1.d0(time,s4) values (10,true)"); String errorMsg = TSStatusCode.SEMANTIC_ERROR.getStatusCode() + ": For delete statement, where clause can only contain time expressions, value filter is not currently supported."; try { - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE s0 <= 300 AND s0 > 0"); + statement.execute("DELETE FROM root.vehicle1.d0.s0 WHERE s0 <= 300 AND s0 > 0"); fail("should not reach here!"); } catch (SQLException e) { assertEquals(errorMsg, e.getMessage()); } try { - statement.execute("DELETE FROM root.vehicle.d0.s3 WHERE s3 = 'text'"); + statement.execute("DELETE FROM root.vehicle1.d0.s3 WHERE s3 = 'text'"); fail("should not reach here!"); } catch (SQLException e) { assertEquals(errorMsg, e.getMessage()); } try { - statement.execute("DELETE FROM root.vehicle.d0.s4 WHERE s4 != true"); + statement.execute("DELETE FROM root.vehicle1.d0.s4 WHERE s4 != true"); fail("should not reach here!"); } catch (SQLException e) { assertEquals(errorMsg, e.getMessage()); } - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle1.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -127,7 +127,7 @@ public void testUnsupportedValueFilter() throws SQLException { assertEquals(1, cnt); } - try (ResultSet set = statement.executeQuery("SELECT s3 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s3 FROM root.vehicle1.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -135,7 +135,7 @@ public void testUnsupportedValueFilter() throws SQLException { assertEquals(1, cnt); } - try (ResultSet set = statement.executeQuery("SELECT s4 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s4 FROM root.vehicle1.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -147,17 +147,17 @@ public void testUnsupportedValueFilter() throws SQLException { @Test public void test() throws SQLException { - prepareData(); + prepareData(2); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time <= 300"); + statement.execute("DELETE FROM root.vehicle2.d0.s0 WHERE time <= 300"); statement.execute( - "DELETE FROM root.vehicle.d0.s1,root.vehicle.d0.s2,root.vehicle.d0.s3" + "DELETE FROM root.vehicle2.d0.s1,root.vehicle2.d0.s2,root.vehicle2.d0.s3" + " WHERE time <= 350"); - statement.execute("DELETE FROM root.vehicle.d0.** WHERE time <= 150"); + statement.execute("DELETE FROM root.vehicle2.d0.** WHERE time <= 150"); - try (ResultSet set = statement.executeQuery("SELECT * FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT * FROM root.vehicle2.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -165,7 +165,7 @@ public void test() throws SQLException { assertEquals(250, cnt); } - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle2.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -173,7 +173,7 @@ public void test() throws SQLException { assertEquals(100, cnt); } - try (ResultSet set = statement.executeQuery("SELECT s1,s2,s3 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s1,s2,s3 FROM root.vehicle2.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -181,25 +181,25 @@ public void test() throws SQLException { assertEquals(50, cnt); } } - cleanData(); + cleanData(2); } @Test public void testDelAfterFlush() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("CREATE DATABASE root.ln.wf01.wt01"); + statement.execute("CREATE DATABASE root.ln3.wf01.wt01"); statement.execute( - "CREATE TIMESERIES root.ln.wf01.wt01.status WITH DATATYPE=BOOLEAN," + " ENCODING=PLAIN"); + "CREATE TIMESERIES root.ln3.wf01.wt01.status WITH DATATYPE=BOOLEAN," + " ENCODING=PLAIN"); statement.execute( - "INSERT INTO root.ln.wf01.wt01(timestamp,status) " + "values(1509465600000,true)"); - statement.execute("INSERT INTO root.ln.wf01.wt01(timestamp,status) VALUES(NOW(), false)"); + "INSERT INTO root.ln3.wf01.wt01(timestamp,status) " + "values(1509465600000,true)"); + statement.execute("INSERT INTO root.ln3.wf01.wt01(timestamp,status) VALUES(NOW(), false)"); - statement.execute("delete from root.ln.wf01.wt01.status where time <= NOW()"); + statement.execute("delete from root.ln3.wf01.wt01.status where time <= NOW()"); statement.execute("flush"); - statement.execute("delete from root.ln.wf01.wt01.status where time <= NOW()"); + statement.execute("delete from root.ln3.wf01.wt01.status where time <= NOW()"); - try (ResultSet resultSet = statement.executeQuery("select status from root.ln.wf01.wt01")) { + try (ResultSet resultSet = statement.executeQuery("select status from root.ln3.wf01.wt01")) { assertFalse(resultSet.next()); } } @@ -207,13 +207,13 @@ public void testDelAfterFlush() throws SQLException { @Test public void testRangeDelete() throws SQLException { - prepareData(); + prepareData(4); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time <= 300"); - statement.execute("DELETE FROM root.vehicle.d0.s1 WHERE time > 150"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle4.d0.s0 WHERE time <= 300"); + statement.execute("DELETE FROM root.vehicle4.d0.s1 WHERE time > 150"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle4.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -221,7 +221,7 @@ public void testRangeDelete() throws SQLException { assertEquals(100, cnt); } - try (ResultSet set = statement.executeQuery("SELECT s1 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s1 FROM root.vehicle4.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -229,8 +229,8 @@ public void testRangeDelete() throws SQLException { assertEquals(150, cnt); } - statement.execute("DELETE FROM root.vehicle.d0.** WHERE time > 50 and time <= 250"); - try (ResultSet set = statement.executeQuery("SELECT * FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle4.d0.** WHERE time > 50 and time <= 250"); + try (ResultSet set = statement.executeQuery("SELECT * FROM root.vehicle4.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -238,22 +238,22 @@ public void testRangeDelete() throws SQLException { assertEquals(200, cnt); } } - cleanData(); + cleanData(4); } @Test public void testFullDeleteWithoutWhereClause() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("DELETE FROM root.vehicle.d0.s0"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle5.d0.s0"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle5.d0")) { int cnt = 0; while (set.next()) { cnt++; } assertEquals(0, cnt); } - cleanData(); + cleanData(5); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -262,12 +262,12 @@ public void testFullDeleteWithoutWhereClause() { @Test public void testPartialPathRangeDelete() throws SQLException { - prepareData(); + prepareData(6); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("DELETE FROM root.vehicle.d0.* WHERE time <= 300 and time > 150"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle6.d0.* WHERE time <= 300 and time > 150"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle6.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -275,8 +275,8 @@ public void testPartialPathRangeDelete() throws SQLException { assertEquals(250, cnt); } - statement.execute("DELETE FROM root.vehicle.*.s0 WHERE time <= 100"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle6.*.s0 WHERE time <= 100"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle6.d0")) { int cnt = 0; while (set.next()) { cnt++; @@ -284,63 +284,65 @@ public void testPartialPathRangeDelete() throws SQLException { assertEquals(150, cnt); } } - cleanData(); + cleanData(6); } @Test public void testDelFlushingMemtable() throws SQLException { + int testNum = 7; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { for (int i = 1; i <= 10000; i++) { statement.execute( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 1500 and time <= 9000"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + statement.execute("DELETE FROM root.vehicle7.d0.s0 WHERE time > 1500 and time <= 9000"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle7.d0")) { int cnt = 0; while (set.next()) { cnt++; } assertEquals(2500, cnt); } - cleanData(); + cleanData(testNum); } } @Test public void testDelMultipleFlushingMemtable() throws SQLException { + int testNum = 8; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { for (int i = 1; i <= 1000; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } statement.executeBatch(); statement.clearBatch(); - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 150 and time <= 300"); - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 300 and time <= 400"); + statement.execute("DELETE FROM root.vehicle8.d0.s0 WHERE time > 150 and time <= 300"); + statement.execute("DELETE FROM root.vehicle8.d0.s0 WHERE time > 300 and time <= 400"); for (int i = 1001; i <= 2000; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } statement.executeBatch(); statement.clearBatch(); - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 500 and time <= 800"); - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 900 and time <= 1100"); - statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 1500 and time <= 1650"); + statement.execute("DELETE FROM root.vehicle8.d0.s0 WHERE time > 500 and time <= 800"); + statement.execute("DELETE FROM root.vehicle8.d0.s0 WHERE time > 900 and time <= 1100"); + statement.execute("DELETE FROM root.vehicle8.d0.s0 WHERE time > 1500 and time <= 1650"); statement.execute("flush"); - try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) { + try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle8.d0")) { int cnt = 0; while (set.next()) { cnt++; } assertEquals(1100, cnt); } - cleanData(); + cleanData(testNum); } } @@ -348,10 +350,11 @@ public void testDelMultipleFlushingMemtable() throws SQLException { public void testDeleteAll() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("insert into root.lz.dev.GPS(time, latitude, longitude) values(9,3.2,9.8)"); - statement.execute("insert into root.lz.dev.GPS(time, latitude) values(11,4.5)"); + statement.execute( + "insert into root.lz9.dev.GPS(time, latitude, longitude) values(9,3.2,9.8)"); + statement.execute("insert into root.lz9.dev.GPS(time, latitude) values(11,4.5)"); - try (ResultSet resultSet = statement.executeQuery("select * from root.lz.dev.GPS")) { + try (ResultSet resultSet = statement.executeQuery("select * from root.lz9.dev.GPS")) { int cnt = 0; while (resultSet.next()) { cnt++; @@ -359,9 +362,9 @@ public void testDeleteAll() throws SQLException { Assert.assertEquals(2, cnt); } - statement.execute("delete from root.lz.**"); + statement.execute("delete from root.lz9.**"); - try (ResultSet resultSet = statement.executeQuery("select * from root.lz.dev.GPS")) { + try (ResultSet resultSet = statement.executeQuery("select * from root.lz9.dev.GPS")) { int cnt = 0; while (resultSet.next()) { cnt++; @@ -376,14 +379,14 @@ public void testDeleteDataFromEmptySeries() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute( - "create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN;"); + "create timeseries root.`0`.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN;"); statement.execute( - "INSERT INTO root.ln.wf01.wt01(Time,status) VALUES (2022-10-11 10:20:50,true),(2022-10-11 10:20:51,true);"); + "INSERT INTO root.ln10.wf01.wt01(Time,status) VALUES (2022-10-11 10:20:50,true),(2022-10-11 10:20:51,true);"); statement.execute( - "create timeseries root.sg.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN;"); + "create timeseries root.sg10.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN;"); statement.execute( - "DELETE FROM root.ln.wf01.wt01.status,root.sg.wf01.wt01.status WHERE time >2022-10-11 10:20:50;"); + "DELETE FROM root.ln10.wf01.wt01.status,root.sg.wf01.wt01.status WHERE time >2022-10-11 10:20:50;"); try (ResultSet resultSet = statement.executeQuery("select ** from root")) { int cnt = 0; @@ -400,11 +403,11 @@ public void testDelSeriesWithSpecialSymbol() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute( - "CREATE TIMESERIES root.ln.d1.`status,01` WITH DATATYPE=BOOLEAN, ENCODING=PLAIN"); - statement.execute("INSERT INTO root.ln.d1(timestamp,`status,01`) VALUES(300, true)"); - statement.execute("INSERT INTO root.ln.d1(timestamp,`status,01`) VALUES(500, false)"); + "CREATE TIMESERIES root.ln11.d1.`status,01` WITH DATATYPE=BOOLEAN, ENCODING=PLAIN"); + statement.execute("INSERT INTO root.ln11.d1(timestamp,`status,01`) VALUES(300, true)"); + statement.execute("INSERT INTO root.ln11.d1(timestamp,`status,01`) VALUES(500, false)"); - try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln.d1")) { + try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln11.d1")) { int cnt = 0; while (resultSet.next()) { cnt++; @@ -412,9 +415,9 @@ public void testDelSeriesWithSpecialSymbol() throws SQLException { Assert.assertEquals(2, cnt); } - statement.execute("DELETE FROM root.ln.d1.`status,01` WHERE time <= 400"); + statement.execute("DELETE FROM root.ln11.d1.`status,01` WHERE time <= 400"); - try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln.d1")) { + try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln11.d1")) { int cnt = 0; while (resultSet.next()) { cnt++; @@ -422,9 +425,9 @@ public void testDelSeriesWithSpecialSymbol() throws SQLException { Assert.assertEquals(1, cnt); } - statement.execute("DELETE FROM root.ln.d1.`status,01`"); + statement.execute("DELETE FROM root.ln11.d1.`status,01`"); - try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln.d1")) { + try (ResultSet resultSet = statement.executeQuery("select `status,01` from root.ln11.d1")) { int cnt = 0; while (resultSet.next()) { cnt++; @@ -447,40 +450,40 @@ private static void prepareSeries() { } } - private void prepareData() throws SQLException { + private void prepareData(int testNum) throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { // prepare BufferWrite file for (int i = 201; i <= 300; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } statement.addBatch("flush"); // prepare Unseq-File for (int i = 1; i <= 100; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } statement.addBatch("flush"); // prepare BufferWrite cache for (int i = 301; i <= 400; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } // prepare Overflow cache for (int i = 101; i <= 200; i++) { statement.addBatch( - String.format(insertTemplate, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); } statement.executeBatch(); } } - private void cleanData() throws SQLException { + private void cleanData(int testNum) throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute(deleteAllTemplate); + statement.execute(String.format(deleteAllTemplate, testNum)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java index 84a733bb71b7..bce1dc1d9acd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java @@ -24,7 +24,6 @@ import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.enums.TSDataType; @@ -321,9 +320,8 @@ public void unpinTsFileResource(final TsFileResource resource) throws IOExceptio final File pinnedFile = getHardlinkOrCopiedFileInPipeDir(resource.getTsFile()); decreaseFileReference(pinnedFile); - final File modFile = new File(pinnedFile + ModificationFileV1.FILE_SUFFIX); - if (modFile.exists()) { - decreaseFileReference(modFile); + if (resource.newModFileExists()) { + decreaseFileReference(resource.getNewModFile().getFile()); } } finally { lock.unlock(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index a94b831c1dca..68aab50907c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -225,6 +226,8 @@ public void clean() { Files.deleteIfExists(tsFile.toPath()); Files.deleteIfExists( new File(tsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).toPath()); + Files.deleteIfExists( + new File(tsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX).toPath()); Files.deleteIfExists( new File(tsFile.getAbsolutePath() + ModificationFileV1.FILE_SUFFIX).toPath()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 437b57b53812..84a29050b762 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -3222,7 +3222,7 @@ private boolean loadTsFileToUnSequence( e.getMessage())); } - loadModFile(tsFileToLoad, targetFile, deleteOriginFile); + loadModFile(tsFileToLoad, targetFile, deleteOriginFile, tsFileResource); // Listen before the tsFile is added into tsFile manager to avoid it being compacted PipeInsertionDataNodeListener.getInstance() @@ -3236,15 +3236,23 @@ private boolean loadTsFileToUnSequence( return true; } - private void loadModFile(File tsFileToLoad, File targetTsFile, boolean deleteOriginFile) + private void loadModFile( + File tsFileToLoad, File targetTsFile, boolean deleteOriginFile, TsFileResource tsFileResource) throws LoadFileException { final File oldModFileToLoad = ModificationFileV1.getNormalMods(tsFileToLoad); - final File oldTargetModFile = ModificationFileV1.getNormalMods(targetTsFile); - moveModFile(oldModFileToLoad, oldTargetModFile, deleteOriginFile); - final File newModFileToLoad = ModificationFile.getNormalMods(tsFileToLoad); - final File newTargetModFile = ModificationFile.getNormalMods(targetTsFile); - moveModFile(newModFileToLoad, newTargetModFile, deleteOriginFile); + if (oldModFileToLoad.exists()) { + final File oldTargetModFile = ModificationFileV1.getNormalMods(targetTsFile); + moveModFile(oldModFileToLoad, oldTargetModFile, deleteOriginFile); + try { + tsFileResource.upgradeModFile(upgradeModFileThreadPool); + } catch (IOException e) { + throw new LoadFileException(e); + } + } else if (newModFileToLoad.exists()) { + final File newTargetModFile = ModificationFile.getNormalMods(targetTsFile); + moveModFile(newModFileToLoad, newTargetModFile, deleteOriginFile); + } } @SuppressWarnings("java:S2139") diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java index 64c813d3dd70..ef0f56a25182 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java @@ -24,7 +24,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.TsFileIdentifier; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -286,7 +286,7 @@ private boolean deleteFile(TsFileIdentifier tsFileIdentifier) { } // delete mods file - file = getFileFromDataDirs(tsFileIdentifier.getFilePath() + ModificationFileV1.FILE_SUFFIX); + file = getFileFromDataDirs(tsFileIdentifier.getFilePath() + ModificationFile.FILE_SUFFIX); if (!checkAndDeleteFile(file)) { success = false; } @@ -294,7 +294,7 @@ private boolean deleteFile(TsFileIdentifier tsFileIdentifier) { // delete compaction mods file file = getFileFromDataDirs( - tsFileIdentifier.getFilePath() + ModificationFileV1.COMPACTION_FILE_SUFFIX); + tsFileIdentifier.getFilePath() + ModificationFile.COMPACTION_FILE_SUFFIX); if (!checkAndDeleteFile(file)) { success = false; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandler.java index 2108518cc6eb..bf59172c5d8e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandler.java @@ -32,7 +32,6 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduler; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceList; @@ -136,8 +135,6 @@ private TSStatus validate() { return RpcUtils.getStatus( TSStatusCode.PATH_NOT_EXIST, "The specified file does not exist in " + path); } - File oldModsFile = ModificationFileV1.getNormalMods(currentTsFile); - hasModsFiles |= oldModsFile.exists(); File newModsFile = ModificationFile.getNormalMods(currentTsFile); hasModsFiles |= newModsFile.exists(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java index c678993536ef..a831431dfb53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.modification; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -37,16 +38,22 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.NoSuchElementException; +import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; import static java.nio.file.StandardOpenOption.APPEND; import static java.nio.file.StandardOpenOption.CREATE; +import static org.apache.iotdb.db.utils.ModificationUtils.sortAndMerge; public class ModificationFile implements AutoCloseable { public static final String FILE_SUFFIX = ".mods2"; public static final String COMPACTION_FILE_SUFFIX = ".compaction.mods2"; + public static final String COMPACT_SUFFIX = ".settle"; private static final Logger LOGGER = LoggerFactory.getLogger(ModificationFile.class); private final File file; @@ -54,6 +61,13 @@ public class ModificationFile implements AutoCloseable { private OutputStream fileOutputStream; private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private static final long COMPACT_THRESHOLD = 1024 * 1024L; + private boolean hasCompacted = false; + + public ModificationFile(String filePath) { + this.file = new File(filePath); + } + public ModificationFile(File file) { this.file = file; } @@ -227,4 +241,44 @@ public void truncate(long size) throws IOException { public String toString() { return "ModificationFile{" + "file=" + file + '}'; } + + public void compact() { + long originFileSize = getSize(); + if (originFileSize > COMPACT_THRESHOLD && !hasCompacted) { + try { + Map> pathModificationMap = + getAllMods().stream().collect(Collectors.groupingBy(ModEntry::keyOfPatternTree)); + String newModsFileName = getFile().getPath() + COMPACT_SUFFIX; + List allSettledModifications = new ArrayList<>(); + try (ModificationFile compactedModificationFile = new ModificationFile(newModsFileName)) { + Set>> modificationsEntrySet = + pathModificationMap.entrySet(); + for (Map.Entry> modificationEntry : modificationsEntrySet) { + List settledModifications = sortAndMerge(modificationEntry.getValue()); + for (ModEntry settledModification : settledModifications) { + compactedModificationFile.write(settledModification); + } + allSettledModifications.addAll(settledModifications); + } + } catch (IOException e) { + LOGGER.error("compact mods file exception of {}", file, e); + } + // remove origin mods file + this.remove(); + // rename new mods file to origin name + Files.move(new File(newModsFileName).toPath(), file.toPath()); + LOGGER.info("{} settle successful", file); + + if (getSize() > COMPACT_THRESHOLD) { + LOGGER.warn( + "After the mod file is settled, the file size is still greater than 1M,the size of the file before settle is {},after settled the file size is {}", + originFileSize, + getSize()); + } + } catch (IOException e) { + LOGGER.error("remove origin file or rename new mods file error.", e); + } + hasCompacted = true; + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index dd18ed2e5df3..6729ef0b036d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -709,10 +709,10 @@ public void moveTo(File targetDir) throws IOException { fsFactory.moveFile( fsFactory.getFile(file.getPath() + RESOURCE_SUFFIX), fsFactory.getFile(targetDir, file.getName() + RESOURCE_SUFFIX)); - File originModFile = fsFactory.getFile(file.getPath() + ModificationFileV1.FILE_SUFFIX); - if (originModFile.exists()) { + + if (newModFileExists()) { fsFactory.moveFile( - originModFile, + getNewModFile().getFile(), fsFactory.getFile(targetDir, file.getName() + ModificationFileV1.FILE_SUFFIX)); } } @@ -1319,6 +1319,7 @@ public ModEntry next() { } } + @SuppressWarnings({"java:S4042", "java:S899", "ResultOfMethodCallIgnored"}) public void upgradeModFile(ExecutorService upgradeModFileThreadPool) throws IOException { ModificationFileV1 oldModFile = ModificationFileV1.getNormalMods(this); if (!oldModFile.exists()) { @@ -1329,6 +1330,7 @@ public void upgradeModFile(ExecutorService upgradeModFileThreadPool) throws IOEx upgradeModFileThreadPool.submit( () -> { ModificationFile newMFile = ModificationFile.getNormalMods(this); + newMFile.getFile().delete(); try { for (Modification oldMod : oldModFile.getModificationsIter()) { newMFile.write(new TreeDeletionEntry((Deletion) oldMod)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 57423ee3b382..622267f7d638 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -39,7 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler.LoadCommand; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.flush.MemTableFlushTask; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; import org.apache.iotdb.db.storageengine.load.active.ActiveLoadAgent; @@ -362,7 +362,7 @@ private static class TsFileWriterManager { private final File taskDir; private Map dataPartition2Writer; private Map dataPartition2LastDevice; - private Map dataPartition2ModificationFile; + private Map dataPartition2ModificationFile; private boolean isClosed; private TsFileWriterManager(File taskDir) { @@ -428,9 +428,7 @@ private void writeDeletion(DataRegion dataRegion, DeletionData deletionData) if (partitionInfo.getDataRegion().equals(dataRegion)) { final TsFileIOWriter writer = entry.getValue(); if (!dataPartition2ModificationFile.containsKey(partitionInfo)) { - File newModificationFile = - SystemFileFactory.INSTANCE.getFile( - writer.getFile().getAbsolutePath() + ModificationFileV1.FILE_SUFFIX); + File newModificationFile = ModificationFile.getNormalMods(writer.getFile()); if (!newModificationFile.createNewFile()) { LOGGER.error( "Can not create ModificationFile {} for writing.", newModificationFile.getPath()); @@ -438,11 +436,11 @@ private void writeDeletion(DataRegion dataRegion, DeletionData deletionData) } dataPartition2ModificationFile.put( - partitionInfo, new ModificationFileV1(newModificationFile.getAbsolutePath())); + partitionInfo, new ModificationFile(newModificationFile.getAbsolutePath())); } - ModificationFileV1 modificationFile = dataPartition2ModificationFile.get(partitionInfo); + ModificationFile modificationFile = dataPartition2ModificationFile.get(partitionInfo); writer.flush(); - deletionData.writeToModificationFile(modificationFile, writer.getFile().length()); + deletionData.writeToModificationFile(modificationFile); } } } @@ -452,7 +450,7 @@ private void loadAll(boolean isGeneratedByPipe, ProgressIndex progressIndex) if (isClosed) { throw new IOException(String.format(MESSAGE_WRITER_MANAGER_HAS_BEEN_CLOSED, taskDir)); } - for (Map.Entry entry : + for (Map.Entry entry : dataPartition2ModificationFile.entrySet()) { entry.getValue().close(); } @@ -512,17 +510,17 @@ private void close() { } } if (dataPartition2ModificationFile != null) { - for (Map.Entry entry : + for (Map.Entry entry : dataPartition2ModificationFile.entrySet()) { try { - final ModificationFileV1 modificationFile = entry.getValue(); + final ModificationFile modificationFile = entry.getValue(); modificationFile.close(); - final Path modificationFilePath = new File(modificationFile.getFilePath()).toPath(); + final Path modificationFilePath = modificationFile.getFile().toPath(); if (Files.exists(modificationFilePath)) { Files.delete(modificationFilePath); } } catch (IOException e) { - LOGGER.warn("Close ModificationFile {} error.", entry.getValue().getFilePath(), e); + LOGGER.warn("Close ModificationFile {} error.", entry.getValue().getFile(), e); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java index 67f4eb5d1d8f..0695c7a84def 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java @@ -20,8 +20,8 @@ package org.apache.iotdb.db.storageengine.load.splitter; import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -31,21 +31,19 @@ import java.io.InputStream; public class DeletionData implements TsFileData { - private final Deletion deletion; + private final ModEntry deletion; - public DeletionData(Deletion deletion) { + public DeletionData(ModEntry deletion) { this.deletion = deletion; } @Override public long getDataSize() { - return deletion.getSerializedSize(); + return deletion.serializedSize(); } - public void writeToModificationFile(ModificationFileV1 modificationFile, long fileOffset) - throws IOException { - deletion.setFileOffset(fileOffset); - modificationFile.writeWithoutSync(deletion); + public void writeToModificationFile(ModificationFile modificationFile) throws IOException { + modificationFile.write(deletion); } @Override @@ -56,11 +54,11 @@ public TsFileDataType getType() { @Override public void serialize(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(getType().ordinal(), stream); - deletion.serializeWithoutFileOffset(stream); + deletion.serialize(stream); } public static DeletionData deserialize(InputStream stream) throws IllegalPathException, IOException { - return new DeletionData(Deletion.deserializeWithoutFileOffset(new DataInputStream(stream))); + return new DeletionData(ModEntry.createFrom(new DataInputStream(stream))); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java index 49e328cb7bbc..01d06d384b77 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java @@ -21,9 +21,8 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.utils.TimePartitionUtils; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Modification; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; @@ -58,7 +57,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.function.Function; public class TsFileSplitter { @@ -67,7 +65,7 @@ public class TsFileSplitter { private final File tsFile; private final Function consumer; private Map offset2ChunkMetadata = new HashMap<>(); - private TreeMap> offset2Deletions = new TreeMap<>(); + private List deletions = new ArrayList<>(); private Map> pageIndex2ChunkData = new HashMap<>(); private Map pageIndex2Times = new HashMap<>(); private boolean isTimeChunkNeedDecode = true; @@ -92,7 +90,7 @@ public TsFileSplitter(File tsFile, Function consumer) { @SuppressWarnings({"squid:S3776", "squid:S6541"}) public void splitTsFileByDataPartition() throws IOException, IllegalStateException { try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { - getAllModification(offset2Deletions); + getAllModification(deletions); if (!checkMagic(reader)) { throw new TsFileRuntimeException( @@ -140,7 +138,7 @@ public void splitTsFileByDataPartition() throws IOException, IllegalStateExcepti } consumeAllAlignedChunkData(reader.position(), pageIndex2ChunkData); - handleModification(offset2Deletions, Long.MAX_VALUE); + handleModification(deletions); } } @@ -149,7 +147,7 @@ private void processTimeChunkOrNonAlignedChunk(TsFileSequenceReader reader, byte long chunkOffset = reader.position(); timeChunkIndexOfCurrentValueColumn = pageIndex2TimesList.size(); consumeAllAlignedChunkData(chunkOffset, pageIndex2ChunkData); - handleModification(offset2Deletions, chunkOffset); + handleModification(deletions); ChunkHeader header = reader.readChunkHeader(marker); String measurementId = header.getMeasurementID(); @@ -367,14 +365,10 @@ private void switchToTimeChunkContextOfCurrentMeasurement( isTimeChunkNeedDecode = isTimeChunkNeedDecodeList.get(index); } - private void getAllModification(Map> offset2Deletions) throws IOException { - try (ModificationFileV1 modificationFile = - new ModificationFileV1(tsFile.getAbsolutePath() + ModificationFileV1.FILE_SUFFIX)) { - for (Modification modification : modificationFile.getModifications()) { - offset2Deletions - .computeIfAbsent(modification.getFileOffset(), o -> new ArrayList<>()) - .add((Deletion) modification); - } + private void getAllModification(List deletions) throws IOException { + try (ModificationFile modificationFile = + new ModificationFile(ModificationFile.getNormalMods(tsFile))) { + deletions.addAll(modificationFile.getAllMods()); } } @@ -421,14 +415,8 @@ private void getChunkMetadata( } } - private void handleModification( - TreeMap> offset2Deletions, long chunkOffset) { - while (!offset2Deletions.isEmpty() && offset2Deletions.firstEntry().getKey() <= chunkOffset) { - offset2Deletions - .pollFirstEntry() - .getValue() - .forEach(o -> consumer.apply(new DeletionData(o))); - } + private void handleModification(List deletions) { + deletions.forEach(o -> consumer.apply(new DeletionData(o))); } private void consumeAllAlignedChunkData( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java index 132211af0fc8..92321a1b43bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java @@ -22,7 +22,6 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; @@ -103,7 +102,7 @@ public TsFileSplitByPartitionTool(TsFileResource resourceToBeRewritten) throws I String file = oldTsFile.getAbsolutePath(); reader = new TsFileSequenceReader(file); partitionWriterMap = new HashMap<>(); - if (FSFactoryProducer.getFSFactory().getFile(file + ModificationFileV1.FILE_SUFFIX).exists()) { + if (resourceToBeRewritten.newModFileExists()) { oldModification = resourceToBeRewritten.getAllModEntries(); modsIterator = oldModification.iterator(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java index feda5fbf3462..6dfd13424b93 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitTool.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; @@ -50,6 +50,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.HashSet; @@ -103,7 +104,7 @@ public TsFileSplitTool(String filename) { /* entry of tool */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning public void run() throws IOException { - if (fsFactory.getFile(filename + ModificationFileV1.FILE_SUFFIX).exists()) { + if (ModificationFile.getNormalMods(new File(filename)).exists()) { throw new IOException("Unsupported to split TsFile with modification currently."); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java index 302a2ee6038d..ea12e41da206 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java @@ -26,9 +26,9 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResource; import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Modification; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -128,15 +128,15 @@ private void createTsfile(String tsfilePath) throws Exception { } private void creatModsFile(String modsFilePath) throws IllegalPathException { - Modification[] modifications = - new Modification[] { - new Deletion(new MeasurementPath("root.lemming.device1.sensor1"), 2, 1), - new Deletion(new MeasurementPath("root.lemming.device1.sensor1"), 3, 2, 5), - new Deletion(new MeasurementPath("root.lemming.**"), 11, 1, Long.MAX_VALUE) + ModEntry[] modifications = + new ModEntry[] { + new TreeDeletionEntry(new MeasurementPath("root.lemming.device1.sensor1"), 1), + new TreeDeletionEntry(new MeasurementPath("root.lemming.device1.sensor1"), 2, 5), + new TreeDeletionEntry(new MeasurementPath("root.lemming.**"), 1, Long.MAX_VALUE) }; - try (ModificationFileV1 mFile = new ModificationFileV1(modsFilePath)) { - for (Modification mod : modifications) { + try (ModificationFile mFile = new ModificationFile(modsFilePath)) { + for (ModEntry mod : modifications) { mFile.write(mod); } } catch (IOException e) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java index 2b3ffb8e3fff..51ff35b9abd0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java @@ -40,7 +40,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -775,8 +775,8 @@ protected void resetFileName(TsFileResource resource, int version) { file.renameTo(new File(newFileName + TsFileResource.RESOURCE_SUFFIX)); // rename mods file - file = new File(resource.getTsFilePath() + ModificationFileV1.FILE_SUFFIX); - file.renameTo(new File(newFileName + ModificationFileV1.FILE_SUFFIX)); + file = ModificationFile.getNormalMods(resource.getTsFile()); + file.renameTo(ModificationFile.getNormalMods(new File(newFileName))); } protected TsFileResource generateSingleAlignedSeriesFile( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java index 85629ce2415a..b0604044bbb8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java @@ -31,7 +31,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.utils.CrossSpaceCompactionCandidate; import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.utils.InsertionCrossCompactionTaskResource; import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionFileGeneratorUtils; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; @@ -235,8 +235,8 @@ public void testRecoverWithTargetModFileNotExistedAndSourceModNotExisted() new File(sourceTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath()); if (unseqResource1.newModFileExists()) { Files.createLink( - new File(targetTsFile.getPath() + ModificationFileV1.FILE_SUFFIX).toPath(), - new File(sourceTsFile.getPath() + ModificationFileV1.FILE_SUFFIX).toPath()); + ModificationFile.getNormalMods(targetTsFile).toPath(), + ModificationFile.getNormalMods(sourceTsFile).toPath()); } } @@ -329,8 +329,8 @@ public void testRecoverWithAllTargetFileExisted() new File(sourceTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath()); if (unseqResource1.newModFileExists()) { Files.createLink( - new File(targetTsFile.getPath() + ModificationFileV1.FILE_SUFFIX).toPath(), - new File(sourceTsFile.getPath() + ModificationFileV1.FILE_SUFFIX).toPath()); + ModificationFile.getNormalMods(targetTsFile).toPath(), + ModificationFile.getNormalMods(sourceTsFile).toPath()); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionSelectorTest.java index a786ed4c565f..2d6203d3c67b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionSelectorTest.java @@ -30,7 +30,6 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.impl.SizeTieredCompactionSelector; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; @@ -669,7 +668,7 @@ public void testSelectWhenModsFileGreaterThan50M() // mock modification file size must greater than 50M MockModiFicationFile mockModiFicationFile = new MockModiFicationFile( - seqResources.get(0).getTsFilePath() + ModificationFileV1.FILE_SUFFIX); + ModificationFile.getNormalMods(seqResources.get(0).getTsFile()).getPath()); mockModiFicationFile.write(new TreeDeletionEntry(new MeasurementPath("root.a.b"), 1)); seqResources.get(0).setNewModFile(mockModiFicationFile); SizeTieredCompactionSelector selector = @@ -693,7 +692,7 @@ public void testSelectWhenModsFileGreaterThan50MWithTsFileResourceStatusIsNotNor // mock modification file size must greater than 50M MockModiFicationFile mockModiFicationFile = new MockModiFicationFile( - seqResources.get(0).getTsFilePath() + ModificationFileV1.FILE_SUFFIX); + ModificationFile.getNormalMods(seqResources.get(0).getTsFile()).getPath()); mockModiFicationFile.write(new TreeDeletionEntry(new MeasurementPath("root.a.b"), 1)); seqResources.get(0).setNewModFile(mockModiFicationFile); seqResources.get(0).setStatusForTest(TsFileResourceStatus.COMPACTING); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java index abfb5465eb29..5664a7f3ae81 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java @@ -34,9 +34,8 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.reader.IDataBlockReader; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.reader.SeriesDataBlockReader; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Modification; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -301,14 +300,13 @@ private static void checkDataFromFrontToEnd( } } - Collection modifications = - ModificationFileV1.getNormalMods(mergedFile).getModifications(); - for (Modification modification : modifications) { - Deletion deletion = (Deletion) modification; - if (mergedData.containsKey(deletion.getPath().getFullPath())) { + Collection modifications = ModificationFile.getNormalMods(mergedFile).getAllMods(); + for (ModEntry modification : modifications) { + TreeDeletionEntry deletion = (TreeDeletionEntry) modification; + if (mergedData.containsKey(deletion.getPathPattern().getFullPath())) { long deletedCount = 0L; Iterator timeValuePairIterator = - mergedData.get(deletion.getPath().getFullPath()).iterator(); + mergedData.get(deletion.getPathPattern().getFullPath()).iterator(); while (timeValuePairIterator.hasNext()) { TimeValuePair timeValuePair = timeValuePairIterator.next(); if (timeValuePair.getTimestamp() >= deletion.getStartTime() @@ -317,9 +315,9 @@ private static void checkDataFromFrontToEnd( deletedCount++; } } - long count = fullPathPointNum.get(deletion.getPath().getFullPath()); + long count = fullPathPointNum.get(deletion.getPathPattern().getFullPath()); count = count - deletedCount; - fullPathPointNum.put(deletion.getPath().getFullPath(), count); + fullPathPointNum.put(deletion.getPathPattern().getFullPath(), count); } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionClearUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionClearUtils.java index 1a0ce3f355d5..71f05ca8ce52 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionClearUtils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionClearUtils.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.storageengine.buffer.ChunkCache; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -43,7 +43,7 @@ public static void clearAllCompactionFiles() throws IOException { FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders(); deleteAllFilesInOneDirBySuffix("target", TsFileConstant.TSFILE_SUFFIX); deleteAllFilesInOneDirBySuffix("target", TsFileResource.RESOURCE_SUFFIX); - deleteAllFilesInOneDirBySuffix("target", ModificationFileV1.FILE_SUFFIX); + deleteAllFilesInOneDirBySuffix("target", ModificationFile.FILE_SUFFIX); deleteAllFilesInOneDirBySuffix("target", IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX); deleteAllFilesInOneDirBySuffix("target", IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX); deleteAllFilesInOneDirBySuffix("target", CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java index 1a883df1426d..271d375be468 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java @@ -21,10 +21,6 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.recover.CompactionRecoverManager; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Modification; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.io.LocalTextModificationAccessor; import org.apache.iotdb.db.utils.constant.TestConstant; import org.junit.Assert; @@ -45,18 +41,18 @@ public class ModificationFileTest { @Test public void readMyWrite() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("mod.temp"); - Modification[] modifications = - new Modification[] { - new Deletion(new MeasurementPath(new String[] {"d1", "s1"}), 1, 1), - new Deletion(new MeasurementPath(new String[] {"d1", "s2"}), 2, 2), - new Deletion(new MeasurementPath(new String[] {"d1", "s3"}), 3, 3, 4), - new Deletion(new MeasurementPath(new String[] {"d1", "s41"}), 4, 4, 5) + ModEntry[] modifications = + new ModEntry[] { + new TreeDeletionEntry(new MeasurementPath(new String[] {"d1", "s1"}), 1), + new TreeDeletionEntry(new MeasurementPath(new String[] {"d1", "s2"}), 2), + new TreeDeletionEntry(new MeasurementPath(new String[] {"d1", "s3"}), 3, 4), + new TreeDeletionEntry(new MeasurementPath(new String[] {"d1", "s41"}), 4, 5) }; - try (ModificationFileV1 mFile = new ModificationFileV1(tempFileName)) { + try (ModificationFile mFile = new ModificationFile(tempFileName)) { for (int i = 0; i < 2; i++) { mFile.write(modifications[i]); } - List modificationList = (List) mFile.getModifications(); + List modificationList = mFile.getAllMods(); for (int i = 0; i < 2; i++) { assertEquals(modifications[i], modificationList.get(i)); } @@ -64,7 +60,7 @@ public void readMyWrite() { for (int i = 2; i < 4; i++) { mFile.write(modifications[i]); } - modificationList = (List) mFile.getModifications(); + modificationList = mFile.getAllMods(); for (int i = 0; i < 4; i++) { assertEquals(modifications[i], modificationList.get(i)); } @@ -78,19 +74,15 @@ public void readMyWrite() { @Test public void writeVerifyTest() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("mod.temp"); - Modification[] modifications = - new Modification[] { - new Deletion(new MeasurementPath(new String[] {"d1", "s1"}), 1, 1), - new Deletion(new MeasurementPath(new String[] {"d1", "s2"}), 2, 2) + ModEntry[] modifications = + new ModEntry[] { + new TreeDeletionEntry(new MeasurementPath(new String[] {"d1", "s1"}), 1), + new TreeDeletionEntry(new MeasurementPath(new String[] {"d1", "s2"}), 2) }; - try (ModificationFileV1 mFile = new ModificationFileV1(tempFileName)) { + try (ModificationFile mFile = new ModificationFile(tempFileName)) { mFile.write(modifications[0]); - try (LocalTextModificationAccessor accessor = - new LocalTextModificationAccessor(tempFileName)) { - accessor.writeInComplete(modifications[1]); - } mFile.write(modifications[1]); - List modificationList = (List) mFile.getModifications(); + List modificationList = mFile.getAllMods(); assertEquals(2, modificationList.size()); for (int i = 0; i < 2; i++) { assertEquals(modifications[i], modificationList.get(i)); @@ -107,20 +99,19 @@ public void writeVerifyTest() { public void testCompact01() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact01.mods"); long time = 1000; - try (ModificationFileV1 modificationFile = new ModificationFileV1(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { while (modificationFile.getSize() < 1024 * 1024) { modificationFile.write( - new Deletion( + new TreeDeletionEntry( new MeasurementPath(new String[] {"root", "sg", "d1"}), - 1000, Long.MIN_VALUE, time += 5000)); } modificationFile.compact(); - List modificationList = new ArrayList<>(modificationFile.getModifications()); + List modificationList = new ArrayList<>(modificationFile.getAllMods()); assertEquals(1, modificationList.size()); - Deletion deletion = (Deletion) modificationList.get(0); + ModEntry deletion = modificationList.get(0); assertEquals(time, deletion.getEndTime()); assertEquals(Long.MIN_VALUE, deletion.getStartTime()); } catch (IOException e) { @@ -135,17 +126,16 @@ public void testCompact01() { public void testCompact02() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact02.mods"); long time = 1000; - try (ModificationFileV1 modificationFile = new ModificationFileV1(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { while (modificationFile.getSize() < 1024 * 100) { modificationFile.write( - new Deletion( + new TreeDeletionEntry( new MeasurementPath(new String[] {"root", "sg", "d1"}), - 1000, Long.MIN_VALUE, time += 5000)); } modificationFile.compact(); - List modificationList = new ArrayList<>(modificationFile.getModifications()); + List modificationList = new ArrayList<>(modificationFile.getAllMods()); assertTrue(modificationList.size() > 1); } catch (IOException e) { fail(e.getMessage()); @@ -158,20 +148,19 @@ public void testCompact02() { @Test public void testCompact03() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact03.mods"); - try (ModificationFileV1 modificationFile = new ModificationFileV1(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { while (modificationFile.getSize() < 1024 * 1024) { modificationFile.write( - new Deletion( + new TreeDeletionEntry( new MeasurementPath(new String[] {"root", "sg", "d1"}), - 1000, Long.MIN_VALUE, Long.MAX_VALUE)); } modificationFile.compact(); - List modificationList = new ArrayList<>(modificationFile.getModifications()); + List modificationList = new ArrayList<>(modificationFile.getAllMods()); assertEquals(1, modificationList.size()); - Deletion deletion = (Deletion) modificationList.get(0); + ModEntry deletion = modificationList.get(0); assertEquals(Long.MAX_VALUE, deletion.getEndTime()); assertEquals(Long.MIN_VALUE, deletion.getStartTime()); } catch (IOException e) { @@ -184,26 +173,24 @@ public void testCompact03() { @Test public void testCompact04() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact04.mods"); - try (ModificationFileV1 modificationFile = new ModificationFileV1(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { long time = 0; while (modificationFile.getSize() < 1024 * 1024) { for (int i = 0; i < 5; i++) { modificationFile.write( - new Deletion( + new TreeDeletionEntry( new MeasurementPath(new String[] {"root", "sg", "d1"}), - 1000, Long.MIN_VALUE, time += 5000)); modificationFile.write( - new Deletion( + new TreeDeletionEntry( new MeasurementPath(new String[] {"root", "sg", "*"}), - 1000, Long.MIN_VALUE, time += 5000)); } } modificationFile.compact(); - List modificationList = new ArrayList<>(modificationFile.getModifications()); + List modificationList = new ArrayList<>(modificationFile.getAllMods()); assertEquals(2, modificationList.size()); } catch (IOException e) { fail(e.getMessage()); @@ -218,19 +205,17 @@ public void testRecover01() { String modsFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact01.mods"); String modsSettleFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact01.mods.settle"); - try (ModificationFileV1 modsFile = new ModificationFileV1(modsFileName); - ModificationFileV1 modsSettleFile = new ModificationFileV1(modsSettleFileName)) { + try (ModificationFile modsFile = new ModificationFile(modsFileName); + ModificationFile modsSettleFile = new ModificationFile(modsSettleFileName)) { modsFile.write( - new Deletion( + new TreeDeletionEntry( new MeasurementPath(new String[] {"root", "sg", "d1"}), - 1000, Long.MIN_VALUE, Long.MAX_VALUE)); modsSettleFile.write( - new Deletion( + new TreeDeletionEntry( new MeasurementPath(new String[] {"root", "sg", "d1"}), - 1000, Long.MIN_VALUE, Long.MAX_VALUE)); @@ -256,11 +241,10 @@ public void testRecover01() { public void testRecover02() { String modsSettleFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact02.mods.settle"); String originModsFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact02.mods"); - try (ModificationFileV1 modsSettleFile = new ModificationFileV1(modsSettleFileName)) { + try (ModificationFile modsSettleFile = new ModificationFile(modsSettleFileName)) { modsSettleFile.write( - new Deletion( + new TreeDeletionEntry( new MeasurementPath(new String[] {"root", "sg", "d1"}), - 1000, Long.MIN_VALUE, Long.MAX_VALUE)); modsSettleFile.close(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoerTest.java index 248824ca916c..1479c407bd39 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoerTest.java @@ -30,7 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.TsFileUtilsForRecoverTest; import org.apache.iotdb.db.utils.EnvironmentUtils; @@ -105,7 +105,7 @@ public void tearDown() throws Exception { if (tsFileResource != null) { tsFileResource.close(); } - File modsFile = new File(FILE_NAME.concat(ModificationFileV1.FILE_SUFFIX)); + File modsFile = ModificationFile.getNormalMods(new File(FILE_NAME)); if (modsFile.exists()) { modsFile.delete(); } @@ -604,7 +604,7 @@ public void testRedoDeleteDataNode() throws Exception { Long.MAX_VALUE); // redo DeleteDataNode, vsg processor is used to test IdTable, don't test IdTable here - File modsFile = new File(FILE_NAME.concat(ModificationFileV1.FILE_SUFFIX)); + File modsFile = ModificationFile.getNormalMods(new File(FILE_NAME)); assertFalse(modsFile.exists()); TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource); planRedoer.redoDelete(deleteDataNode); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java index 9e8eff8cffdf..c03e89a7f47a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java @@ -30,7 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.PrimitiveMemTable; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALInfoEntry; @@ -179,7 +179,7 @@ public void testRedoDeletePlan() throws Exception { generateCrashedFile(file); assertTrue(file.exists()); assertFalse(new File(FILE_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); - assertFalse(new File(FILE_NAME.concat(ModificationFileV1.FILE_SUFFIX)).exists()); + assertFalse(ModificationFile.getNormalMods(new File(FILE_NAME)).exists()); // generate InsertRowPlan DeleteDataNode deleteDataNode = new DeleteDataNode( @@ -227,7 +227,7 @@ public void testRedoDeletePlan() throws Exception { // check file existence assertTrue(file.exists()); assertTrue(new File(FILE_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); - assertTrue(new File(FILE_NAME.concat(ModificationFileV1.FILE_SUFFIX)).exists()); + assertTrue(ModificationFile.getCompactionMods(new File(FILE_NAME)).exists()); } private void generateCrashedFile(File tsFile) throws IOException, WriteProcessException { @@ -282,7 +282,7 @@ public void testRecoverNullInsertRowPlan() throws Exception { generateCrashedFile(file); assertTrue(file.exists()); assertFalse(new File(FILE_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); - assertFalse(new File(FILE_NAME.concat(ModificationFileV1.FILE_SUFFIX)).exists()); + assertFalse(ModificationFile.getCompactionMods(new File(FILE_NAME)).exists()); // generate InsertRowNode with null long time = 4; InsertRowNode insertRowNode = @@ -343,7 +343,7 @@ public void testRecoverDuplicate() generateCrashedFile(file); assertTrue(file.exists()); assertFalse(new File(FILE_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); - assertFalse(new File(FILE_NAME.concat(ModificationFileV1.FILE_SUFFIX)).exists()); + assertFalse(ModificationFile.getNormalMods(new File(FILE_NAME)).exists()); tsFileResource = new TsFileResource(file); int fakeMemTableId = 1; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java index 54a66f21c10c..ca0e5fe5f772 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java @@ -25,10 +25,9 @@ import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Modification; -import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.tools.settle.TsFileAndModSettleTool; import org.apache.iotdb.db.utils.EnvironmentUtils; @@ -167,15 +166,14 @@ private void createFile( } public void createlModificationFile(String timeseriesPath) { - String modFilePath = path + ModificationFileV1.FILE_SUFFIX; - ModificationFileV1 modificationFile = new ModificationFileV1(modFilePath); - List mods = new ArrayList<>(); + ModificationFile modificationFile = new ModificationFile(new File(path)); + List mods = new ArrayList<>(); try { MeasurementPath partialPath = new MeasurementPath(timeseriesPath); - mods.add(new Deletion(partialPath, 10000000, 1500, 10000)); - mods.add(new Deletion(partialPath, 10000000, 20000, 30000)); - mods.add(new Deletion(partialPath, 10000000, 45000, 50000)); - for (Modification mod : mods) { + mods.add(new TreeDeletionEntry(partialPath, 1500, 10000)); + mods.add(new TreeDeletionEntry(partialPath, 20000, 30000)); + mods.add(new TreeDeletionEntry(partialPath, 45000, 50000)); + for (ModEntry mod : mods) { modificationFile.write(mod); } modificationFile.close();