From 959ebb869532916560406269662c1118e329aba0 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Thu, 2 Jan 2025 18:59:01 +0800 Subject: [PATCH 1/7] modify tablet usage --- .../tsfile/TsFileWriteAlignedWithTablet.java | 5 +-- .../apache/tsfile/TsFileWriteWithTablet.java | 3 +- .../tsfile/utils/TsFileGeneratorUtils.java | 5 +-- .../apache/tsfile/write/record/Tablet.java | 15 +++++--- .../tsfile/tableview/PerformanceTest.java | 15 +++----- .../write/DefaultSchemaTemplateTest.java | 5 +-- .../write/MetadataIndexConstructorTest.java | 5 +-- .../tsfile/write/TsFileWriteApiTest.java | 36 +++++-------------- .../apache/tsfile/write/TsFileWriterTest.java | 13 ++++--- 9 files changed, 34 insertions(+), 68 deletions(-) diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java index 784df63ba..ca13fdb36 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java @@ -133,8 +133,6 @@ private static void writeNonAlignedWithTablet(TsFileWriter tsFileWriter) measurementSchemas.add(new MeasurementSchema(SENSOR_1, TSDataType.INT64, TSEncoding.RLE)); measurementSchemas.add(new MeasurementSchema(SENSOR_2, TSDataType.INT64, TSEncoding.RLE)); Tablet tablet = new Tablet(DEVICE_2, measurementSchemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; int rowNum = 100; int sensorNum = measurementSchemas.size(); long timestamp = 1; @@ -143,8 +141,7 @@ private static void writeNonAlignedWithTablet(TsFileWriter tsFileWriter) int row = tablet.getRowSize(); tablet.addTimestamp(row, timestamp++); for (int i = 0; i < sensorNum; i++) { - long[] sensor = (long[]) values[i]; - sensor[row] = value; + tablet.addValue(row, i, value); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java index bb0293a52..05446ce8c 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java @@ -88,12 +88,11 @@ private static void writeWithTablet( long startValue) throws IOException, WriteProcessException { Tablet tablet = new Tablet(deviceId, schemas); - long[] timestamps = tablet.timestamps; long sensorNum = schemas.size(); for (long r = 0; r < rowNum; r++, startValue++) { int row = tablet.getRowSize(); - timestamps[row] = startTime++; + tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum; i++) { tablet.addValue( schemas.get(i).getMeasurementName(), diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java index 6b87f6e17..e4b8abcdf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java @@ -117,16 +117,13 @@ public static void writeWithTablet( boolean isAligned) throws IOException, WriteProcessException { Tablet tablet = new Tablet(deviceId, schemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; long sensorNum = schemas.size(); for (long r = 0; r < rowNum; r++, startValue++) { int row = tablet.getRowSize(); tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum; i++) { - long[] sensor = (long[]) values[i]; - sensor[row] = startValue; + tablet.addValue(row, i, startValue); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 2580a2a47..e36394a01 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -292,6 +292,7 @@ public void initBitMaps() { public void addTimestamp(int rowIndex, long timestamp) { timestamps[rowIndex] = timestamp; this.rowSize = Math.max(this.rowSize, rowIndex + 1); + initBitMapsWithApiUsage(); } public void addValue(final String measurementId, final int rowIndex, final Object value) { @@ -521,6 +522,15 @@ private int getColumnIndexByMeasurement(String measurement) { } private void updateBitMap(int rowIndex, int columnIndex, boolean mark) { + initBitMapsWithApiUsage(); + if (mark) { + bitMaps[columnIndex].mark(rowIndex); + } else { + bitMaps[columnIndex].unmark(rowIndex); + } + } + + private void initBitMapsWithApiUsage() { if (bitMaps == null) { initBitMaps(); } @@ -530,11 +540,6 @@ private void updateBitMap(int rowIndex, int columnIndex, boolean mark) { bitMap.markAll(); } } - if (mark) { - bitMaps[columnIndex].mark(rowIndex); - } else { - bitMaps[columnIndex].unmark(rowIndex); - } } public List getSchemas() { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java b/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java index 1cef0a40c..9f454d4e6 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java @@ -274,15 +274,13 @@ private Tablet initTreeTablet() { private void fillTreeTablet(Tablet tablet, int tableNum, int deviceNum, int tabletNum) { tablet.setDeviceId(genTreeDeviceId(tableNum, deviceNum).toString()); for (int i = 0; i < measurementSchemaCnt; i++) { - long[] values = (long[]) tablet.values[i]; for (int valNum = 0; valNum < pointPerSeries; valNum++) { - values[valNum] = (long) tabletNum * pointPerSeries + valNum; + tablet.addValue(valNum, i, tabletNum * pointPerSeries + valNum); } } for (int valNum = 0; valNum < pointPerSeries; valNum++) { - tablet.timestamps[valNum] = (long) tabletNum * pointPerSeries + valNum; + tablet.addTimestamp(valNum, tabletNum * pointPerSeries + valNum); } - tablet.setRowSize(pointPerSeries); } private Tablet initTableTablet() { @@ -303,21 +301,18 @@ private void fillTableTablet(Tablet tablet, int tableNum, int deviceNum, int tab IDeviceID deviceID = genTableDeviceId(tableNum, deviceNum); tablet.setTableName(deviceID.segment(0).toString()); for (int i = 0; i < idSchemaCnt; i++) { - String[] strings = ((String[]) tablet.values[i]); for (int rowNum = 0; rowNum < pointPerSeries; rowNum++) { - strings[rowNum] = deviceID.segment(i + 1).toString(); + tablet.addValue(rowNum, i, deviceID.segment(i + 1).toString()); } } for (int i = 0; i < measurementSchemaCnt; i++) { - long[] values = (long[]) tablet.values[i + idSchemaCnt]; for (int valNum = 0; valNum < pointPerSeries; valNum++) { - values[valNum] = (long) tabletNum * pointPerSeries + valNum; + tablet.addValue(valNum, i + idSchemaCnt, tabletNum * pointPerSeries + valNum); } } for (int valNum = 0; valNum < pointPerSeries; valNum++) { - tablet.timestamps[valNum] = (long) tabletNum * pointPerSeries + valNum; + tablet.addTimestamp(valNum, tabletNum * pointPerSeries + valNum); } - tablet.setRowSize(pointPerSeries); } private void registerTree(TsFileWriter writer) throws WriteProcessException { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java index 3ff19a1b7..8a9351a0f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java @@ -66,8 +66,6 @@ public void testUsingDefaultSchemaTemplate() throws IOException, WriteProcessExc writer.registerDevice("d1", "defaultTemplate"); Tablet tablet = new Tablet("d1", schemaList); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; long timestamp = 1; long value = 1L; @@ -76,8 +74,7 @@ public void testUsingDefaultSchemaTemplate() throws IOException, WriteProcessExc int row = tablet.getRowSize(); tablet.addTimestamp(row, timestamp++); for (int i = 0; i < 2; i++) { - long[] sensor = (long[]) values[i]; - sensor[row] = value; + tablet.addValue(row, i, value); } // write Tablet to TsFile if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java index 32cbb6e3e..799082704 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java @@ -477,16 +477,13 @@ private void generateFile( // add measurements into TSFileWriter // construct the tablet Tablet tablet = new Tablet(device.toString(), tabletSchema); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; long timestamp = 1; long value = 1000000L; for (int r = 0; r < rowNum; r++, value++) { int row = tablet.getRowSize(); tablet.addTimestamp(row, timestamp++); for (int j = 0; j < measurementNum; j++) { - long[] sensor = (long[]) values[j]; - sensor[row] = value; + tablet.addValue(row, j, value); } // write Tablet to TsFile if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java index 0a4d92f17..40abd8124 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java @@ -18,7 +18,6 @@ */ package org.apache.tsfile.write; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; @@ -37,7 +36,6 @@ import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsFileGeneratorUtils; import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; import org.apache.tsfile.write.chunk.ChunkWriterImpl; @@ -404,9 +402,6 @@ public void writeNonAlignedWithTabletWithNullValue() { tsFileWriter.registerTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = 0; for (long r = 0; r < 10000; r++) { @@ -417,14 +412,12 @@ public void writeNonAlignedWithTabletWithNullValue() { tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - Binary[] textSensor = (Binary[]) values[i]; - textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); + tablet.addValue(row, i, "testString........."); } if (r > 1000) { tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; - dateSensor[row] = LocalDate.of(2024, 4, 1); + tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { @@ -457,8 +450,6 @@ public void writeNonAlignedWithTabletWithNegativeTimestamps() { tsFileWriter.registerTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = -100; @@ -470,14 +461,12 @@ public void writeNonAlignedWithTabletWithNegativeTimestamps() { tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - Binary[] textSensor = (Binary[]) values[i]; - textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); + tablet.addValue(row, i, "testString........."); } if (r > 1000) { tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; - dateSensor[row] = LocalDate.of(2024, 4, 1); + tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { @@ -510,28 +499,23 @@ public void writeAlignedWithTabletWithNullValue() { tsFileWriter.registerAlignedTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = 0; for (long r = 0; r < 10000; r++) { int row = tablet.getRowSize(); tablet.addTimestamp(row, startTime++); - timestamps[row] = startTime++; for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - Binary[] textSensor = (Binary[]) values[i]; - textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); + tablet.addValue(row, i, "testString........."); } if (r > 1000) { tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; - dateSensor[row] = LocalDate.of(2024, 4, 1); + tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { @@ -589,8 +573,6 @@ public void writeDataToTabletsWithNegativeTimestamps() { tsFileWriter.registerAlignedTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = -1000; @@ -602,14 +584,12 @@ public void writeDataToTabletsWithNegativeTimestamps() { tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - Binary[] textSensor = (Binary[]) values[i]; - textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); + tablet.addValue(row, i, "testString........."); } if (r > 1000) { tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; - dateSensor[row] = LocalDate.of(2024, 4, 1); + tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java index 676aecfce..baf979a72 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java @@ -244,10 +244,9 @@ public void writeTablet() throws IOException, WriteProcessException { "s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY), new MeasurementSchema( "s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY))); - tablet.timestamps[0] = 10000; - ((float[]) tablet.values[0])[0] = 5.0f; - ((int[]) tablet.values[1])[0] = 5; - tablet.setRowSize(1); + tablet.addTimestamp(0, 10000); + tablet.addValue(0, 0, 5.0f); + tablet.addValue(0, 1, 5); writer.writeTree(tablet); closeFile(); readOneRow(); @@ -263,9 +262,9 @@ public void writeTablet2() throws IOException, WriteProcessException { "s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY), new MeasurementSchema( "s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY))); - tablet.timestamps[0] = 10000; - ((float[]) tablet.values[0])[0] = 5.0f; - tablet.setRowSize(1); + tablet.addTimestamp(0, 10000); + tablet.addValue(0, 0, 5.0f); + tablet.addValue(0, 1, 0); writer.writeTree(tablet); closeFile(); // in this case, the value of s2 = 0 at time 10000. From 9883e36f806c92e0a84db1a80dc53372890b21be Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 6 Jan 2025 17:49:08 +0800 Subject: [PATCH 2/7] modify tablet --- java/tsfile/pom.xml | 4 +- java/tsfile/pom.xml.versionsBackup | 293 ++++++++++++++++++ .../chunk/AlignedChunkGroupWriterImpl.java | 23 +- .../chunk/NonAlignedChunkGroupWriterImpl.java | 31 +- .../apache/tsfile/write/record/Tablet.java | 184 ++++++----- .../tsfile/write/TsFileWriteApiTest.java | 16 +- .../tsfile/write/record/TabletTest.java | 20 +- 7 files changed, 455 insertions(+), 116 deletions(-) create mode 100644 java/tsfile/pom.xml.versionsBackup diff --git a/java/tsfile/pom.xml b/java/tsfile/pom.xml index 0a2441b87..24e3f30e3 100644 --- a/java/tsfile/pom.xml +++ b/java/tsfile/pom.xml @@ -24,7 +24,7 @@ org.apache.tsfile tsfile-java - 2.0.0-SNAPSHOT + 1.2.0-tablet tsfile TsFile: Java: TsFile @@ -38,7 +38,7 @@ org.apache.tsfile common - 2.0.0-SNAPSHOT + 1.2.0-tablet com.github.luben diff --git a/java/tsfile/pom.xml.versionsBackup b/java/tsfile/pom.xml.versionsBackup new file mode 100644 index 000000000..0a2441b87 --- /dev/null +++ b/java/tsfile/pom.xml.versionsBackup @@ -0,0 +1,293 @@ + + + + 4.0.0 + + org.apache.tsfile + tsfile-java + 2.0.0-SNAPSHOT + + tsfile + TsFile: Java: TsFile + A columnar file format designed for time-series data + + ${tsfile.test.skip} + false + ${tsfile.test.skip} + + + + org.apache.tsfile + common + 2.0.0-SNAPSHOT + + + com.github.luben + zstd-jni + + + org.xerial.snappy + snappy-java + + + commons-io + commons-io + + + org.apache.commons + commons-lang3 + + + org.lz4 + lz4-java + + + org.tukaani + xz + + + org.slf4j + slf4j-api + + + org.antlr + antlr4-runtime + + + junit + junit + test + + + org.mockito + mockito-core + test + + + ch.qos.logback + logback-classic + test + + + com.google.code.gson + gson + test + + + + + + org.antlr + antlr4-maven-plugin + + + + antlr4 + + + false + true + src/main/antlr4/org/apache/tsfile/parser + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + + add-source + + generate-sources + + + ${project.build.directory}/generated-sources/antlr4 + ${project.build.directory}/generated-sources/freemarker + + + + + + + org.apache.maven.plugins + maven-clean-plugin + + + + ${basedir}/gen + + + ${basedir}/src + + **/*.tokens + + + + + + + + org.apache.felix + maven-bundle-plugin + + true + + <_include>-bnd.bnd + <_removeheaders>Bnd-LastModified,Built-By + dependencies + + + + + bundle-manifest + + manifest + + process-classes + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + ${project.build.outputDirectory}/META-INF/MANIFEST.MF + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${tsfile.ut.skip} + false + random + + + + org.apache.maven.plugins + maven-failsafe-plugin + + ${tsfile.test.skip} + ${tsfile.it.skip} + false + random + + + + run-integration-tests + + integration-test + verify + + integration-test + + + + + + org.apache.drill.tools + drill-fmpp-maven-plugin + + + generate-fmpp + + generate + + generate-sources + + ${project.basedir}/src/main/codegen/config.fmpp + ${project.build.directory}/generated-sources/freemarker + ${project.basedir}/src/main/codegen/templates/ + + + + + + + + + skipTsfileTests + + + skipTests + true + + + + true + true + true + + + + skipUT_TsFileTests + + + skipUTs + true + + + + true + + + + get-jar-with-dependencies + + + + org.apache.maven.plugins + maven-assembly-plugin + + + jar-with-dependencies + + + + + make-assembly + + + single + + + package + + + + + + + + diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java index 0b53b75ae..c051a702d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java @@ -233,7 +233,7 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) // TODO: changing to a column-first style by calculating the remaining page space of each // column firsts for (int row = startRowIndex; row < endRowIndex; row++) { - long time = tablet.timestamps[row]; + long time = tablet.getTimestamps()[row]; checkIsHistoryData(time); for (int columnIndex = 0; columnIndex < tablet.getSchemas().size(); columnIndex++) { if (tablet.getColumnTypes() != null @@ -242,18 +242,19 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) } boolean isNull = - tablet.bitMaps != null - && tablet.bitMaps[columnIndex] != null - && tablet.bitMaps[columnIndex].isMarked(row); + tablet.getBitMaps() != null + && tablet.getBitMaps()[columnIndex] != null + && tablet.getBitMaps()[columnIndex].isMarked(row); // check isNull by bitMap in tablet ValueChunkWriter valueChunkWriter = tryToAddSeriesWriterInternal(measurementSchemas.get(columnIndex)); switch (measurementSchemas.get(columnIndex).getType()) { case BOOLEAN: - valueChunkWriter.write(time, ((boolean[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write( + time, ((boolean[]) tablet.getValues()[columnIndex])[row], isNull); break; case INT32: - valueChunkWriter.write(time, ((int[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((int[]) tablet.getValues()[columnIndex])[row], isNull); break; case DATE: valueChunkWriter.write( @@ -261,23 +262,23 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) isNull ? 0 : DateUtils.parseDateExpressionToInt( - ((LocalDate[]) tablet.values[columnIndex])[row]), + ((LocalDate[]) tablet.getValues()[columnIndex])[row]), isNull); break; case INT64: case TIMESTAMP: - valueChunkWriter.write(time, ((long[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((long[]) tablet.getValues()[columnIndex])[row], isNull); break; case FLOAT: - valueChunkWriter.write(time, ((float[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((float[]) tablet.getValues()[columnIndex])[row], isNull); break; case DOUBLE: - valueChunkWriter.write(time, ((double[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((double[]) tablet.getValues()[columnIndex])[row], isNull); break; case TEXT: case BLOB: case STRING: - valueChunkWriter.write(time, ((Binary[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((Binary[]) tablet.getValues()[columnIndex])[row], isNull); break; default: throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java index 3f4d8aa0f..cc20b2520 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java @@ -121,42 +121,51 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) pointCount = 0; for (int row = startRowIndex; row < endRowIndex; row++) { // check isNull in tablet - if (tablet.bitMaps != null - && tablet.bitMaps[column] != null - && tablet.bitMaps[column].isMarked(row)) { + if (tablet.getBitMaps() != null + && tablet.getBitMaps()[column] != null + && tablet.getBitMaps()[column].isMarked(row)) { continue; } - long time = tablet.timestamps[row]; + long time = tablet.getTimestamps()[row]; checkIsHistoryData(measurementId, time); pointCount++; switch (tsDataType) { case INT32: - chunkWriters.get(measurementId).write(time, ((int[]) tablet.values[column])[row]); + chunkWriters.get(measurementId).write(time, ((int[]) tablet.getValues()[column])[row]); break; case DATE: chunkWriters .get(measurementId) .write( time, - DateUtils.parseDateExpressionToInt(((LocalDate[]) tablet.values[column])[row])); + DateUtils.parseDateExpressionToInt( + ((LocalDate[]) tablet.getValues()[column])[row])); break; case INT64: case TIMESTAMP: - chunkWriters.get(measurementId).write(time, ((long[]) tablet.values[column])[row]); + chunkWriters.get(measurementId).write(time, ((long[]) tablet.getValues()[column])[row]); break; case FLOAT: - chunkWriters.get(measurementId).write(time, ((float[]) tablet.values[column])[row]); + chunkWriters + .get(measurementId) + .write(time, ((float[]) tablet.getValues()[column])[row]); break; case DOUBLE: - chunkWriters.get(measurementId).write(time, ((double[]) tablet.values[column])[row]); + chunkWriters + .get(measurementId) + .write(time, ((double[]) tablet.getValues()[column])[row]); break; case BOOLEAN: - chunkWriters.get(measurementId).write(time, ((boolean[]) tablet.values[column])[row]); + chunkWriters + .get(measurementId) + .write(time, ((boolean[]) tablet.getValues()[column])[row]); break; case TEXT: case BLOB: case STRING: - chunkWriters.get(measurementId).write(time, ((Binary[]) tablet.values[column])[row]); + chunkWriters + .get(measurementId) + .write(time, ((Binary[]) tablet.getValues()[column])[row]); break; default: throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index e36394a01..d2af77ba2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -80,14 +80,11 @@ public class Tablet { /** MeasurementId->indexOf({@link MeasurementSchema}) */ private final Map measurementIndex; - /** Timestamps in this {@link Tablet} */ - public long[] timestamps; + private long[] timestamps; - /** Each object is a primitive type array, which represents values of one measurement */ - public Object[] values; + private Object[] values; - /** Each {@link BitMap} represents the existence of each value in the current column. */ - public BitMap[] bitMaps; + private BitMap[] bitMaps; /** * For compatibility with the usage of directly modifying Tablet content through public fields. @@ -232,9 +229,9 @@ public Tablet( this.insertTargetName = deviceId; this.schemas = schemas; setColumnCategories(ColumnCategory.nCopy(ColumnCategory.FIELD, schemas.size())); - this.timestamps = timestamps; - this.values = values; - this.bitMaps = bitMaps; + this.setTimestamps(timestamps); + this.setValues(values); + this.setBitMaps(bitMaps); this.maxRowNumber = maxRowNumber; // rowSize == maxRowNumber in this case this.rowSize = maxRowNumber; @@ -254,9 +251,9 @@ public Tablet( this.insertTargetName = tableName; this.schemas = schemas; setColumnCategories(columnCategories); - this.timestamps = timestamps; - this.values = values; - this.bitMaps = bitMaps; + this.setTimestamps(timestamps); + this.setValues(values); + this.setBitMaps(bitMaps); this.maxRowNumber = maxRowNumber; // rowSize == maxRowNumber in this case this.rowSize = maxRowNumber; @@ -281,16 +278,16 @@ public void setSchemas(List schemas) { } public void initBitMaps() { - this.bitMaps = new BitMap[schemas.size()]; + this.setBitMaps(new BitMap[schemas.size()]); for (int column = 0; column < schemas.size(); column++) { BitMap bitMap = new BitMap(getMaxRowNumber()); - this.bitMaps[column] = bitMap; + this.getBitMaps()[column] = bitMap; } } @TsFileApi public void addTimestamp(int rowIndex, long timestamp) { - timestamps[rowIndex] = timestamp; + getTimestamps()[rowIndex] = timestamp; this.rowSize = Math.max(this.rowSize, rowIndex + 1); initBitMapsWithApiUsage(); } @@ -317,7 +314,7 @@ private void addValueOfDataType( "Expected value of type Binary for data type %s, but got %s", dataType, value.getClass().getName())); } - final Binary[] sensor = (Binary[]) values[indexOfSchema]; + final Binary[] sensor = (Binary[]) getValues()[indexOfSchema]; if (value instanceof Binary) { sensor[rowIndex] = (Binary) value; } else { @@ -336,7 +333,7 @@ private void addValueOfDataType( "Expected value of type Float for data type %s, but got %s", dataType, value.getClass().getName())); } - final float[] sensor = (float[]) values[indexOfSchema]; + final float[] sensor = (float[]) getValues()[indexOfSchema]; sensor[rowIndex] = value != null ? (float) value : Float.MIN_VALUE; break; } @@ -348,7 +345,7 @@ private void addValueOfDataType( "Expected value of type Integer for data type %s, but got %s", dataType, value.getClass().getName())); } - final int[] sensor = (int[]) values[indexOfSchema]; + final int[] sensor = (int[]) getValues()[indexOfSchema]; sensor[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE; break; } @@ -360,7 +357,7 @@ private void addValueOfDataType( "Expected value of type LocalDate for data type %s, but got %s", dataType, value.getClass().getName())); } - final LocalDate[] sensor = (LocalDate[]) values[indexOfSchema]; + final LocalDate[] sensor = (LocalDate[]) getValues()[indexOfSchema]; sensor[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; break; } @@ -373,7 +370,7 @@ private void addValueOfDataType( "Expected value of type Long for data type %s, but got %s", dataType, value.getClass().getName())); } - final long[] sensor = (long[]) values[indexOfSchema]; + final long[] sensor = (long[]) getValues()[indexOfSchema]; sensor[rowIndex] = value != null ? (long) value : Long.MIN_VALUE; break; } @@ -385,7 +382,7 @@ private void addValueOfDataType( "Expected value of type Double for data type %s, but got %s", dataType, value.getClass().getName())); } - final double[] sensor = (double[]) values[indexOfSchema]; + final double[] sensor = (double[]) getValues()[indexOfSchema]; sensor[rowIndex] = value != null ? (double) value : Double.MIN_VALUE; break; } @@ -397,7 +394,7 @@ private void addValueOfDataType( "Expected value of type Boolean for data type %s, but got %s", dataType, value.getClass().getName())); } - final boolean[] sensor = (boolean[]) values[indexOfSchema]; + final boolean[] sensor = (boolean[]) getValues()[indexOfSchema]; sensor[rowIndex] = value != null && (boolean) value; break; } @@ -414,7 +411,7 @@ public void addValue(int rowIndex, String measurement, int val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, int val) { - final int[] sensor = (int[]) values[columnIndex]; + final int[] sensor = (int[]) getValues()[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -427,7 +424,7 @@ public void addValue(int rowIndex, String measurement, long val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, long val) { - final long[] sensor = (long[]) values[columnIndex]; + final long[] sensor = (long[]) getValues()[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -440,7 +437,7 @@ public void addValue(int rowIndex, String measurement, float val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, float val) { - final float[] sensor = (float[]) values[columnIndex]; + final float[] sensor = (float[]) getValues()[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -453,7 +450,7 @@ public void addValue(int rowIndex, String measurement, double val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, double val) { - final double[] sensor = (double[]) values[columnIndex]; + final double[] sensor = (double[]) getValues()[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -466,7 +463,7 @@ public void addValue(int rowIndex, String measurement, boolean val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, boolean val) { - final boolean[] sensor = (boolean[]) values[columnIndex]; + final boolean[] sensor = (boolean[]) getValues()[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -479,7 +476,7 @@ public void addValue(int rowIndex, String measurement, String val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, String val) { - final Binary[] sensor = (Binary[]) values[columnIndex]; + final Binary[] sensor = (Binary[]) getValues()[columnIndex]; sensor[rowIndex] = new Binary(val, TSFileConfig.STRING_CHARSET); updateBitMap(rowIndex, columnIndex, false); } @@ -492,7 +489,7 @@ public void addValue(int rowIndex, String measurement, byte[] val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, byte[] val) { - final Binary[] sensor = (Binary[]) values[columnIndex]; + final Binary[] sensor = (Binary[]) getValues()[columnIndex]; sensor[rowIndex] = new Binary(val); updateBitMap(rowIndex, columnIndex, false); } @@ -505,7 +502,7 @@ public void addValue(int rowIndex, String measurement, LocalDate val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, LocalDate val) { - final LocalDate[] sensor = (LocalDate[]) values[columnIndex]; + final LocalDate[] sensor = (LocalDate[]) getValues()[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -524,19 +521,19 @@ private int getColumnIndexByMeasurement(String measurement) { private void updateBitMap(int rowIndex, int columnIndex, boolean mark) { initBitMapsWithApiUsage(); if (mark) { - bitMaps[columnIndex].mark(rowIndex); + getBitMaps()[columnIndex].mark(rowIndex); } else { - bitMaps[columnIndex].unmark(rowIndex); + getBitMaps()[columnIndex].unmark(rowIndex); } } private void initBitMapsWithApiUsage() { - if (bitMaps == null) { + if (getBitMaps() == null) { initBitMaps(); } if (!autoUpdateBitMaps) { autoUpdateBitMaps = true; - for (BitMap bitMap : bitMaps) { + for (BitMap bitMap : getBitMaps()) { bitMap.markAll(); } } @@ -554,8 +551,8 @@ public int getMaxRowNumber() { /** Reset Tablet to the default state - set the rowSize to 0 and reset bitMaps */ public void reset() { this.rowSize = 0; - if (bitMaps != null) { - for (BitMap bitMap : bitMaps) { + if (getBitMaps() != null) { + for (BitMap bitMap : getBitMaps()) { if (bitMap == null) { continue; } @@ -570,18 +567,18 @@ public void reset() { private void createColumns() { // create timestamp column - timestamps = new long[maxRowNumber]; + setTimestamps(new long[maxRowNumber]); // calculate total value column size int valueColumnsSize = schemas.size(); // value column - values = new Object[valueColumnsSize]; + setValues(new Object[valueColumnsSize]); int columnIndex = 0; for (int i = 0; i < schemas.size(); i++) { IMeasurementSchema schema = schemas.get(i); TSDataType dataType = schema.getType(); - values[columnIndex] = createValueColumnOfDataType(dataType); + getValues()[columnIndex] = createValueColumnOfDataType(dataType); columnIndex++; } } @@ -658,26 +655,27 @@ private void writeMeasurementSchemas(DataOutputStream stream) throws IOException } private void writeTimes(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(BytesUtils.boolToByte(timestamps != null), stream); - if (timestamps != null) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(getTimestamps() != null), stream); + if (getTimestamps() != null) { for (int i = 0; i < rowSize; i++) { - ReadWriteIOUtils.write(timestamps[i], stream); + ReadWriteIOUtils.write(getTimestamps()[i], stream); } } } /** Serialize {@link BitMap}s */ private void writeBitMaps(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(BytesUtils.boolToByte(bitMaps != null), stream); - if (bitMaps != null) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(getBitMaps() != null), stream); + if (getBitMaps() != null) { int size = (schemas == null ? 0 : schemas.size()); for (int i = 0; i < size; i++) { - if (bitMaps[i] == null || bitMaps[i].isAllUnmarked(rowSize)) { + if (getBitMaps()[i] == null || getBitMaps()[i].isAllUnmarked(rowSize)) { ReadWriteIOUtils.write(BytesUtils.boolToByte(false), stream); } else { ReadWriteIOUtils.write(BytesUtils.boolToByte(true), stream); ReadWriteIOUtils.write(rowSize, stream); - ReadWriteIOUtils.write(new Binary(bitMaps[i].getTruncatedByteArray(rowSize)), stream); + ReadWriteIOUtils.write( + new Binary(getBitMaps()[i].getTruncatedByteArray(rowSize)), stream); } } } @@ -685,11 +683,11 @@ private void writeBitMaps(DataOutputStream stream) throws IOException { /** Serialize values */ private void writeValues(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(BytesUtils.boolToByte(values != null), stream); - if (values != null) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(getValues() != null), stream); + if (getValues() != null) { int size = (schemas == null ? 0 : schemas.size()); for (int i = 0; i < size; i++) { - serializeColumn(schemas.get(i).getType(), values[i], stream, columnCategories.get(i)); + serializeColumn(schemas.get(i).getType(), getValues()[i], stream, columnCategories.get(i)); } } } @@ -945,36 +943,36 @@ public boolean equals(Object o) { // assert timestamps and bitmaps int columns = (schemas == null ? 0 : schemas.size()); - if (!isTimestampsEqual(this.timestamps, that.timestamps, rowSize) - || !isBitMapsEqual(this.bitMaps, that.bitMaps, columns)) { + if (!isTimestampsEqual(this.getTimestamps(), that.getTimestamps(), rowSize) + || !isBitMapsEqual(this.getBitMaps(), that.getBitMaps(), columns)) { return false; } // assert values - Object[] thatValues = that.values; - if (thatValues == values) { + Object[] thatValues = that.getValues(); + if (thatValues == getValues()) { return true; } - if (thatValues == null || values == null) { + if (thatValues == null || getValues() == null) { return false; } - if (thatValues.length != values.length) { + if (thatValues.length != getValues().length) { return false; } - for (int i = 0, n = values.length; i < n; i++) { - if (thatValues[i] == values[i]) { + for (int i = 0, n = getValues().length; i < n; i++) { + if (thatValues[i] == getValues()[i]) { continue; } - if (thatValues[i] == null || values[i] == null) { + if (thatValues[i] == null || getValues()[i] == null) { return false; } - if (!thatValues[i].getClass().equals(values[i].getClass())) { + if (!thatValues[i].getClass().equals(getValues()[i].getClass())) { return false; } switch (schemas.get(i).getType()) { case INT32: - int[] thisIntValues = (int[]) values[i]; + int[] thisIntValues = (int[]) getValues()[i]; int[] thatIntValues = (int[]) thatValues[i]; if (thisIntValues.length < rowSize || thatIntValues.length < rowSize) { return false; @@ -986,7 +984,7 @@ public boolean equals(Object o) { } break; case DATE: - LocalDate[] thisDateValues = (LocalDate[]) values[i]; + LocalDate[] thisDateValues = (LocalDate[]) getValues()[i]; LocalDate[] thatDateValues = (LocalDate[]) thatValues[i]; if (thisDateValues.length < rowSize || thatDateValues.length < rowSize) { return false; @@ -999,7 +997,7 @@ public boolean equals(Object o) { break; case INT64: case TIMESTAMP: - long[] thisLongValues = (long[]) values[i]; + long[] thisLongValues = (long[]) getValues()[i]; long[] thatLongValues = (long[]) thatValues[i]; if (thisLongValues.length < rowSize || thatLongValues.length < rowSize) { return false; @@ -1011,7 +1009,7 @@ public boolean equals(Object o) { } break; case FLOAT: - float[] thisFloatValues = (float[]) values[i]; + float[] thisFloatValues = (float[]) getValues()[i]; float[] thatFloatValues = (float[]) thatValues[i]; if (thisFloatValues.length < rowSize || thatFloatValues.length < rowSize) { return false; @@ -1023,7 +1021,7 @@ public boolean equals(Object o) { } break; case DOUBLE: - double[] thisDoubleValues = (double[]) values[i]; + double[] thisDoubleValues = (double[]) getValues()[i]; double[] thatDoubleValues = (double[]) thatValues[i]; if (thisDoubleValues.length < rowSize || thatDoubleValues.length < rowSize) { return false; @@ -1035,7 +1033,7 @@ public boolean equals(Object o) { } break; case BOOLEAN: - boolean[] thisBooleanValues = (boolean[]) values[i]; + boolean[] thisBooleanValues = (boolean[]) getValues()[i]; boolean[] thatBooleanValues = (boolean[]) thatValues[i]; if (thisBooleanValues.length < rowSize || thatBooleanValues.length < rowSize) { return false; @@ -1049,7 +1047,7 @@ public boolean equals(Object o) { case TEXT: case STRING: case BLOB: - Binary[] thisBinaryValues = (Binary[]) values[i]; + Binary[] thisBinaryValues = (Binary[]) getValues()[i]; Binary[] thatBinaryValues = (Binary[]) thatValues[i]; if (thisBinaryValues.length < rowSize || thatBinaryValues.length < rowSize) { return false; @@ -1128,7 +1126,7 @@ private boolean isBitMapEqual(BitMap thisBitMap, BitMap thatBitMap) { } public boolean isNull(int i, int j) { - return bitMaps != null && bitMaps[j] != null && bitMaps[j].isMarked(i); + return getBitMaps() != null && getBitMaps()[j] != null && getBitMaps()[j].isMarked(i); } /** @@ -1144,20 +1142,20 @@ public Object getValue(int i, int j) { case BLOB: case TEXT: case STRING: - return ((Binary[]) values[j])[i]; + return ((Binary[]) getValues()[j])[i]; case INT32: - return ((int[]) values[j])[i]; + return ((int[]) getValues()[j])[i]; case FLOAT: - return ((float[]) values[j])[i]; + return ((float[]) getValues()[j])[i]; case DOUBLE: - return ((double[]) values[j])[i]; + return ((double[]) getValues()[j])[i]; case BOOLEAN: - return ((boolean[]) values[j])[i]; + return ((boolean[]) getValues()[j])[i]; case INT64: case TIMESTAMP: - return ((long[]) values[j])[i]; + return ((long[]) getValues()[j])[i]; case DATE: - return ((LocalDate[]) values[j])[i]; + return ((LocalDate[]) getValues()[j])[i]; default: throw new IllegalArgumentException("Unsupported type: " + schemas.get(j).getType()); } @@ -1200,6 +1198,35 @@ public void setRowSize(int rowSize) { this.rowSize = rowSize; } + public long getTimestamp(int i) { + return timestamps[i]; + } + + public long[] getTimestamps() { + return timestamps; + } + + public void setTimestamps(long[] timestamps) { + this.timestamps = timestamps; + } + + public Object[] getValues() { + return values; + } + + public void setValues(Object[] values) { + this.values = values; + } + + /** Each {@link BitMap} represents the existence of each value in the current column. */ + public BitMap[] getBitMaps() { + return bitMaps; + } + + public void setBitMaps(BitMap[] bitMaps) { + this.bitMaps = bitMaps; + } + public enum ColumnCategory { TAG, FIELD, @@ -1252,4 +1279,13 @@ public void setTableName(String tableName) { public List getColumnTypes() { return columnCategories; } + + public boolean isSorted() { + for (int i = 1; i < rowSize; i++) { + if (timestamps[i] < timestamps[i - 1]) { + return false; + } + } + return true; + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java index 40abd8124..5902172cb 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java @@ -409,13 +409,13 @@ public void writeNonAlignedWithTabletWithNullValue() { tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); continue; } tablet.addValue(row, i, "testString........."); } if (r > 1000) { - tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } @@ -458,13 +458,13 @@ public void writeNonAlignedWithTabletWithNegativeTimestamps() { tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); continue; } tablet.addValue(row, i, "testString........."); } if (r > 1000) { - tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } @@ -507,13 +507,13 @@ public void writeAlignedWithTabletWithNullValue() { tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); continue; } tablet.addValue(row, i, "testString........."); } if (r > 1000) { - tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } @@ -581,13 +581,13 @@ public void writeDataToTabletsWithNegativeTimestamps() { tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); continue; } tablet.addValue(row, i, "testString........."); } if (r > 1000) { - tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java index a73e22f9f..07aedad4f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java @@ -66,21 +66,21 @@ public void testAddValue() { Assert.assertFalse((Boolean) tablet.getValue(1, 0)); Assert.assertTrue((Boolean) tablet.getValue(1, 1)); Assert.assertTrue((Boolean) tablet.getValue(2, 0)); - Assert.assertFalse(tablet.bitMaps[0].isMarked(0)); - Assert.assertFalse(tablet.bitMaps[0].isMarked(1)); - Assert.assertFalse(tablet.bitMaps[0].isMarked(2)); - Assert.assertFalse(tablet.bitMaps[1].isMarked(0)); - Assert.assertFalse(tablet.bitMaps[1].isMarked(1)); - Assert.assertTrue(tablet.bitMaps[1].isMarked(2)); + Assert.assertFalse(tablet.getBitMaps()[0].isMarked(0)); + Assert.assertFalse(tablet.getBitMaps()[0].isMarked(1)); + Assert.assertFalse(tablet.getBitMaps()[0].isMarked(2)); + Assert.assertFalse(tablet.getBitMaps()[1].isMarked(0)); + Assert.assertFalse(tablet.getBitMaps()[1].isMarked(1)); + Assert.assertTrue(tablet.getBitMaps()[1].isMarked(2)); tablet.addTimestamp(9, 9); Assert.assertEquals(10, tablet.getRowSize()); tablet.reset(); Assert.assertEquals(0, tablet.getRowSize()); - Assert.assertTrue(tablet.bitMaps[0].isAllMarked()); - Assert.assertTrue(tablet.bitMaps[0].isAllMarked()); - Assert.assertTrue(tablet.bitMaps[0].isAllMarked()); + Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked()); + Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked()); + Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked()); } @Test @@ -163,7 +163,7 @@ public void testSerializationAndDeSerializationWithMoreData() { i, LocalDate.of(2000 + i, i / 100 + 1, i / 100 + 1)); - tablet.bitMaps[i % measurementSchemas.size()].mark(i); + tablet.getBitMaps()[i % measurementSchemas.size()].mark(i); } // Test add null From 1f07eaa53650a491996a01a290945dcb8dc7d8ec Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 6 Jan 2025 17:58:08 +0800 Subject: [PATCH 3/7] modify tablet --- .../apache/tsfile/write/record/Tablet.java | 119 +++++++++--------- 1 file changed, 59 insertions(+), 60 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index d2af77ba2..fc476f373 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -281,13 +281,13 @@ public void initBitMaps() { this.setBitMaps(new BitMap[schemas.size()]); for (int column = 0; column < schemas.size(); column++) { BitMap bitMap = new BitMap(getMaxRowNumber()); - this.getBitMaps()[column] = bitMap; + this.bitMaps[column] = bitMap; } } @TsFileApi public void addTimestamp(int rowIndex, long timestamp) { - getTimestamps()[rowIndex] = timestamp; + timestamps[rowIndex] = timestamp; this.rowSize = Math.max(this.rowSize, rowIndex + 1); initBitMapsWithApiUsage(); } @@ -314,7 +314,7 @@ private void addValueOfDataType( "Expected value of type Binary for data type %s, but got %s", dataType, value.getClass().getName())); } - final Binary[] sensor = (Binary[]) getValues()[indexOfSchema]; + final Binary[] sensor = (Binary[]) values[indexOfSchema]; if (value instanceof Binary) { sensor[rowIndex] = (Binary) value; } else { @@ -333,7 +333,7 @@ private void addValueOfDataType( "Expected value of type Float for data type %s, but got %s", dataType, value.getClass().getName())); } - final float[] sensor = (float[]) getValues()[indexOfSchema]; + final float[] sensor = (float[]) values[indexOfSchema]; sensor[rowIndex] = value != null ? (float) value : Float.MIN_VALUE; break; } @@ -345,7 +345,7 @@ private void addValueOfDataType( "Expected value of type Integer for data type %s, but got %s", dataType, value.getClass().getName())); } - final int[] sensor = (int[]) getValues()[indexOfSchema]; + final int[] sensor = (int[]) values[indexOfSchema]; sensor[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE; break; } @@ -357,7 +357,7 @@ private void addValueOfDataType( "Expected value of type LocalDate for data type %s, but got %s", dataType, value.getClass().getName())); } - final LocalDate[] sensor = (LocalDate[]) getValues()[indexOfSchema]; + final LocalDate[] sensor = (LocalDate[]) values[indexOfSchema]; sensor[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; break; } @@ -370,7 +370,7 @@ private void addValueOfDataType( "Expected value of type Long for data type %s, but got %s", dataType, value.getClass().getName())); } - final long[] sensor = (long[]) getValues()[indexOfSchema]; + final long[] sensor = (long[]) values[indexOfSchema]; sensor[rowIndex] = value != null ? (long) value : Long.MIN_VALUE; break; } @@ -382,7 +382,7 @@ private void addValueOfDataType( "Expected value of type Double for data type %s, but got %s", dataType, value.getClass().getName())); } - final double[] sensor = (double[]) getValues()[indexOfSchema]; + final double[] sensor = (double[]) values[indexOfSchema]; sensor[rowIndex] = value != null ? (double) value : Double.MIN_VALUE; break; } @@ -394,7 +394,7 @@ private void addValueOfDataType( "Expected value of type Boolean for data type %s, but got %s", dataType, value.getClass().getName())); } - final boolean[] sensor = (boolean[]) getValues()[indexOfSchema]; + final boolean[] sensor = (boolean[]) values[indexOfSchema]; sensor[rowIndex] = value != null && (boolean) value; break; } @@ -411,7 +411,7 @@ public void addValue(int rowIndex, String measurement, int val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, int val) { - final int[] sensor = (int[]) getValues()[columnIndex]; + final int[] sensor = (int[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -424,7 +424,7 @@ public void addValue(int rowIndex, String measurement, long val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, long val) { - final long[] sensor = (long[]) getValues()[columnIndex]; + final long[] sensor = (long[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -437,7 +437,7 @@ public void addValue(int rowIndex, String measurement, float val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, float val) { - final float[] sensor = (float[]) getValues()[columnIndex]; + final float[] sensor = (float[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -450,7 +450,7 @@ public void addValue(int rowIndex, String measurement, double val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, double val) { - final double[] sensor = (double[]) getValues()[columnIndex]; + final double[] sensor = (double[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -463,7 +463,7 @@ public void addValue(int rowIndex, String measurement, boolean val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, boolean val) { - final boolean[] sensor = (boolean[]) getValues()[columnIndex]; + final boolean[] sensor = (boolean[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -476,7 +476,7 @@ public void addValue(int rowIndex, String measurement, String val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, String val) { - final Binary[] sensor = (Binary[]) getValues()[columnIndex]; + final Binary[] sensor = (Binary[]) values[columnIndex]; sensor[rowIndex] = new Binary(val, TSFileConfig.STRING_CHARSET); updateBitMap(rowIndex, columnIndex, false); } @@ -489,7 +489,7 @@ public void addValue(int rowIndex, String measurement, byte[] val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, byte[] val) { - final Binary[] sensor = (Binary[]) getValues()[columnIndex]; + final Binary[] sensor = (Binary[]) values[columnIndex]; sensor[rowIndex] = new Binary(val); updateBitMap(rowIndex, columnIndex, false); } @@ -502,7 +502,7 @@ public void addValue(int rowIndex, String measurement, LocalDate val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, LocalDate val) { - final LocalDate[] sensor = (LocalDate[]) getValues()[columnIndex]; + final LocalDate[] sensor = (LocalDate[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); } @@ -521,19 +521,19 @@ private int getColumnIndexByMeasurement(String measurement) { private void updateBitMap(int rowIndex, int columnIndex, boolean mark) { initBitMapsWithApiUsage(); if (mark) { - getBitMaps()[columnIndex].mark(rowIndex); + bitMaps[columnIndex].mark(rowIndex); } else { - getBitMaps()[columnIndex].unmark(rowIndex); + bitMaps[columnIndex].unmark(rowIndex); } } private void initBitMapsWithApiUsage() { - if (getBitMaps() == null) { + if (bitMaps == null) { initBitMaps(); } if (!autoUpdateBitMaps) { autoUpdateBitMaps = true; - for (BitMap bitMap : getBitMaps()) { + for (BitMap bitMap : bitMaps) { bitMap.markAll(); } } @@ -551,8 +551,8 @@ public int getMaxRowNumber() { /** Reset Tablet to the default state - set the rowSize to 0 and reset bitMaps */ public void reset() { this.rowSize = 0; - if (getBitMaps() != null) { - for (BitMap bitMap : getBitMaps()) { + if (bitMaps != null) { + for (BitMap bitMap : bitMaps) { if (bitMap == null) { continue; } @@ -578,7 +578,7 @@ private void createColumns() { for (int i = 0; i < schemas.size(); i++) { IMeasurementSchema schema = schemas.get(i); TSDataType dataType = schema.getType(); - getValues()[columnIndex] = createValueColumnOfDataType(dataType); + values[columnIndex] = createValueColumnOfDataType(dataType); columnIndex++; } } @@ -655,27 +655,27 @@ private void writeMeasurementSchemas(DataOutputStream stream) throws IOException } private void writeTimes(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(BytesUtils.boolToByte(getTimestamps() != null), stream); - if (getTimestamps() != null) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(timestamps != null), stream); + if (timestamps != null) { for (int i = 0; i < rowSize; i++) { - ReadWriteIOUtils.write(getTimestamps()[i], stream); + ReadWriteIOUtils.write(timestamps[i], stream); } } } /** Serialize {@link BitMap}s */ private void writeBitMaps(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(BytesUtils.boolToByte(getBitMaps() != null), stream); - if (getBitMaps() != null) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(bitMaps != null), stream); + if (bitMaps != null) { int size = (schemas == null ? 0 : schemas.size()); for (int i = 0; i < size; i++) { - if (getBitMaps()[i] == null || getBitMaps()[i].isAllUnmarked(rowSize)) { + if (bitMaps[i] == null || bitMaps[i].isAllUnmarked(rowSize)) { ReadWriteIOUtils.write(BytesUtils.boolToByte(false), stream); } else { ReadWriteIOUtils.write(BytesUtils.boolToByte(true), stream); ReadWriteIOUtils.write(rowSize, stream); ReadWriteIOUtils.write( - new Binary(getBitMaps()[i].getTruncatedByteArray(rowSize)), stream); + new Binary(bitMaps[i].getTruncatedByteArray(rowSize)), stream); } } } @@ -683,11 +683,11 @@ private void writeBitMaps(DataOutputStream stream) throws IOException { /** Serialize values */ private void writeValues(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(BytesUtils.boolToByte(getValues() != null), stream); - if (getValues() != null) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(values != null), stream); + if (values != null) { int size = (schemas == null ? 0 : schemas.size()); for (int i = 0; i < size; i++) { - serializeColumn(schemas.get(i).getType(), getValues()[i], stream, columnCategories.get(i)); + serializeColumn(schemas.get(i).getType(), values[i], stream, columnCategories.get(i)); } } } @@ -943,36 +943,36 @@ public boolean equals(Object o) { // assert timestamps and bitmaps int columns = (schemas == null ? 0 : schemas.size()); - if (!isTimestampsEqual(this.getTimestamps(), that.getTimestamps(), rowSize) - || !isBitMapsEqual(this.getBitMaps(), that.getBitMaps(), columns)) { + if (!isTimestampsEqual(this.timestamps, that.timestamps, rowSize) + || !isBitMapsEqual(this.bitMaps, that.bitMaps, columns)) { return false; } // assert values - Object[] thatValues = that.getValues(); - if (thatValues == getValues()) { + Object[] thatValues = that.values; + if (thatValues == values) { return true; } - if (thatValues == null || getValues() == null) { + if (thatValues == null || values == null) { return false; } - if (thatValues.length != getValues().length) { + if (thatValues.length != values.length) { return false; } - for (int i = 0, n = getValues().length; i < n; i++) { - if (thatValues[i] == getValues()[i]) { + for (int i = 0, n = values.length; i < n; i++) { + if (thatValues[i] == values[i]) { continue; } - if (thatValues[i] == null || getValues()[i] == null) { + if (thatValues[i] == null || values[i] == null) { return false; } - if (!thatValues[i].getClass().equals(getValues()[i].getClass())) { + if (!thatValues[i].getClass().equals(values[i].getClass())) { return false; } switch (schemas.get(i).getType()) { case INT32: - int[] thisIntValues = (int[]) getValues()[i]; + int[] thisIntValues = (int[]) values[i]; int[] thatIntValues = (int[]) thatValues[i]; if (thisIntValues.length < rowSize || thatIntValues.length < rowSize) { return false; @@ -984,7 +984,7 @@ public boolean equals(Object o) { } break; case DATE: - LocalDate[] thisDateValues = (LocalDate[]) getValues()[i]; + LocalDate[] thisDateValues = (LocalDate[]) values[i]; LocalDate[] thatDateValues = (LocalDate[]) thatValues[i]; if (thisDateValues.length < rowSize || thatDateValues.length < rowSize) { return false; @@ -997,7 +997,7 @@ public boolean equals(Object o) { break; case INT64: case TIMESTAMP: - long[] thisLongValues = (long[]) getValues()[i]; + long[] thisLongValues = (long[]) values[i]; long[] thatLongValues = (long[]) thatValues[i]; if (thisLongValues.length < rowSize || thatLongValues.length < rowSize) { return false; @@ -1009,7 +1009,7 @@ public boolean equals(Object o) { } break; case FLOAT: - float[] thisFloatValues = (float[]) getValues()[i]; + float[] thisFloatValues = (float[]) values[i]; float[] thatFloatValues = (float[]) thatValues[i]; if (thisFloatValues.length < rowSize || thatFloatValues.length < rowSize) { return false; @@ -1021,7 +1021,7 @@ public boolean equals(Object o) { } break; case DOUBLE: - double[] thisDoubleValues = (double[]) getValues()[i]; + double[] thisDoubleValues = (double[]) values[i]; double[] thatDoubleValues = (double[]) thatValues[i]; if (thisDoubleValues.length < rowSize || thatDoubleValues.length < rowSize) { return false; @@ -1033,7 +1033,7 @@ public boolean equals(Object o) { } break; case BOOLEAN: - boolean[] thisBooleanValues = (boolean[]) getValues()[i]; + boolean[] thisBooleanValues = (boolean[]) values[i]; boolean[] thatBooleanValues = (boolean[]) thatValues[i]; if (thisBooleanValues.length < rowSize || thatBooleanValues.length < rowSize) { return false; @@ -1047,7 +1047,7 @@ public boolean equals(Object o) { case TEXT: case STRING: case BLOB: - Binary[] thisBinaryValues = (Binary[]) getValues()[i]; + Binary[] thisBinaryValues = (Binary[]) values[i]; Binary[] thatBinaryValues = (Binary[]) thatValues[i]; if (thisBinaryValues.length < rowSize || thatBinaryValues.length < rowSize) { return false; @@ -1126,7 +1126,7 @@ private boolean isBitMapEqual(BitMap thisBitMap, BitMap thatBitMap) { } public boolean isNull(int i, int j) { - return getBitMaps() != null && getBitMaps()[j] != null && getBitMaps()[j].isMarked(i); + return bitMaps != null && bitMaps[j] != null && bitMaps[j].isMarked(i); } /** @@ -1142,20 +1142,20 @@ public Object getValue(int i, int j) { case BLOB: case TEXT: case STRING: - return ((Binary[]) getValues()[j])[i]; + return ((Binary[]) values[j])[i]; case INT32: - return ((int[]) getValues()[j])[i]; + return ((int[]) values[j])[i]; case FLOAT: - return ((float[]) getValues()[j])[i]; + return ((float[]) values[j])[i]; case DOUBLE: - return ((double[]) getValues()[j])[i]; + return ((double[]) values[j])[i]; case BOOLEAN: - return ((boolean[]) getValues()[j])[i]; + return ((boolean[]) values[j])[i]; case INT64: case TIMESTAMP: - return ((long[]) getValues()[j])[i]; + return ((long[]) values[j])[i]; case DATE: - return ((LocalDate[]) getValues()[j])[i]; + return ((LocalDate[]) values[j])[i]; default: throw new IllegalArgumentException("Unsupported type: " + schemas.get(j).getType()); } @@ -1218,7 +1218,6 @@ public void setValues(Object[] values) { this.values = values; } - /** Each {@link BitMap} represents the existence of each value in the current column. */ public BitMap[] getBitMaps() { return bitMaps; } From d880d4f9ebf3e2ef5168e2fa344ba8dbfd9a8f1b Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 6 Jan 2025 18:01:16 +0800 Subject: [PATCH 4/7] modify tablet --- .../apache/tsfile/write/record/Tablet.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index fc476f373..318374d18 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -229,9 +229,9 @@ public Tablet( this.insertTargetName = deviceId; this.schemas = schemas; setColumnCategories(ColumnCategory.nCopy(ColumnCategory.FIELD, schemas.size())); - this.setTimestamps(timestamps); - this.setValues(values); - this.setBitMaps(bitMaps); + this.timestamps = timestamps; + this.values = values; + this.bitMaps = bitMaps; this.maxRowNumber = maxRowNumber; // rowSize == maxRowNumber in this case this.rowSize = maxRowNumber; @@ -251,9 +251,9 @@ public Tablet( this.insertTargetName = tableName; this.schemas = schemas; setColumnCategories(columnCategories); - this.setTimestamps(timestamps); - this.setValues(values); - this.setBitMaps(bitMaps); + this.timestamps = timestamps; + this.values = values; + this.bitMaps = bitMaps; this.maxRowNumber = maxRowNumber; // rowSize == maxRowNumber in this case this.rowSize = maxRowNumber; @@ -278,7 +278,7 @@ public void setSchemas(List schemas) { } public void initBitMaps() { - this.setBitMaps(new BitMap[schemas.size()]); + this.bitMaps = new BitMap[schemas.size()]; for (int column = 0; column < schemas.size(); column++) { BitMap bitMap = new BitMap(getMaxRowNumber()); this.bitMaps[column] = bitMap; @@ -567,13 +567,13 @@ public void reset() { private void createColumns() { // create timestamp column - setTimestamps(new long[maxRowNumber]); + timestamps = new long[maxRowNumber]; // calculate total value column size int valueColumnsSize = schemas.size(); // value column - setValues(new Object[valueColumnsSize]); + values = new Object[valueColumnsSize]; int columnIndex = 0; for (int i = 0; i < schemas.size(); i++) { IMeasurementSchema schema = schemas.get(i); @@ -674,8 +674,7 @@ private void writeBitMaps(DataOutputStream stream) throws IOException { } else { ReadWriteIOUtils.write(BytesUtils.boolToByte(true), stream); ReadWriteIOUtils.write(rowSize, stream); - ReadWriteIOUtils.write( - new Binary(bitMaps[i].getTruncatedByteArray(rowSize)), stream); + ReadWriteIOUtils.write(new Binary(bitMaps[i].getTruncatedByteArray(rowSize)), stream); } } } From 1305477d4c6bb7d2f4c2e82575cdfbf1ca30791f Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 6 Jan 2025 18:02:45 +0800 Subject: [PATCH 5/7] revert pom --- java/tsfile/pom.xml | 4 +- java/tsfile/pom.xml.versionsBackup | 293 ----------------------------- 2 files changed, 2 insertions(+), 295 deletions(-) delete mode 100644 java/tsfile/pom.xml.versionsBackup diff --git a/java/tsfile/pom.xml b/java/tsfile/pom.xml index 24e3f30e3..0a2441b87 100644 --- a/java/tsfile/pom.xml +++ b/java/tsfile/pom.xml @@ -24,7 +24,7 @@ org.apache.tsfile tsfile-java - 1.2.0-tablet + 2.0.0-SNAPSHOT tsfile TsFile: Java: TsFile @@ -38,7 +38,7 @@ org.apache.tsfile common - 1.2.0-tablet + 2.0.0-SNAPSHOT com.github.luben diff --git a/java/tsfile/pom.xml.versionsBackup b/java/tsfile/pom.xml.versionsBackup deleted file mode 100644 index 0a2441b87..000000000 --- a/java/tsfile/pom.xml.versionsBackup +++ /dev/null @@ -1,293 +0,0 @@ - - - - 4.0.0 - - org.apache.tsfile - tsfile-java - 2.0.0-SNAPSHOT - - tsfile - TsFile: Java: TsFile - A columnar file format designed for time-series data - - ${tsfile.test.skip} - false - ${tsfile.test.skip} - - - - org.apache.tsfile - common - 2.0.0-SNAPSHOT - - - com.github.luben - zstd-jni - - - org.xerial.snappy - snappy-java - - - commons-io - commons-io - - - org.apache.commons - commons-lang3 - - - org.lz4 - lz4-java - - - org.tukaani - xz - - - org.slf4j - slf4j-api - - - org.antlr - antlr4-runtime - - - junit - junit - test - - - org.mockito - mockito-core - test - - - ch.qos.logback - logback-classic - test - - - com.google.code.gson - gson - test - - - - - - org.antlr - antlr4-maven-plugin - - - - antlr4 - - - false - true - src/main/antlr4/org/apache/tsfile/parser - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-source - - add-source - - generate-sources - - - ${project.build.directory}/generated-sources/antlr4 - ${project.build.directory}/generated-sources/freemarker - - - - - - - org.apache.maven.plugins - maven-clean-plugin - - - - ${basedir}/gen - - - ${basedir}/src - - **/*.tokens - - - - - - - - org.apache.felix - maven-bundle-plugin - - true - - <_include>-bnd.bnd - <_removeheaders>Bnd-LastModified,Built-By - dependencies - - - - - bundle-manifest - - manifest - - process-classes - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - ${project.build.outputDirectory}/META-INF/MANIFEST.MF - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${tsfile.ut.skip} - false - random - - - - org.apache.maven.plugins - maven-failsafe-plugin - - ${tsfile.test.skip} - ${tsfile.it.skip} - false - random - - - - run-integration-tests - - integration-test - verify - - integration-test - - - - - - org.apache.drill.tools - drill-fmpp-maven-plugin - - - generate-fmpp - - generate - - generate-sources - - ${project.basedir}/src/main/codegen/config.fmpp - ${project.build.directory}/generated-sources/freemarker - ${project.basedir}/src/main/codegen/templates/ - - - - - - - - - skipTsfileTests - - - skipTests - true - - - - true - true - true - - - - skipUT_TsFileTests - - - skipUTs - true - - - - true - - - - get-jar-with-dependencies - - - - org.apache.maven.plugins - maven-assembly-plugin - - - jar-with-dependencies - - - - - make-assembly - - - single - - - package - - - - - - - - From f2fdd9b3f638ee1b4b74e5e242793cb590cc4e95 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 7 Jan 2025 10:06:29 +0800 Subject: [PATCH 6/7] add ut --- .../read/query/dataset/AbstractResultSet.java | 2 +- .../tsfile/write/TsFileWriteApiTest.java | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java index 5b07dd46d..103527908 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java @@ -168,7 +168,7 @@ protected Field getNonNullField(int columnIndex) { } protected Field getField(int columnIndex) { - if (columnIndex > this.columnNameToColumnIndexMap.size()) { + if (columnIndex > this.columnNameToColumnIndexMap.size() || columnIndex <= 0) { throw new IndexOutOfBoundsException("column index " + columnIndex + " out of bound"); } Field field; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java index 5902172cb..160866e4c 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java @@ -20,6 +20,7 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.read.ReadProcessException; import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; @@ -36,6 +37,9 @@ import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.read.v4.ITsFileReader; +import org.apache.tsfile.read.v4.TsFileReaderBuilder; import org.apache.tsfile.utils.TsFileGeneratorUtils; import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; import org.apache.tsfile.write.chunk.ChunkWriterImpl; @@ -45,6 +49,8 @@ import org.apache.tsfile.write.record.datapoint.StringDataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; import org.apache.tsfile.write.writer.TsFileIOWriter; import org.junit.After; @@ -927,4 +933,42 @@ public void writeTableTsFileWithUpperCaseColumns() throws IOException, WriteProc Assert.assertTrue(reader.getAllMeasurements().containsKey("measurementcolumn")); } } + + @Test + public void writeAllNullValueTablet() + throws IOException, WriteProcessException, ReadProcessException { + setEnv(100 * 1024 * 1024, 10 * 1024); + Tablet tablet = + new Tablet( + "table1", + Arrays.asList("tag1", "field1"), + Arrays.asList(TSDataType.STRING, TSDataType.BOOLEAN), + Arrays.asList(Tablet.ColumnCategory.TAG, Tablet.ColumnCategory.FIELD)); + tablet.addTimestamp(0, 0); + tablet.addTimestamp(1, 1); + TableSchema tableSchema = + new TableSchema( + "Table1", + Arrays.asList( + new ColumnSchema("tag1", TSDataType.STRING, Tablet.ColumnCategory.TAG), + new ColumnSchema("field1", TSDataType.BOOLEAN, Tablet.ColumnCategory.FIELD))); + Assert.assertEquals("table1", tableSchema.getTableName()); + try (ITsFileWriter writer = + new TsFileWriterBuilder().file(f).tableSchema(tableSchema).build()) { + writer.write(tablet); + } + try (ITsFileReader reader = new TsFileReaderBuilder().file(f).build(); + ResultSet resultSet = + reader.query( + "table1", Arrays.asList("tag1", "field1"), Long.MIN_VALUE, Long.MAX_VALUE)) { + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(0, resultSet.getLong(1)); + Assert.assertTrue(resultSet.isNull(2)); + Assert.assertTrue(resultSet.isNull(3)); + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(1, resultSet.getLong(1)); + Assert.assertTrue(resultSet.isNull(2)); + Assert.assertTrue(resultSet.isNull(3)); + } + } } From 38bf8d19a54ef5f430e1a824a38aedcb50d6a832 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 7 Jan 2025 10:19:42 +0800 Subject: [PATCH 7/7] modify exception message --- .../apache/tsfile/write/record/Tablet.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 318374d18..93e3f1e9a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -411,6 +411,10 @@ public void addValue(int rowIndex, String measurement, int val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, int val) { + if (!(values[columnIndex] instanceof int[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not INT32"); + } final int[] sensor = (int[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -424,6 +428,10 @@ public void addValue(int rowIndex, String measurement, long val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, long val) { + if (!(values[columnIndex] instanceof long[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not INT64/TIMESTAMP"); + } final long[] sensor = (long[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -437,6 +445,10 @@ public void addValue(int rowIndex, String measurement, float val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, float val) { + if (!(values[columnIndex] instanceof float[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not FLOAT"); + } final float[] sensor = (float[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -450,6 +462,10 @@ public void addValue(int rowIndex, String measurement, double val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, double val) { + if (!(values[columnIndex] instanceof double[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not DOUBLE"); + } final double[] sensor = (double[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -463,6 +479,10 @@ public void addValue(int rowIndex, String measurement, boolean val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, boolean val) { + if (!(values[columnIndex] instanceof boolean[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not BOOLEAN"); + } final boolean[] sensor = (boolean[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -476,6 +496,10 @@ public void addValue(int rowIndex, String measurement, String val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, String val) { + if (!(values[columnIndex] instanceof Binary[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not TEXT/STRING/BLOB"); + } final Binary[] sensor = (Binary[]) values[columnIndex]; sensor[rowIndex] = new Binary(val, TSFileConfig.STRING_CHARSET); updateBitMap(rowIndex, columnIndex, false); @@ -489,6 +513,10 @@ public void addValue(int rowIndex, String measurement, byte[] val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, byte[] val) { + if (!(values[columnIndex] instanceof Binary[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not TEXT/STRING/BLOB"); + } final Binary[] sensor = (Binary[]) values[columnIndex]; sensor[rowIndex] = new Binary(val); updateBitMap(rowIndex, columnIndex, false); @@ -502,6 +530,10 @@ public void addValue(int rowIndex, String measurement, LocalDate val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, LocalDate val) { + if (!(values[columnIndex] instanceof LocalDate[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not DATE"); + } final LocalDate[] sensor = (LocalDate[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false);