diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java index 784df63ba..ca13fdb36 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java @@ -133,8 +133,6 @@ private static void writeNonAlignedWithTablet(TsFileWriter tsFileWriter) measurementSchemas.add(new MeasurementSchema(SENSOR_1, TSDataType.INT64, TSEncoding.RLE)); measurementSchemas.add(new MeasurementSchema(SENSOR_2, TSDataType.INT64, TSEncoding.RLE)); Tablet tablet = new Tablet(DEVICE_2, measurementSchemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; int rowNum = 100; int sensorNum = measurementSchemas.size(); long timestamp = 1; @@ -143,8 +141,7 @@ private static void writeNonAlignedWithTablet(TsFileWriter tsFileWriter) int row = tablet.getRowSize(); tablet.addTimestamp(row, timestamp++); for (int i = 0; i < sensorNum; i++) { - long[] sensor = (long[]) values[i]; - sensor[row] = value; + tablet.addValue(row, i, value); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java index bb0293a52..05446ce8c 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java @@ -88,12 +88,11 @@ private static void writeWithTablet( long startValue) throws IOException, WriteProcessException { Tablet tablet = new Tablet(deviceId, schemas); - long[] timestamps = tablet.timestamps; long sensorNum = schemas.size(); for (long r = 0; r < rowNum; r++, startValue++) { int row = tablet.getRowSize(); - timestamps[row] = startTime++; + tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum; i++) { tablet.addValue( schemas.get(i).getMeasurementName(), diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java index 5b07dd46d..103527908 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java @@ -168,7 +168,7 @@ protected Field getNonNullField(int columnIndex) { } protected Field getField(int columnIndex) { - if (columnIndex > this.columnNameToColumnIndexMap.size()) { + if (columnIndex > this.columnNameToColumnIndexMap.size() || columnIndex <= 0) { throw new IndexOutOfBoundsException("column index " + columnIndex + " out of bound"); } Field field; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java index 6b87f6e17..e4b8abcdf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java @@ -117,16 +117,13 @@ public static void writeWithTablet( boolean isAligned) throws IOException, WriteProcessException { Tablet tablet = new Tablet(deviceId, schemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; long sensorNum = schemas.size(); for (long r = 0; r < rowNum; r++, startValue++) { int row = tablet.getRowSize(); tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum; i++) { - long[] sensor = (long[]) values[i]; - sensor[row] = startValue; + tablet.addValue(row, i, startValue); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java index 0b53b75ae..c051a702d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java @@ -233,7 +233,7 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) // TODO: changing to a column-first style by calculating the remaining page space of each // column firsts for (int row = startRowIndex; row < endRowIndex; row++) { - long time = tablet.timestamps[row]; + long time = tablet.getTimestamps()[row]; checkIsHistoryData(time); for (int columnIndex = 0; columnIndex < tablet.getSchemas().size(); columnIndex++) { if (tablet.getColumnTypes() != null @@ -242,18 +242,19 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) } boolean isNull = - tablet.bitMaps != null - && tablet.bitMaps[columnIndex] != null - && tablet.bitMaps[columnIndex].isMarked(row); + tablet.getBitMaps() != null + && tablet.getBitMaps()[columnIndex] != null + && tablet.getBitMaps()[columnIndex].isMarked(row); // check isNull by bitMap in tablet ValueChunkWriter valueChunkWriter = tryToAddSeriesWriterInternal(measurementSchemas.get(columnIndex)); switch (measurementSchemas.get(columnIndex).getType()) { case BOOLEAN: - valueChunkWriter.write(time, ((boolean[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write( + time, ((boolean[]) tablet.getValues()[columnIndex])[row], isNull); break; case INT32: - valueChunkWriter.write(time, ((int[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((int[]) tablet.getValues()[columnIndex])[row], isNull); break; case DATE: valueChunkWriter.write( @@ -261,23 +262,23 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) isNull ? 0 : DateUtils.parseDateExpressionToInt( - ((LocalDate[]) tablet.values[columnIndex])[row]), + ((LocalDate[]) tablet.getValues()[columnIndex])[row]), isNull); break; case INT64: case TIMESTAMP: - valueChunkWriter.write(time, ((long[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((long[]) tablet.getValues()[columnIndex])[row], isNull); break; case FLOAT: - valueChunkWriter.write(time, ((float[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((float[]) tablet.getValues()[columnIndex])[row], isNull); break; case DOUBLE: - valueChunkWriter.write(time, ((double[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((double[]) tablet.getValues()[columnIndex])[row], isNull); break; case TEXT: case BLOB: case STRING: - valueChunkWriter.write(time, ((Binary[]) tablet.values[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((Binary[]) tablet.getValues()[columnIndex])[row], isNull); break; default: throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java index 3f4d8aa0f..cc20b2520 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java @@ -121,42 +121,51 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) pointCount = 0; for (int row = startRowIndex; row < endRowIndex; row++) { // check isNull in tablet - if (tablet.bitMaps != null - && tablet.bitMaps[column] != null - && tablet.bitMaps[column].isMarked(row)) { + if (tablet.getBitMaps() != null + && tablet.getBitMaps()[column] != null + && tablet.getBitMaps()[column].isMarked(row)) { continue; } - long time = tablet.timestamps[row]; + long time = tablet.getTimestamps()[row]; checkIsHistoryData(measurementId, time); pointCount++; switch (tsDataType) { case INT32: - chunkWriters.get(measurementId).write(time, ((int[]) tablet.values[column])[row]); + chunkWriters.get(measurementId).write(time, ((int[]) tablet.getValues()[column])[row]); break; case DATE: chunkWriters .get(measurementId) .write( time, - DateUtils.parseDateExpressionToInt(((LocalDate[]) tablet.values[column])[row])); + DateUtils.parseDateExpressionToInt( + ((LocalDate[]) tablet.getValues()[column])[row])); break; case INT64: case TIMESTAMP: - chunkWriters.get(measurementId).write(time, ((long[]) tablet.values[column])[row]); + chunkWriters.get(measurementId).write(time, ((long[]) tablet.getValues()[column])[row]); break; case FLOAT: - chunkWriters.get(measurementId).write(time, ((float[]) tablet.values[column])[row]); + chunkWriters + .get(measurementId) + .write(time, ((float[]) tablet.getValues()[column])[row]); break; case DOUBLE: - chunkWriters.get(measurementId).write(time, ((double[]) tablet.values[column])[row]); + chunkWriters + .get(measurementId) + .write(time, ((double[]) tablet.getValues()[column])[row]); break; case BOOLEAN: - chunkWriters.get(measurementId).write(time, ((boolean[]) tablet.values[column])[row]); + chunkWriters + .get(measurementId) + .write(time, ((boolean[]) tablet.getValues()[column])[row]); break; case TEXT: case BLOB: case STRING: - chunkWriters.get(measurementId).write(time, ((Binary[]) tablet.values[column])[row]); + chunkWriters + .get(measurementId) + .write(time, ((Binary[]) tablet.getValues()[column])[row]); break; default: throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 2580a2a47..93e3f1e9a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -80,14 +80,11 @@ public class Tablet { /** MeasurementId->indexOf({@link MeasurementSchema}) */ private final Map measurementIndex; - /** Timestamps in this {@link Tablet} */ - public long[] timestamps; + private long[] timestamps; - /** Each object is a primitive type array, which represents values of one measurement */ - public Object[] values; + private Object[] values; - /** Each {@link BitMap} represents the existence of each value in the current column. */ - public BitMap[] bitMaps; + private BitMap[] bitMaps; /** * For compatibility with the usage of directly modifying Tablet content through public fields. @@ -292,6 +289,7 @@ public void initBitMaps() { public void addTimestamp(int rowIndex, long timestamp) { timestamps[rowIndex] = timestamp; this.rowSize = Math.max(this.rowSize, rowIndex + 1); + initBitMapsWithApiUsage(); } public void addValue(final String measurementId, final int rowIndex, final Object value) { @@ -413,6 +411,10 @@ public void addValue(int rowIndex, String measurement, int val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, int val) { + if (!(values[columnIndex] instanceof int[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not INT32"); + } final int[] sensor = (int[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -426,6 +428,10 @@ public void addValue(int rowIndex, String measurement, long val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, long val) { + if (!(values[columnIndex] instanceof long[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not INT64/TIMESTAMP"); + } final long[] sensor = (long[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -439,6 +445,10 @@ public void addValue(int rowIndex, String measurement, float val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, float val) { + if (!(values[columnIndex] instanceof float[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not FLOAT"); + } final float[] sensor = (float[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -452,6 +462,10 @@ public void addValue(int rowIndex, String measurement, double val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, double val) { + if (!(values[columnIndex] instanceof double[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not DOUBLE"); + } final double[] sensor = (double[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -465,6 +479,10 @@ public void addValue(int rowIndex, String measurement, boolean val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, boolean val) { + if (!(values[columnIndex] instanceof boolean[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not BOOLEAN"); + } final boolean[] sensor = (boolean[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -478,6 +496,10 @@ public void addValue(int rowIndex, String measurement, String val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, String val) { + if (!(values[columnIndex] instanceof Binary[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not TEXT/STRING/BLOB"); + } final Binary[] sensor = (Binary[]) values[columnIndex]; sensor[rowIndex] = new Binary(val, TSFileConfig.STRING_CHARSET); updateBitMap(rowIndex, columnIndex, false); @@ -491,6 +513,10 @@ public void addValue(int rowIndex, String measurement, byte[] val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, byte[] val) { + if (!(values[columnIndex] instanceof Binary[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not TEXT/STRING/BLOB"); + } final Binary[] sensor = (Binary[]) values[columnIndex]; sensor[rowIndex] = new Binary(val); updateBitMap(rowIndex, columnIndex, false); @@ -504,6 +530,10 @@ public void addValue(int rowIndex, String measurement, LocalDate val) { @TsFileApi public void addValue(int rowIndex, int columnIndex, LocalDate val) { + if (!(values[columnIndex] instanceof LocalDate[])) { + throw new IllegalArgumentException( + "The data type of column index " + columnIndex + " is not DATE"); + } final LocalDate[] sensor = (LocalDate[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -521,6 +551,15 @@ private int getColumnIndexByMeasurement(String measurement) { } private void updateBitMap(int rowIndex, int columnIndex, boolean mark) { + initBitMapsWithApiUsage(); + if (mark) { + bitMaps[columnIndex].mark(rowIndex); + } else { + bitMaps[columnIndex].unmark(rowIndex); + } + } + + private void initBitMapsWithApiUsage() { if (bitMaps == null) { initBitMaps(); } @@ -530,11 +569,6 @@ private void updateBitMap(int rowIndex, int columnIndex, boolean mark) { bitMap.markAll(); } } - if (mark) { - bitMaps[columnIndex].mark(rowIndex); - } else { - bitMaps[columnIndex].unmark(rowIndex); - } } public List getSchemas() { @@ -1195,6 +1229,34 @@ public void setRowSize(int rowSize) { this.rowSize = rowSize; } + public long getTimestamp(int i) { + return timestamps[i]; + } + + public long[] getTimestamps() { + return timestamps; + } + + public void setTimestamps(long[] timestamps) { + this.timestamps = timestamps; + } + + public Object[] getValues() { + return values; + } + + public void setValues(Object[] values) { + this.values = values; + } + + public BitMap[] getBitMaps() { + return bitMaps; + } + + public void setBitMaps(BitMap[] bitMaps) { + this.bitMaps = bitMaps; + } + public enum ColumnCategory { TAG, FIELD, @@ -1247,4 +1309,13 @@ public void setTableName(String tableName) { public List getColumnTypes() { return columnCategories; } + + public boolean isSorted() { + for (int i = 1; i < rowSize; i++) { + if (timestamps[i] < timestamps[i - 1]) { + return false; + } + } + return true; + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java b/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java index 1cef0a40c..9f454d4e6 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java @@ -274,15 +274,13 @@ private Tablet initTreeTablet() { private void fillTreeTablet(Tablet tablet, int tableNum, int deviceNum, int tabletNum) { tablet.setDeviceId(genTreeDeviceId(tableNum, deviceNum).toString()); for (int i = 0; i < measurementSchemaCnt; i++) { - long[] values = (long[]) tablet.values[i]; for (int valNum = 0; valNum < pointPerSeries; valNum++) { - values[valNum] = (long) tabletNum * pointPerSeries + valNum; + tablet.addValue(valNum, i, tabletNum * pointPerSeries + valNum); } } for (int valNum = 0; valNum < pointPerSeries; valNum++) { - tablet.timestamps[valNum] = (long) tabletNum * pointPerSeries + valNum; + tablet.addTimestamp(valNum, tabletNum * pointPerSeries + valNum); } - tablet.setRowSize(pointPerSeries); } private Tablet initTableTablet() { @@ -303,21 +301,18 @@ private void fillTableTablet(Tablet tablet, int tableNum, int deviceNum, int tab IDeviceID deviceID = genTableDeviceId(tableNum, deviceNum); tablet.setTableName(deviceID.segment(0).toString()); for (int i = 0; i < idSchemaCnt; i++) { - String[] strings = ((String[]) tablet.values[i]); for (int rowNum = 0; rowNum < pointPerSeries; rowNum++) { - strings[rowNum] = deviceID.segment(i + 1).toString(); + tablet.addValue(rowNum, i, deviceID.segment(i + 1).toString()); } } for (int i = 0; i < measurementSchemaCnt; i++) { - long[] values = (long[]) tablet.values[i + idSchemaCnt]; for (int valNum = 0; valNum < pointPerSeries; valNum++) { - values[valNum] = (long) tabletNum * pointPerSeries + valNum; + tablet.addValue(valNum, i + idSchemaCnt, tabletNum * pointPerSeries + valNum); } } for (int valNum = 0; valNum < pointPerSeries; valNum++) { - tablet.timestamps[valNum] = (long) tabletNum * pointPerSeries + valNum; + tablet.addTimestamp(valNum, tabletNum * pointPerSeries + valNum); } - tablet.setRowSize(pointPerSeries); } private void registerTree(TsFileWriter writer) throws WriteProcessException { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java index 3ff19a1b7..8a9351a0f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java @@ -66,8 +66,6 @@ public void testUsingDefaultSchemaTemplate() throws IOException, WriteProcessExc writer.registerDevice("d1", "defaultTemplate"); Tablet tablet = new Tablet("d1", schemaList); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; long timestamp = 1; long value = 1L; @@ -76,8 +74,7 @@ public void testUsingDefaultSchemaTemplate() throws IOException, WriteProcessExc int row = tablet.getRowSize(); tablet.addTimestamp(row, timestamp++); for (int i = 0; i < 2; i++) { - long[] sensor = (long[]) values[i]; - sensor[row] = value; + tablet.addValue(row, i, value); } // write Tablet to TsFile if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java index 32cbb6e3e..799082704 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java @@ -477,16 +477,13 @@ private void generateFile( // add measurements into TSFileWriter // construct the tablet Tablet tablet = new Tablet(device.toString(), tabletSchema); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; long timestamp = 1; long value = 1000000L; for (int r = 0; r < rowNum; r++, value++) { int row = tablet.getRowSize(); tablet.addTimestamp(row, timestamp++); for (int j = 0; j < measurementNum; j++) { - long[] sensor = (long[]) values[j]; - sensor[row] = value; + tablet.addValue(row, j, value); } // write Tablet to TsFile if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java index 0a4d92f17..160866e4c 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java @@ -18,9 +18,9 @@ */ package org.apache.tsfile.write; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.read.ReadProcessException; import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; @@ -37,7 +37,9 @@ import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.read.v4.ITsFileReader; +import org.apache.tsfile.read.v4.TsFileReaderBuilder; import org.apache.tsfile.utils.TsFileGeneratorUtils; import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; import org.apache.tsfile.write.chunk.ChunkWriterImpl; @@ -47,6 +49,8 @@ import org.apache.tsfile.write.record.datapoint.StringDataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; import org.apache.tsfile.write.writer.TsFileIOWriter; import org.junit.After; @@ -404,9 +408,6 @@ public void writeNonAlignedWithTabletWithNullValue() { tsFileWriter.registerTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = 0; for (long r = 0; r < 10000; r++) { @@ -414,17 +415,15 @@ public void writeNonAlignedWithTabletWithNullValue() { tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - Binary[] textSensor = (Binary[]) values[i]; - textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); + tablet.addValue(row, i, "testString........."); } if (r > 1000) { - tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; - dateSensor[row] = LocalDate.of(2024, 4, 1); + tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { @@ -457,8 +456,6 @@ public void writeNonAlignedWithTabletWithNegativeTimestamps() { tsFileWriter.registerTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = -100; @@ -467,17 +464,15 @@ public void writeNonAlignedWithTabletWithNegativeTimestamps() { tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - Binary[] textSensor = (Binary[]) values[i]; - textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); + tablet.addValue(row, i, "testString........."); } if (r > 1000) { - tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; - dateSensor[row] = LocalDate.of(2024, 4, 1); + tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { @@ -510,28 +505,23 @@ public void writeAlignedWithTabletWithNullValue() { tsFileWriter.registerAlignedTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = 0; for (long r = 0; r < 10000; r++) { int row = tablet.getRowSize(); tablet.addTimestamp(row, startTime++); - timestamps[row] = startTime++; for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - Binary[] textSensor = (Binary[]) values[i]; - textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); + tablet.addValue(row, i, "testString........."); } if (r > 1000) { - tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; - dateSensor[row] = LocalDate.of(2024, 4, 1); + tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { @@ -589,8 +579,6 @@ public void writeDataToTabletsWithNegativeTimestamps() { tsFileWriter.registerAlignedTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = -1000; @@ -599,17 +587,15 @@ public void writeDataToTabletsWithNegativeTimestamps() { tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - Binary[] textSensor = (Binary[]) values[i]; - textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); + tablet.addValue(row, i, "testString........."); } if (r > 1000) { - tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; - dateSensor[row] = LocalDate.of(2024, 4, 1); + tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { @@ -947,4 +933,42 @@ public void writeTableTsFileWithUpperCaseColumns() throws IOException, WriteProc Assert.assertTrue(reader.getAllMeasurements().containsKey("measurementcolumn")); } } + + @Test + public void writeAllNullValueTablet() + throws IOException, WriteProcessException, ReadProcessException { + setEnv(100 * 1024 * 1024, 10 * 1024); + Tablet tablet = + new Tablet( + "table1", + Arrays.asList("tag1", "field1"), + Arrays.asList(TSDataType.STRING, TSDataType.BOOLEAN), + Arrays.asList(Tablet.ColumnCategory.TAG, Tablet.ColumnCategory.FIELD)); + tablet.addTimestamp(0, 0); + tablet.addTimestamp(1, 1); + TableSchema tableSchema = + new TableSchema( + "Table1", + Arrays.asList( + new ColumnSchema("tag1", TSDataType.STRING, Tablet.ColumnCategory.TAG), + new ColumnSchema("field1", TSDataType.BOOLEAN, Tablet.ColumnCategory.FIELD))); + Assert.assertEquals("table1", tableSchema.getTableName()); + try (ITsFileWriter writer = + new TsFileWriterBuilder().file(f).tableSchema(tableSchema).build()) { + writer.write(tablet); + } + try (ITsFileReader reader = new TsFileReaderBuilder().file(f).build(); + ResultSet resultSet = + reader.query( + "table1", Arrays.asList("tag1", "field1"), Long.MIN_VALUE, Long.MAX_VALUE)) { + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(0, resultSet.getLong(1)); + Assert.assertTrue(resultSet.isNull(2)); + Assert.assertTrue(resultSet.isNull(3)); + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(1, resultSet.getLong(1)); + Assert.assertTrue(resultSet.isNull(2)); + Assert.assertTrue(resultSet.isNull(3)); + } + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java index 676aecfce..baf979a72 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java @@ -244,10 +244,9 @@ public void writeTablet() throws IOException, WriteProcessException { "s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY), new MeasurementSchema( "s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY))); - tablet.timestamps[0] = 10000; - ((float[]) tablet.values[0])[0] = 5.0f; - ((int[]) tablet.values[1])[0] = 5; - tablet.setRowSize(1); + tablet.addTimestamp(0, 10000); + tablet.addValue(0, 0, 5.0f); + tablet.addValue(0, 1, 5); writer.writeTree(tablet); closeFile(); readOneRow(); @@ -263,9 +262,9 @@ public void writeTablet2() throws IOException, WriteProcessException { "s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY), new MeasurementSchema( "s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY))); - tablet.timestamps[0] = 10000; - ((float[]) tablet.values[0])[0] = 5.0f; - tablet.setRowSize(1); + tablet.addTimestamp(0, 10000); + tablet.addValue(0, 0, 5.0f); + tablet.addValue(0, 1, 0); writer.writeTree(tablet); closeFile(); // in this case, the value of s2 = 0 at time 10000. diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java index a73e22f9f..07aedad4f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java @@ -66,21 +66,21 @@ public void testAddValue() { Assert.assertFalse((Boolean) tablet.getValue(1, 0)); Assert.assertTrue((Boolean) tablet.getValue(1, 1)); Assert.assertTrue((Boolean) tablet.getValue(2, 0)); - Assert.assertFalse(tablet.bitMaps[0].isMarked(0)); - Assert.assertFalse(tablet.bitMaps[0].isMarked(1)); - Assert.assertFalse(tablet.bitMaps[0].isMarked(2)); - Assert.assertFalse(tablet.bitMaps[1].isMarked(0)); - Assert.assertFalse(tablet.bitMaps[1].isMarked(1)); - Assert.assertTrue(tablet.bitMaps[1].isMarked(2)); + Assert.assertFalse(tablet.getBitMaps()[0].isMarked(0)); + Assert.assertFalse(tablet.getBitMaps()[0].isMarked(1)); + Assert.assertFalse(tablet.getBitMaps()[0].isMarked(2)); + Assert.assertFalse(tablet.getBitMaps()[1].isMarked(0)); + Assert.assertFalse(tablet.getBitMaps()[1].isMarked(1)); + Assert.assertTrue(tablet.getBitMaps()[1].isMarked(2)); tablet.addTimestamp(9, 9); Assert.assertEquals(10, tablet.getRowSize()); tablet.reset(); Assert.assertEquals(0, tablet.getRowSize()); - Assert.assertTrue(tablet.bitMaps[0].isAllMarked()); - Assert.assertTrue(tablet.bitMaps[0].isAllMarked()); - Assert.assertTrue(tablet.bitMaps[0].isAllMarked()); + Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked()); + Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked()); + Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked()); } @Test @@ -163,7 +163,7 @@ public void testSerializationAndDeSerializationWithMoreData() { i, LocalDate.of(2000 + i, i / 100 + 1, i / 100 + 1)); - tablet.bitMaps[i % measurementSchemas.size()].mark(i); + tablet.getBitMaps()[i % measurementSchemas.size()].mark(i); } // Test add null