From fe35c8f6d42fe97033e7712473f1e52bbf4dc95d Mon Sep 17 00:00:00 2001 From: Zhihao Shen Date: Fri, 31 May 2024 11:00:42 +0800 Subject: [PATCH] Make UDTF execution framework batch processing (#12562) * Add `.putFloats()` to insert multiple rows at the same time. * Complete all batch insert method in `BatchData.java`. * Add overload methods with indexes to avoid copying. * Add Java doc for new methods in `BatchData.java`. * Merge two `BatchDataTest.java` files. * Add `.putColumns()` method to `SerializableRowRecordList.java`. * Finish batch insert method prototype in row record list and tv list. * UDTF execution framework can support running really simple UDTF. * Change TVList to list of columns to avoid copying. * Compile DataNode without error. * Format all files by `mvn splotless:apply`. * Optimize import by using IDEA shortcut. * Fix put data into existing list error. * Generate data in terminate method. * Format again using `mvn spotless:apply`. * Fix some bugs and implement RowWindow UDTF. * Basically pass all UDF-related IT. * Add license header to all newly-added files. * Add memory control strategy for binary list. * Add memory control strategy for row list that contains binary field. * Copy subregion to avoid memory leakage. * Update iterator by recording first data index when apply new memory strategy. * Unify `newXXX` series methods name to `construct`. * Basically refactor storage layer. * Add and update copy method to Column based classes. * Rewrite unit test for ElasticSerializableRowList. * Rewrite unit test for SerializableRowList. * Rewrite unit test for SerializableTVList. * Rewrite unit test for ElasticSerializableTVList. * Sync changes between TVList and RowList. * Refine UT of ESRowList and ESTVList with random size strings. * Do not cache internal list in some situation. * Add forgotten files. * Revert `BatchData.java` relevant files. * Remove redundant `timeHeap.clear()` in MultiInputColumnIntermediateLayer. * Add constant folding in BinaryTransformer. * Refactor computational layer. * Replace origin normal column in constant reader with RLEColumn. * Batch sliding time row window in SingleInputColumnSingleReferenceIntermediateLayer. * Batch other row windows in SingleInputColumnSingleReferenceIntermediateLayer. * Add forgotten `TransformUtils.java` file. * Batch all row windows in SingleInputColumnMultiReferenceIntermediateLayer. * Batch all row windows in all intermediate layer. * Remove code with no usage. * Remove code with no usage agaion. * Remove nonexistent files. * Refactor project's structure. * Optimize when there is only one transformer. * Fix losing data bug in benchmarking. * Refactor UDTF executor. * Skip cache update when getting column size. * Discard TsBlock when there is only one UDF. * Fix transformer IT errors. * Remove unnecessary bitmap in ESRowList. * Update TsFile version. * Fix some IT OOM bugs. * Try to fix UT OOM error. * Apply spotless plugin. * Evict data in UDF output storage. * Revert optimization when there is only one UDTF. * Fix losing data error. * Update internal column count when ESList apply new memory control strategy. * Add new types in SerializableRowList. * Apply mvn spotless plugin. * Fix integration-tests errors. * Modify code according to committer's advices.[Part One] * Modify code according to committer's advices.[Part Two] * Modify code according to committer's advices.[Part Three] --- .../java/org/apache/iotdb/udf/api/UDTF.java | 5 + .../org/apache/iotdb/udf/api/access/Row.java | 2 +- .../apache/iotdb/udf/api/utils/RowImpl.java | 4 +- .../operator/process/TransformOperator.java | 127 +-- .../builtin/BuiltInScalarFunctionHelper.java | 6 +- .../builtin/helper/CastFunctionHelper.java | 6 +- .../builtin/helper/DiffFunctionHelper.java | 6 +- .../builtin/helper/ReplaceFunctionHelper.java | 6 +- .../builtin/helper/RoundFunctionHelper.java | 6 +- .../helper/SubStringFunctionHelper.java | 6 +- .../visitor/IntermediateLayerVisitor.java | 146 ++-- .../{LayerRowReader.java => LayerReader.java} | 20 +- .../api/LayerRowWindowReader.java | 3 - ...eRowRecordListBackedMultiColumnWindow.java | 6 +- ...rdListBackedMultiColumnWindowIterator.java | 6 +- ...rializableTVListBackedSingleColumnRow.java | 2 +- ...LayerPointReaderBackedSingleColumnRow.java | 99 --- .../dag/adapter/PointCollectorAdaptor.java | 97 +++ .../dag/builder/EvaluationDAGBuilder.java | 12 +- .../dag/input/ConstantInputReader.java | 114 ++- .../dag/input/IUDFInputDataSet.java | 21 +- .../dag/input/QueryDataSetInputLayer.java | 294 ++----- .../dag/input/TsBlockInputDataSet.java | 37 +- .../ConstantIntermediateLayer.java | 17 +- .../dag/intermediate/IntermediateLayer.java | 7 +- .../MultiInputColumnIntermediateLayer.java | 797 ------------------ .../dag/intermediate/MultiInputLayer.java | 666 +++++++++++++++ ...ColumnMultiReferenceIntermediateLayer.java | 797 ------------------ ...olumnSingleReferenceIntermediateLayer.java | 668 --------------- .../SingleInputMultiReferenceLayer.java | 724 ++++++++++++++++ .../SingleInputSingleReferenceLayer.java | 648 ++++++++++++++ .../transformation/dag/memory/SafetyLine.java | 4 + .../dag/transformer/Transformer.java | 102 +-- .../binary/ArithmeticAdditionTransformer.java | 7 +- .../binary/ArithmeticBinaryTransformer.java | 31 +- .../binary/ArithmeticDivisionTransformer.java | 7 +- .../binary/ArithmeticModuloTransformer.java | 7 +- .../ArithmeticMultiplicationTransformer.java | 7 +- .../ArithmeticSubtractionTransformer.java | 7 +- .../transformer/binary/BinaryTransformer.java | 273 +++--- .../binary/CompareBinaryTransformer.java | 33 +- .../binary/CompareEqualToTransformer.java | 22 +- .../CompareGreaterEqualTransformer.java | 22 +- .../binary/CompareGreaterThanTransformer.java | 21 +- .../binary/CompareLessEqualTransformer.java | 22 +- .../binary/CompareLessThanTransformer.java | 21 +- .../binary/CompareNonEqualTransformer.java | 21 +- .../binary/LogicAndTransformer.java | 6 +- .../binary/LogicBinaryTransformer.java | 259 +++--- .../binary/LogicOrTransformer.java | 6 +- .../multi/MappableUDFQueryRowTransformer.java | 102 +-- .../multi/UDFQueryRowTransformer.java | 44 +- .../multi/UDFQueryRowWindowTransformer.java | 21 +- .../multi/UDFQueryTransformer.java | 12 +- .../multi/UniversalUDFQueryTransformer.java | 73 +- .../ternary/BetweenTransformer.java | 41 +- .../ternary/CompareTernaryTransformer.java | 55 +- .../ternary/TernaryTransformer.java | 342 ++++---- .../unary/ArithmeticNegationTransformer.java | 92 +- .../dag/transformer/unary/InTransformer.java | 144 +++- .../transformer/unary/IsNullTransformer.java | 39 +- .../unary/LogicNotTransformer.java | 32 +- .../transformer/unary/RegularTransformer.java | 35 +- .../unary/TransparentTransformer.java | 37 +- .../transformer/unary/UnaryTransformer.java | 70 +- .../unary/scalar/CastFunctionTransformer.java | 493 ++++++++--- .../unary/scalar/DiffFunctionTransformer.java | 159 ++-- .../scalar/ReplaceFunctionTransformer.java | 42 +- .../scalar/RoundFunctionTransformer.java | 94 ++- .../scalar/SubStringFunctionTransformer.java | 53 +- .../transformation/dag/udf/UDTFExecutor.java | 153 +++- .../dag/util/LayerCacheUtils.java | 189 +---- .../dag/util/TransformUtils.java | 46 +- .../transformation/dag/util/TypeUtils.java | 79 ++ .../datastructure/TVColumns.java | 74 ++ .../iterator/ListForwardIterator.java} | 34 +- .../iterator/RowListForwardIterator.java | 100 +++ .../iterator/TVListForwardIterator.java | 109 +++ .../row/ElasticSerializableRowList.java | 485 +++++++++++ .../row/ElasticSerializableRowRecordList.java | 314 ------- .../row/SerializableRowList.java | 373 ++++++++ .../row/SerializableRowRecordList.java | 254 ------ .../tv/ElasticSerializableBinaryTVList.java | 160 +++- .../tv/ElasticSerializableTVList.java | 364 +++----- .../tv/SerializableBooleanTVList.java | 75 -- .../tv/SerializableDoubleTVList.java | 75 -- .../tv/SerializableFloatTVList.java | 75 -- .../tv/SerializableIntTVList.java | 73 -- .../tv/SerializableLongTVList.java | 75 -- .../datastructure/tv/SerializableTVList.java | 383 ++++++++- .../BinaryUtils.java} | 47 +- .../builder/EvaluationDAGBuilderTest.java | 4 +- .../ElasticSerializableRowListTest.java | 415 +++++++++ .../ElasticSerializableRowRecordListTest.java | 294 ------- .../ElasticSerializableTVListTest.java | 352 ++------ .../SerializableBinaryTVListTest.java | 87 -- .../SerializableBooleanTVListTest.java | 85 -- .../SerializableDoubleTVListTest.java | 84 -- .../SerializableFloatTVListTest.java | 84 -- .../SerializableIntTVListTest.java | 84 -- .../SerializableLongTVListTest.java | 84 -- ...Test.java => SerializableRowListTest.java} | 119 ++- .../datastructure/SerializableTVListTest.java | 452 +++++++++- .../commons/udf/utils/MasterRepairUtil.java | 2 +- .../library/series/util/ConsecutiveUtil.java | 2 +- 105 files changed, 7148 insertions(+), 6750 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/{LayerRowReader.java => LayerReader.java} (71%) delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/LayerPointReaderBackedSingleColumnRow.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/PointCollectorAdaptor.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputColumnIntermediateLayer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputColumnMultiReferenceIntermediateLayer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputColumnSingleReferenceIntermediateLayer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/TVColumns.java rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/{api/LayerPointReader.java => datastructure/iterator/ListForwardIterator.java} (52%) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/iterator/RowListForwardIterator.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/iterator/TVListForwardIterator.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/ElasticSerializableRowList.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/ElasticSerializableRowRecordList.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowRecordList.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableBooleanTVList.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableDoubleTVList.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableFloatTVList.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableIntTVList.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableLongTVList.java rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/{tv/SerializableBinaryTVList.java => util/BinaryUtils.java} (51%) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableRowListTest.java delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableRowRecordListTest.java delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableBinaryTVListTest.java delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableBooleanTVListTest.java delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableDoubleTVListTest.java delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableFloatTVListTest.java delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableIntTVListTest.java delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableLongTVListTest.java rename iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/{SerializableRowRecordListTest.java => SerializableRowListTest.java} (57%) diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/UDTF.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/UDTF.java index 002e174476dc..194b209ed323 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/UDTF.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/UDTF.java @@ -85,6 +85,11 @@ public interface UDTF extends UDF { @SuppressWarnings("squid:S112") void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception; + default void transform(Column[] columns, ColumnBuilder timesBuilder, ColumnBuilder valuesBuilder) + throws Exception { + throw new UnsupportedOperationException(); + } + /** * When the user specifies {@link RowByRowAccessStrategy} to access the original data in {@link * UDTFConfigurations}, this method will be called to process the transformation. In a single UDF diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/access/Row.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/access/Row.java index ea04fe008586..e2dc13a39ab2 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/access/Row.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/access/Row.java @@ -127,7 +127,7 @@ public interface Row { * @param columnIndex index of the specified column * @return {@code true} if the value of the specified column is null */ - boolean isNull(int columnIndex); + boolean isNull(int columnIndex) throws IOException; /** * Returns the number of columns. diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java index b54dc15248ad..e0ca3a53fc65 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java @@ -88,9 +88,11 @@ public boolean isNull(int columnIndex) { return rowRecord[columnIndex] == null; } + // Value columns count + // thus exclude time column @Override public int size() { - return size; + return size - 1; } public void setRowRecord(Object[] rowRecord) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java index 3d8c54ae1806..be418dba0496 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java @@ -30,7 +30,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; import org.apache.iotdb.db.queryengine.plan.expression.Expression; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.queryengine.transformation.dag.builder.EvaluationDAGBuilder; import org.apache.iotdb.db.queryengine.transformation.dag.input.QueryDataSetInputLayer; @@ -39,11 +39,13 @@ import org.apache.iotdb.db.utils.datastructure.TimeSelector; import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -72,10 +74,12 @@ public class TransformOperator implements ProcessOperator { protected QueryDataSetInputLayer inputLayer; protected UDTFContext udtfContext; - protected LayerPointReader[] transformers; + protected LayerReader[] transformers; protected List outputDataTypes; protected TimeSelector timeHeap; + protected TsBlock[] outputColumns; + protected int[] currentIndexes; protected boolean[] shouldIterateReadersToNextValid; private final String udtfQueryId; @@ -102,6 +106,10 @@ public TransformOperator( initInputLayer(inputDataTypes); initUdtfContext(outputExpressions, zoneId); initTransformers(inputLocations, outputExpressions, expressionTypes); + + outputColumns = new TsBlock[transformers.length]; + currentIndexes = new int[transformers.length]; + timeHeap = new TimeSelector(transformers.length << 1, isAscending); shouldIterateReadersToNextValid = new boolean[outputExpressions.length]; Arrays.fill(shouldIterateReadersToNextValid, true); @@ -141,7 +149,7 @@ protected void initTransformers( .buildLayerMemoryAssigner() .bindInputLayerColumnIndexWithExpression() .buildResultColumnPointReaders() - .getOutputPointReaders(); + .getOutputReaders(); } finally { UDFManagementService.getInstance().releaseLock(); } @@ -150,7 +158,7 @@ protected void initTransformers( protected YieldableState iterateAllColumnsToNextValid() throws Exception { for (int i = 0, n = shouldIterateReadersToNextValid.length; i < n; ++i) { if (shouldIterateReadersToNextValid[i]) { - final YieldableState yieldableState = iterateReaderToNextValid(transformers[i]); + final YieldableState yieldableState = iterateReaderToNextValid(i); if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; } @@ -161,20 +169,45 @@ protected YieldableState iterateAllColumnsToNextValid() throws Exception { } @SuppressWarnings("squid:S135") - protected YieldableState iterateReaderToNextValid(LayerPointReader reader) throws Exception { + protected YieldableState iterateReaderToNextValid(int index) throws Exception { // Since a constant operand is not allowed to be a result column, the reader will not be // a ConstantLayerPointReader. // If keepNull is false, we must iterate the reader until a non-null row is returned. - YieldableState yieldableState; - while ((yieldableState = reader.yield()) == YieldableState.YIELDABLE) { - if (reader.isCurrentNull() && !keepNull) { - reader.readyForNext(); - continue; + while (true) { + if (outputColumns[index] == null) { + YieldableState state = transformers[index].yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + + Column[] columns = transformers[index].current(); + + int count = columns[0].getPositionCount(); + Column[] valueColumns = {columns[0]}; + TsBlock.wrapBlocksWithoutCopy(count, (TimeColumn) columns[1], valueColumns); + TsBlock block = new TsBlock((TimeColumn) columns[1], columns[0]); + outputColumns[index] = block; + currentIndexes[index] = 0; + } + + Column outputValueColumn = outputColumns[index].getColumn(0); + while (outputValueColumn.isNull(currentIndexes[index]) && !keepNull) { + currentIndexes[index]++; + if (currentIndexes[index] == outputValueColumn.getPositionCount()) { + transformers[index].consumedAll(); + outputColumns[index] = null; + currentIndexes[index] = 0; + break; + } + } + + if (outputColumns[index] != null) { + TimeColumn outputTimeColumn = outputColumns[index].getTimeColumn(); + long time = outputTimeColumn.getLong(currentIndexes[index]); + timeHeap.add(time); + return YieldableState.YIELDABLE; } - timeHeap.add(reader.currentTime()); - break; } - return yieldableState; } @SuppressWarnings("squid:S112") @@ -218,7 +251,7 @@ public TsBlock next() throws Exception { // values for (int i = 0; i < columnCount; ++i) { - yieldableState = collectDataPoint(transformers[i], columnBuilders[i], currentTime, i); + yieldableState = collectDataPoint(columnBuilders[i], currentTime, i); if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { for (int j = 0; j <= i; ++j) { shouldIterateReadersToNextValid[j] = false; @@ -253,8 +286,8 @@ public TsBlock next() throws Exception { protected void prepareTsBlockBuilder(TsBlockBuilder tsBlockBuilder) { if (outputDataTypes == null) { outputDataTypes = new ArrayList<>(); - for (LayerPointReader reader : transformers) { - outputDataTypes.add(reader.getDataType()); + for (LayerReader reader : transformers) { + outputDataTypes.add(reader.getDataTypes()[0]); } } tsBlockBuilder.buildValueColumnBuilders(outputDataTypes); @@ -263,73 +296,61 @@ protected void prepareTsBlockBuilder(TsBlockBuilder tsBlockBuilder) { private void prepareEachColumn(int columnCount) { for (int i = 0; i < columnCount; ++i) { if (shouldIterateReadersToNextValid[i]) { - transformers[i].readyForNext(); + currentIndexes[i]++; + if (currentIndexes[i] == outputColumns[i].getColumn(0).getPositionCount()) { + transformers[i].consumedAll(); + outputColumns[i] = null; + currentIndexes[i] = 0; + } } } } - protected boolean collectReaderAppendIsNull(LayerPointReader reader, long currentTime) - throws Exception { - final YieldableState yieldableState = reader.yield(); - - if (yieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { - return true; - } - - if (yieldableState != YieldableState.YIELDABLE) { - return false; - } - - if (reader.currentTime() != currentTime) { - return true; - } - - return reader.isCurrentNull(); - } - - protected YieldableState collectDataPoint( - LayerPointReader reader, ColumnBuilder writer, long currentTime, int readerIndex) + protected YieldableState collectDataPoint(ColumnBuilder writer, long currentTime, int index) throws Exception { - final YieldableState yieldableState = reader.yield(); - if (yieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { + YieldableState state = transformers[index].yield(); + if (state == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { writer.appendNull(); return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; } - if (yieldableState != YieldableState.YIELDABLE) { - return yieldableState; + if (state != YieldableState.YIELDABLE) { + return state; } - if (reader.currentTime() != currentTime) { + TimeColumn timeColumn = outputColumns[index].getTimeColumn(); + Column valueColumn = outputColumns[index].getColumn(0); + int currentIndex = currentIndexes[index]; + if (timeColumn.getLong(currentIndex) != currentTime) { writer.appendNull(); return YieldableState.YIELDABLE; } - if (reader.isCurrentNull()) { + if (valueColumn.isNull(currentIndex)) { writer.appendNull(); } else { - TSDataType type = reader.getDataType(); + TSDataType type = transformers[index].getDataTypes()[0]; switch (type) { case INT32: case DATE: - writer.writeInt(reader.currentInt()); + writer.writeInt(valueColumn.getInt(currentIndex)); break; case INT64: case TIMESTAMP: - writer.writeLong(reader.currentLong()); + writer.writeLong(valueColumn.getLong(currentIndex)); break; case FLOAT: - writer.writeFloat(reader.currentFloat()); + writer.writeFloat(valueColumn.getFloat(currentIndex)); break; case DOUBLE: - writer.writeDouble(reader.currentDouble()); + writer.writeDouble(valueColumn.getDouble(currentIndex)); break; case BOOLEAN: - writer.writeBoolean(reader.currentBoolean()); + writer.writeBoolean(valueColumn.getBoolean(currentIndex)); break; case TEXT: case BLOB: case STRING: - writer.writeBinary(reader.currentBinary()); + writer.writeBinary(valueColumn.getBinary(currentIndex)); break; default: throw new UnSupportedDataTypeException( @@ -337,7 +358,7 @@ protected YieldableState collectDataPoint( } } - shouldIterateReadersToNextValid[readerIndex] = true; + shouldIterateReadersToNextValid[index] = true; return YieldableState.YIELDABLE; } @@ -398,7 +419,7 @@ public long calculateRetainedSizeAfterCallingNext() { } @TestOnly - public LayerPointReader[] getTransformers() { + public LayerReader[] getTransformers() { return transformers; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/BuiltInScalarFunctionHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/BuiltInScalarFunctionHelper.java index 1e729f907a9d..621b06660a88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/BuiltInScalarFunctionHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/BuiltInScalarFunctionHelper.java @@ -21,7 +21,7 @@ import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer; @@ -74,11 +74,11 @@ ColumnTransformer getBuiltInScalarFunctionColumnTransformer( * org.apache.iotdb.db.queryengine.plan.expression.visitor.IntermediateLayerVisitor} * * @param expression The FunctionExpression representing the scalar function - * @param layerPointReader input reader + * @param layerReader input reader * @return Specific Transformer of this scalar function */ Transformer getBuiltInScalarFunctionTransformer( - FunctionExpression expression, LayerPointReader layerPointReader); + FunctionExpression expression, LayerReader layerReader); /** * Some built-in scalar functions may have a different header. This method will be called by diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/CastFunctionHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/CastFunctionHelper.java index 025fdf0258e9..a7a480c644e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/CastFunctionHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/CastFunctionHelper.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; import org.apache.iotdb.db.queryengine.plan.expression.multi.builtin.BuiltInScalarFunctionHelper; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.CastFunctionColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer; @@ -61,9 +61,9 @@ public ColumnTransformer getBuiltInScalarFunctionColumnTransformer( @Override public Transformer getBuiltInScalarFunctionTransformer( - FunctionExpression expression, LayerPointReader layerPointReader) { + FunctionExpression expression, LayerReader layerReader) { return new CastFunctionTransformer( - layerPointReader, this.getBuiltInScalarFunctionReturnType(expression)); + layerReader, this.getBuiltInScalarFunctionReturnType(expression)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/DiffFunctionHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/DiffFunctionHelper.java index b6abcdd06a9c..858d21ecad8b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/DiffFunctionHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/DiffFunctionHelper.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; import org.apache.iotdb.db.queryengine.plan.expression.multi.builtin.BuiltInScalarFunctionHelper; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer; @@ -60,9 +60,9 @@ public ColumnTransformer getBuiltInScalarFunctionColumnTransformer( @Override public Transformer getBuiltInScalarFunctionTransformer( - FunctionExpression expression, LayerPointReader layerPointReader) { + FunctionExpression expression, LayerReader layerReader) { return new DiffFunctionTransformer( - layerPointReader, + layerReader, Boolean.parseBoolean( expression.getFunctionAttributes().getOrDefault("ignoreNull", "true"))); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/ReplaceFunctionHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/ReplaceFunctionHelper.java index 0eef1c60747c..af28d6cc857f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/ReplaceFunctionHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/ReplaceFunctionHelper.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; import org.apache.iotdb.db.queryengine.plan.expression.multi.builtin.BuiltInScalarFunctionHelper; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ReplaceFunctionColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer; @@ -65,10 +65,10 @@ public ColumnTransformer getBuiltInScalarFunctionColumnTransformer( @Override public Transformer getBuiltInScalarFunctionTransformer( - FunctionExpression expression, LayerPointReader layerPointReader) { + FunctionExpression expression, LayerReader layerReader) { checkFromAndToAttributes(expression); return new ReplaceFunctionTransformer( - layerPointReader, + layerReader, expression.getFunctionAttributes().get(REPLACE_FROM), expression.getFunctionAttributes().get(REPLACE_TO)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/RoundFunctionHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/RoundFunctionHelper.java index b94cbe4b51ee..0a4a8bf4a738 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/RoundFunctionHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/RoundFunctionHelper.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; import org.apache.iotdb.db.queryengine.plan.expression.multi.builtin.BuiltInScalarFunctionHelper; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RoundFunctionColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer; @@ -70,9 +70,9 @@ public ColumnTransformer getBuiltInScalarFunctionColumnTransformer( @Override public Transformer getBuiltInScalarFunctionTransformer( - FunctionExpression expression, LayerPointReader layerPointReader) { + FunctionExpression expression, LayerReader layerReader) { return new RoundFunctionTransformer( - layerPointReader, + layerReader, this.getBuiltInScalarFunctionReturnType(expression), Integer.parseInt(expression.getFunctionAttributes().getOrDefault(ROUND_PLACES, "0"))); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/SubStringFunctionHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/SubStringFunctionHelper.java index b1ba2247fabe..561805cc5a24 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/SubStringFunctionHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/SubStringFunctionHelper.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; import org.apache.iotdb.db.queryengine.plan.expression.multi.builtin.BuiltInScalarFunctionHelper; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.SubStringFunctionColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer; @@ -97,7 +97,7 @@ public ColumnTransformer getBuiltInScalarFunctionColumnTransformer( @Override public Transformer getBuiltInScalarFunctionTransformer( - FunctionExpression expression, LayerPointReader layerPointReader) { + FunctionExpression expression, LayerReader layerReader) { LinkedHashMap functionAttributes = expression.getFunctionAttributes(); int subStringLength = Integer.parseInt( @@ -107,7 +107,7 @@ public Transformer getBuiltInScalarFunctionTransformer( throw new SemanticException( "Argument exception,the scalar function [SUBSTRING] beginPosition and length must be greater than 0"); } - return new SubStringFunctionTransformer(layerPointReader, subStringStart, subStringLength); + return new SubStringFunctionTransformer(layerReader, subStringStart, subStringLength); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/IntermediateLayerVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/IntermediateLayerVisitor.java index fcab260d7715..ff96ab0020db 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/IntermediateLayerVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/IntermediateLayerVisitor.java @@ -37,13 +37,13 @@ import org.apache.iotdb.db.queryengine.plan.expression.unary.LikeExpression; import org.apache.iotdb.db.queryengine.plan.expression.unary.RegularExpression; import org.apache.iotdb.db.queryengine.plan.expression.unary.UnaryExpression; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.input.QueryDataSetInputLayer; import org.apache.iotdb.db.queryengine.transformation.dag.intermediate.ConstantIntermediateLayer; import org.apache.iotdb.db.queryengine.transformation.dag.intermediate.IntermediateLayer; -import org.apache.iotdb.db.queryengine.transformation.dag.intermediate.MultiInputColumnIntermediateLayer; -import org.apache.iotdb.db.queryengine.transformation.dag.intermediate.SingleInputColumnMultiReferenceIntermediateLayer; -import org.apache.iotdb.db.queryengine.transformation.dag.intermediate.SingleInputColumnSingleReferenceIntermediateLayer; +import org.apache.iotdb.db.queryengine.transformation.dag.intermediate.MultiInputLayer; +import org.apache.iotdb.db.queryengine.transformation.dag.intermediate.SingleInputMultiReferenceLayer; +import org.apache.iotdb.db.queryengine.transformation.dag.intermediate.SingleInputSingleReferenceLayer; import org.apache.iotdb.db.queryengine.transformation.dag.memory.LayerMemoryAssigner; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary.ArithmeticAdditionTransformer; @@ -100,12 +100,10 @@ public IntermediateLayer visitUnaryExpression( if (!context.expressionIntermediateLayerMap.containsKey(unaryExpression)) { float memoryBudgetInMB = context.memoryAssigner.assign(); - IntermediateLayer parentLayerPointReader = - this.process(unaryExpression.getExpression(), context); + IntermediateLayer intermediateLayer = this.process(unaryExpression.getExpression(), context); Transformer transformer = - getConcreteUnaryTransformer( - unaryExpression, parentLayerPointReader.constructPointReader()); + getConcreteUnaryTransformer(unaryExpression, intermediateLayer.constructReader()); // SingleInputColumnMultiReferenceIntermediateLayer doesn't support ConstantLayerPointReader // yet. And since a ConstantLayerPointReader won't produce too much IO, @@ -114,9 +112,9 @@ public IntermediateLayer visitUnaryExpression( unaryExpression, context.memoryAssigner.getReference(unaryExpression) == 1 || unaryExpression.isConstantOperand() - ? new SingleInputColumnSingleReferenceIntermediateLayer( + ? new SingleInputSingleReferenceLayer( unaryExpression, context.queryId, memoryBudgetInMB, transformer) - : new SingleInputColumnMultiReferenceIntermediateLayer( + : new SingleInputMultiReferenceLayer( unaryExpression, context.queryId, memoryBudgetInMB, transformer)); } @@ -137,8 +135,8 @@ public IntermediateLayer visitBinaryExpression( Transformer transformer = getConcreteBinaryTransformer( binaryExpression, - leftParentIntermediateLayer.constructPointReader(), - rightParentIntermediateLayer.constructPointReader()); + leftParentIntermediateLayer.constructReader(), + rightParentIntermediateLayer.constructReader()); // SingleInputColumnMultiReferenceIntermediateLayer doesn't support ConstantLayerPointReader // yet. And since a ConstantLayerPointReader won't produce too much IO, @@ -147,9 +145,9 @@ public IntermediateLayer visitBinaryExpression( binaryExpression, context.memoryAssigner.getReference(binaryExpression) == 1 || binaryExpression.isConstantOperand() - ? new SingleInputColumnSingleReferenceIntermediateLayer( + ? new SingleInputSingleReferenceLayer( binaryExpression, context.queryId, memoryBudgetInMB, transformer) - : new SingleInputColumnMultiReferenceIntermediateLayer( + : new SingleInputMultiReferenceLayer( binaryExpression, context.queryId, memoryBudgetInMB, transformer)); } @@ -171,9 +169,9 @@ public IntermediateLayer visitTernaryExpression( Transformer transformer = getConcreteTernaryTransformer( ternaryExpression, - firstParentIntermediateLayer.constructPointReader(), - secondParentIntermediateLayer.constructPointReader(), - thirdParentIntermediateLayer.constructPointReader()); + firstParentIntermediateLayer.constructReader(), + secondParentIntermediateLayer.constructReader(), + thirdParentIntermediateLayer.constructReader()); // SingleInputColumnMultiReferenceIntermediateLayer doesn't support ConstantLayerPointReader // yet. And since a ConstantLayerPointReader won't produce too much IO, @@ -182,9 +180,9 @@ public IntermediateLayer visitTernaryExpression( ternaryExpression, context.memoryAssigner.getReference(ternaryExpression) == 1 || ternaryExpression.isConstantOperand() - ? new SingleInputColumnSingleReferenceIntermediateLayer( + ? new SingleInputSingleReferenceLayer( ternaryExpression, context.queryId, memoryBudgetInMB, transformer) - : new SingleInputColumnMultiReferenceIntermediateLayer( + : new SingleInputMultiReferenceLayer( ternaryExpression, context.queryId, memoryBudgetInMB, transformer)); } @@ -200,7 +198,7 @@ public IntermediateLayer visitFunctionExpression( if (functionExpression.isAggregationFunctionExpression()) { transformer = new TransparentTransformer( - context.rawTimeSeriesInputLayer.constructValuePointReader( + context.rawTimeSeriesInputLayer.constructValueReader( functionExpression.getInputColumnIndex())); } else if (functionExpression.isBuiltInScalarFunctionExpression()) { transformer = getBuiltInScalarFunctionTransformer(functionExpression, context); @@ -216,9 +214,9 @@ public IntermediateLayer visitFunctionExpression( context.expressionIntermediateLayerMap.put( functionExpression, context.memoryAssigner.getReference(functionExpression) == 1 - ? new SingleInputColumnSingleReferenceIntermediateLayer( + ? new SingleInputSingleReferenceLayer( functionExpression, context.queryId, memoryBudgetInMB, transformer) - : new SingleInputColumnMultiReferenceIntermediateLayer( + : new SingleInputMultiReferenceLayer( functionExpression, context.queryId, memoryBudgetInMB, transformer)); } @@ -228,10 +226,10 @@ public IntermediateLayer visitFunctionExpression( private Transformer getBuiltInScalarFunctionTransformer( FunctionExpression expression, IntermediateLayerVisitorContext context) { - LayerPointReader childPointReader = - this.process(expression.getExpressions().get(0), context).constructPointReader(); + LayerReader childReader = + this.process(expression.getExpressions().get(0), context).constructReader(); return BuiltInScalarFunctionHelperFactory.createHelper(expression.getFunctionName()) - .getBuiltInScalarFunctionTransformer(expression, childPointReader); + .getBuiltInScalarFunctionTransformer(expression, childReader); } @Override @@ -240,16 +238,15 @@ public IntermediateLayer visitTimeStampOperand( if (!context.expressionIntermediateLayerMap.containsKey(timestampOperand)) { float memoryBudgetInMB = context.memoryAssigner.assign(); - LayerPointReader parentLayerPointReader = - context.rawTimeSeriesInputLayer.constructTimePointReader(); + LayerReader parentLayerReader = context.rawTimeSeriesInputLayer.constructTimeReader(); context.expressionIntermediateLayerMap.put( timestampOperand, context.memoryAssigner.getReference(timestampOperand) == 1 - ? new SingleInputColumnSingleReferenceIntermediateLayer( - timestampOperand, context.queryId, memoryBudgetInMB, parentLayerPointReader) - : new SingleInputColumnMultiReferenceIntermediateLayer( - timestampOperand, context.queryId, memoryBudgetInMB, parentLayerPointReader)); + ? new SingleInputSingleReferenceLayer( + timestampOperand, context.queryId, memoryBudgetInMB, parentLayerReader) + : new SingleInputMultiReferenceLayer( + timestampOperand, context.queryId, memoryBudgetInMB, parentLayerReader)); } return context.expressionIntermediateLayerMap.get(timestampOperand); @@ -261,17 +258,17 @@ public IntermediateLayer visitTimeSeriesOperand( if (!context.expressionIntermediateLayerMap.containsKey(timeSeriesOperand)) { float memoryBudgetInMB = context.memoryAssigner.assign(); - LayerPointReader parentLayerPointReader = - context.rawTimeSeriesInputLayer.constructValuePointReader( + LayerReader parentLayerReader = + context.rawTimeSeriesInputLayer.constructValueReader( timeSeriesOperand.getInputColumnIndex()); context.expressionIntermediateLayerMap.put( timeSeriesOperand, context.memoryAssigner.getReference(timeSeriesOperand) == 1 - ? new SingleInputColumnSingleReferenceIntermediateLayer( - timeSeriesOperand, context.queryId, memoryBudgetInMB, parentLayerPointReader) - : new SingleInputColumnMultiReferenceIntermediateLayer( - timeSeriesOperand, context.queryId, memoryBudgetInMB, parentLayerPointReader)); + ? new SingleInputSingleReferenceLayer( + timeSeriesOperand, context.queryId, memoryBudgetInMB, parentLayerReader) + : new SingleInputMultiReferenceLayer( + timeSeriesOperand, context.queryId, memoryBudgetInMB, parentLayerReader)); } return context.expressionIntermediateLayerMap.get(timeSeriesOperand); @@ -300,25 +297,24 @@ public IntermediateLayer visitCaseWhenThenExpression( throw new UnsupportedOperationException("CASE expression cannot be used with non-mappable UDF"); } - private Transformer getConcreteUnaryTransformer( - Expression expression, LayerPointReader pointReader) { + private Transformer getConcreteUnaryTransformer(Expression expression, LayerReader parentReader) { switch (expression.getExpressionType()) { case IN: InExpression inExpression = (InExpression) expression; - return new InTransformer(pointReader, inExpression.isNotIn(), inExpression.getValues()); + return new InTransformer(parentReader, inExpression.isNotIn(), inExpression.getValues()); case IS_NULL: IsNullExpression isNullExpression = (IsNullExpression) expression; - return new IsNullTransformer(pointReader, isNullExpression.isNot()); + return new IsNullTransformer(parentReader, isNullExpression.isNot()); case LOGIC_NOT: - return new LogicNotTransformer(pointReader); + return new LogicNotTransformer(parentReader); case NEGATION: - return new ArithmeticNegationTransformer(pointReader); + return new ArithmeticNegationTransformer(parentReader); case LIKE: LikeExpression likeExpression = (LikeExpression) expression; - return new RegularTransformer(pointReader, likeExpression.getPattern()); + return new RegularTransformer(parentReader, likeExpression.getPattern()); case REGEXP: RegularExpression regularExpression = (RegularExpression) expression; - return new RegularTransformer(pointReader, regularExpression.getPattern()); + return new RegularTransformer(parentReader, regularExpression.getPattern()); default: throw new UnsupportedOperationException( "Unsupported Expression Type: " + expression.getExpressionType()); @@ -327,46 +323,36 @@ private Transformer getConcreteUnaryTransformer( private Transformer getConcreteBinaryTransformer( Expression expression, - LayerPointReader leftParentLayerPointReader, - LayerPointReader rightParentLayerPointReader) { + LayerReader leftParentLayerReader, + LayerReader rightParentLayerReader) { switch (expression.getExpressionType()) { case ADDITION: - return new ArithmeticAdditionTransformer( - leftParentLayerPointReader, rightParentLayerPointReader); + return new ArithmeticAdditionTransformer(leftParentLayerReader, rightParentLayerReader); case SUBTRACTION: - return new ArithmeticSubtractionTransformer( - leftParentLayerPointReader, rightParentLayerPointReader); + return new ArithmeticSubtractionTransformer(leftParentLayerReader, rightParentLayerReader); case MULTIPLICATION: return new ArithmeticMultiplicationTransformer( - leftParentLayerPointReader, rightParentLayerPointReader); + leftParentLayerReader, rightParentLayerReader); case DIVISION: - return new ArithmeticDivisionTransformer( - leftParentLayerPointReader, rightParentLayerPointReader); + return new ArithmeticDivisionTransformer(leftParentLayerReader, rightParentLayerReader); case MODULO: - return new ArithmeticModuloTransformer( - leftParentLayerPointReader, rightParentLayerPointReader); + return new ArithmeticModuloTransformer(leftParentLayerReader, rightParentLayerReader); case EQUAL_TO: - return new CompareEqualToTransformer( - leftParentLayerPointReader, rightParentLayerPointReader); + return new CompareEqualToTransformer(leftParentLayerReader, rightParentLayerReader); case NON_EQUAL: - return new CompareNonEqualTransformer( - leftParentLayerPointReader, rightParentLayerPointReader); + return new CompareNonEqualTransformer(leftParentLayerReader, rightParentLayerReader); case GREATER_THAN: - return new CompareGreaterThanTransformer( - leftParentLayerPointReader, rightParentLayerPointReader); + return new CompareGreaterThanTransformer(leftParentLayerReader, rightParentLayerReader); case GREATER_EQUAL: - return new CompareGreaterEqualTransformer( - leftParentLayerPointReader, rightParentLayerPointReader); + return new CompareGreaterEqualTransformer(leftParentLayerReader, rightParentLayerReader); case LESS_THAN: - return new CompareLessThanTransformer( - leftParentLayerPointReader, rightParentLayerPointReader); + return new CompareLessThanTransformer(leftParentLayerReader, rightParentLayerReader); case LESS_EQUAL: - return new CompareLessEqualTransformer( - leftParentLayerPointReader, rightParentLayerPointReader); + return new CompareLessEqualTransformer(leftParentLayerReader, rightParentLayerReader); case LOGIC_AND: - return new LogicAndTransformer(leftParentLayerPointReader, rightParentLayerPointReader); + return new LogicAndTransformer(leftParentLayerReader, rightParentLayerReader); case LOGIC_OR: - return new LogicOrTransformer(leftParentLayerPointReader, rightParentLayerPointReader); + return new LogicOrTransformer(leftParentLayerReader, rightParentLayerReader); default: throw new UnsupportedOperationException( "Unsupported Expression Type: " + expression.getExpressionType()); @@ -375,15 +361,15 @@ private Transformer getConcreteBinaryTransformer( private Transformer getConcreteTernaryTransformer( Expression expression, - LayerPointReader firstParentLayerPointReader, - LayerPointReader secondParentLayerPointReader, - LayerPointReader thirdParentLayerPointReader) { + LayerReader firstParentLayerReader, + LayerReader secondParentLayerReader, + LayerReader thirdParentLayerReader) { if (expression.getExpressionType() == ExpressionType.BETWEEN) { BetweenExpression betweenExpression = (BetweenExpression) expression; return new BetweenTransformer( - firstParentLayerPointReader, - secondParentLayerPointReader, - thirdParentLayerPointReader, + firstParentLayerReader, + secondParentLayerReader, + thirdParentLayerReader, betweenExpression.isNotBetween()); } throw new UnsupportedOperationException( @@ -409,9 +395,9 @@ private UDFQueryTransformer getUdfTransformer( switch (accessStrategy.getAccessStrategyType()) { case MAPPABLE_ROW_BY_ROW: return new MappableUDFQueryRowTransformer( - udfInputIntermediateLayer.constructRowReader(), executor); + udfInputIntermediateLayer.constructReader(), executor); case ROW_BY_ROW: - return new UDFQueryRowTransformer(udfInputIntermediateLayer.constructRowReader(), executor); + return new UDFQueryRowTransformer(udfInputIntermediateLayer.constructReader(), executor); case SLIDING_SIZE_WINDOW: case SLIDING_TIME_WINDOW: case SESSION_TIME_WINDOW: @@ -433,12 +419,12 @@ private IntermediateLayer constructUdfInputIntermediateLayer( } return intermediateLayers.size() == 1 ? intermediateLayers.get(0) - : new MultiInputColumnIntermediateLayer( + : new MultiInputLayer( functionExpression, context.queryId, context.memoryAssigner.assign(), intermediateLayers.stream() - .map(IntermediateLayer::constructPointReader) + .map(IntermediateLayer::constructReader) .collect(Collectors.toList())); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerRowReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerReader.java similarity index 71% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerRowReader.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerReader.java index 6382176c74e5..4d58b100bfb0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerRowReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerReader.java @@ -19,25 +19,19 @@ package org.apache.iotdb.db.queryengine.transformation.api; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.udf.api.access.Row; - +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import java.io.IOException; -public interface LayerRowReader extends YieldableReader { +public interface LayerReader extends YieldableReader { + // As its name suggests, + // This method is only used in PointReader + boolean isConstantPointReader(); - boolean next() throws IOException, QueryProcessException; + void consumedAll(); - void readyForNext(); + Column[] current() throws IOException; TSDataType[] getDataTypes(); - - long currentTime() throws IOException; - - Row currentRow(); - - /** whether current row fields are all null. */ - boolean isCurrentNull(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerRowWindowReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerRowWindowReader.java index aa5f92b2d579..327f90d10b16 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerRowWindowReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerRowWindowReader.java @@ -27,9 +27,6 @@ import java.io.IOException; public interface LayerRowWindowReader extends YieldableReader { - - boolean next() throws IOException, QueryProcessException; - void readyForNext() throws IOException, QueryProcessException; TSDataType[] getDataTypes(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnWindow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnWindow.java index 16b0ffcf93ea..caf44ad66dd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnWindow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnWindow.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.queryengine.transformation.dag.adapter; import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; -import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowRecordList; +import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowList; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.access.RowIterator; import org.apache.iotdb.udf.api.access.RowWindow; @@ -32,7 +32,7 @@ public class ElasticSerializableRowRecordListBackedMultiColumnWindow implements RowWindow { - private final ElasticSerializableRowRecordList rowRecordList; + private final ElasticSerializableRowList rowRecordList; private final TSDataType[] dataTypes; private int beginIndex; @@ -46,7 +46,7 @@ public class ElasticSerializableRowRecordListBackedMultiColumnWindow implements private ElasticSerializableRowRecordListBackedMultiColumnWindowIterator rowIterator; public ElasticSerializableRowRecordListBackedMultiColumnWindow( - ElasticSerializableRowRecordList rowRecordList) { + ElasticSerializableRowList rowRecordList) { this.rowRecordList = rowRecordList; this.dataTypes = rowRecordList.getDataTypes(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnWindowIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnWindowIterator.java index 3ccc64bb6f9f..396b34f3f400 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnWindowIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnWindowIterator.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.queryengine.transformation.dag.adapter; -import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowRecordList; +import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowList; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.access.RowIterator; @@ -28,7 +28,7 @@ public class ElasticSerializableRowRecordListBackedMultiColumnWindowIterator implements RowIterator { - private final ElasticSerializableRowRecordList rowRecordList; + private final ElasticSerializableRowList rowRecordList; private final int beginIndex; private final int size; @@ -36,7 +36,7 @@ public class ElasticSerializableRowRecordListBackedMultiColumnWindowIterator private int rowIndex; public ElasticSerializableRowRecordListBackedMultiColumnWindowIterator( - ElasticSerializableRowRecordList rowRecordList, int beginIndex, int endIndex) { + ElasticSerializableRowList rowRecordList, int beginIndex, int endIndex) { this.rowRecordList = rowRecordList; this.beginIndex = beginIndex; size = endIndex - beginIndex; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableTVListBackedSingleColumnRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableTVListBackedSingleColumnRow.java index 92a257182b54..a6ab72224572 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableTVListBackedSingleColumnRow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableTVListBackedSingleColumnRow.java @@ -85,7 +85,7 @@ public Type getDataType(int columnIndex) { } @Override - public boolean isNull(int columnIndex) { + public boolean isNull(int columnIndex) throws IOException { return tvList.isNull(currentRowIndex); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/LayerPointReaderBackedSingleColumnRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/LayerPointReaderBackedSingleColumnRow.java deleted file mode 100644 index 8156dec8c4af..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/LayerPointReaderBackedSingleColumnRow.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.adapter; - -import org.apache.iotdb.commons.udf.utils.UDFBinaryTransformer; -import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; -import org.apache.iotdb.udf.api.access.Row; -import org.apache.iotdb.udf.api.type.Binary; -import org.apache.iotdb.udf.api.type.Type; - -import org.apache.tsfile.common.conf.TSFileConfig; - -import java.io.IOException; - -public class LayerPointReaderBackedSingleColumnRow implements Row { - - private final LayerPointReader layerPointReader; - - public LayerPointReaderBackedSingleColumnRow(LayerPointReader layerPointReader) { - this.layerPointReader = layerPointReader; - } - - @Override - public long getTime() throws IOException { - return layerPointReader.currentTime(); - } - - @Override - public int getInt(int columnIndex) throws IOException { - return layerPointReader.currentInt(); - } - - @Override - public long getLong(int columnIndex) throws IOException { - return layerPointReader.currentLong(); - } - - @Override - public float getFloat(int columnIndex) throws IOException { - return layerPointReader.currentFloat(); - } - - @Override - public double getDouble(int columnIndex) throws IOException { - return layerPointReader.currentDouble(); - } - - @Override - public boolean getBoolean(int columnIndex) throws IOException { - return layerPointReader.currentBoolean(); - } - - @Override - public Binary getBinary(int columnIndex) throws IOException { - return UDFBinaryTransformer.transformToUDFBinary(layerPointReader.currentBinary()); - } - - @Override - public String getString(int columnIndex) throws IOException { - return layerPointReader.currentBinary().getStringValue(TSFileConfig.STRING_CHARSET); - } - - @Override - public Type getDataType(int columnIndex) { - return UDFDataTypeTransformer.transformToUDFDataType(layerPointReader.getDataType()); - } - - @Override - public boolean isNull(int columnIndex) { - try { - return layerPointReader.isCurrentNull(); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - @Override - public int size() { - return 1; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/PointCollectorAdaptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/PointCollectorAdaptor.java new file mode 100644 index 000000000000..8ffb0a5bf346 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/PointCollectorAdaptor.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.adapter; + +import org.apache.iotdb.commons.udf.utils.UDFBinaryTransformer; +import org.apache.iotdb.udf.api.collector.PointCollector; +import org.apache.iotdb.udf.api.type.Binary; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.utils.BytesUtils; + +import java.io.IOException; + +public class PointCollectorAdaptor implements PointCollector { + private final ColumnBuilder timeColumnBuilder; + + private final ColumnBuilder valueColumnBuilder; + + public PointCollectorAdaptor(ColumnBuilder timeColumnBuilder, ColumnBuilder valueColumnBuilder) { + this.timeColumnBuilder = timeColumnBuilder; + this.valueColumnBuilder = valueColumnBuilder; + } + + @Override + public void putInt(long timestamp, int value) throws IOException { + timeColumnBuilder.writeLong(timestamp); + valueColumnBuilder.writeInt(value); + } + + @Override + public void putLong(long timestamp, long value) throws IOException { + timeColumnBuilder.writeLong(timestamp); + valueColumnBuilder.writeLong(value); + } + + @Override + public void putFloat(long timestamp, float value) throws IOException { + timeColumnBuilder.writeLong(timestamp); + valueColumnBuilder.writeFloat(value); + } + + @Override + public void putDouble(long timestamp, double value) throws IOException { + timeColumnBuilder.writeLong(timestamp); + valueColumnBuilder.writeDouble(value); + } + + @Override + public void putBoolean(long timestamp, boolean value) throws IOException { + timeColumnBuilder.writeLong(timestamp); + valueColumnBuilder.writeBoolean(value); + } + + @Override + public void putBinary(long timestamp, Binary value) throws IOException { + timeColumnBuilder.writeLong(timestamp); + valueColumnBuilder.writeBinary(UDFBinaryTransformer.transformToBinary(value)); + } + + @Override + public void putString(long timestamp, String value) throws IOException { + timeColumnBuilder.writeLong(timestamp); + valueColumnBuilder.writeBinary(BytesUtils.valueOf(value)); + } + + public void putNull(long timestamp) { + timeColumnBuilder.writeLong(timestamp); + valueColumnBuilder.appendNull(); + } + + public TimeColumn buildTimeColumn() { + return (TimeColumn) timeColumnBuilder.build(); + } + + public Column buildValueColumn() { + return valueColumnBuilder.build(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/builder/EvaluationDAGBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/builder/EvaluationDAGBuilder.java index e94a222ddb1f..a404a63f8d31 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/builder/EvaluationDAGBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/builder/EvaluationDAGBuilder.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.Expression; import org.apache.iotdb.db.queryengine.plan.expression.visitor.IntermediateLayerVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.input.QueryDataSetInputLayer; import org.apache.iotdb.db.queryengine.transformation.dag.intermediate.IntermediateLayer; import org.apache.iotdb.db.queryengine.transformation.dag.memory.LayerMemoryAssigner; @@ -43,7 +43,7 @@ public class EvaluationDAGBuilder { private final Map> inputLocations; private final Expression[] outputExpressions; - private final LayerPointReader[] outputPointReaders; + private final LayerReader[] outputReaders; private final Map, TSDataType> expressionTypes; @@ -72,7 +72,7 @@ public EvaluationDAGBuilder( this.expressionTypes = expressionTypes; this.udtfContext = udtfContext; - outputPointReaders = new LayerPointReader[outputExpressions.length]; + outputReaders = new LayerReader[outputExpressions.length]; memoryAssigner = new LayerMemoryAssigner(memoryBudgetInMB); @@ -105,12 +105,12 @@ public EvaluationDAGBuilder buildResultColumnPointReaders() { expressionTypes, memoryAssigner); for (int i = 0; i < outputExpressions.length; ++i) { - outputPointReaders[i] = visitor.process(outputExpressions[i], context).constructPointReader(); + outputReaders[i] = visitor.process(outputExpressions[i], context).constructReader(); } return this; } - public LayerPointReader[] getOutputPointReaders() { - return outputPointReaders; + public LayerReader[] getOutputReaders() { + return outputReaders; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java index 542c43ea8b3f..ffe16112f62d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java @@ -21,30 +21,34 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.utils.CommonUtils; -import org.apache.commons.lang3.Validate; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.FloatColumn; +import org.apache.tsfile.read.common.block.column.IntColumn; +import org.apache.tsfile.read.common.block.column.LongColumn; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.Binary; import java.io.IOException; +import java.util.Optional; -/** LayerPointReader for constants. */ -public class ConstantInputReader implements LayerPointReader { +public class ConstantInputReader implements LayerReader { + private final TSDataType dataType; - private final ConstantOperand expression; - - protected int cachedInt; - protected long cachedLong; - protected float cachedFloat; - protected double cachedDouble; - protected boolean cachedBoolean; - protected Binary cachedBinary; + private final Column[] cachedColumns; public ConstantInputReader(ConstantOperand expression) throws QueryProcessException { - this.expression = Validate.notNull(expression); + if (expression == null) { + throw new QueryProcessException("The expression cannot be null"); + } Object value = CommonUtils.parseValue(expression.getDataType(), expression.getValueString()); if (value == null) { @@ -52,24 +56,40 @@ public ConstantInputReader(ConstantOperand expression) throws QueryProcessExcept "Invalid constant operand: " + expression.getExpressionString()); } - switch (expression.getDataType()) { + // Use RLEColumn to mimic column filled with same values + dataType = expression.getDataType(); + cachedColumns = new Column[1]; + int count = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber(); + switch (dataType) { case INT32: - cachedInt = (int) value; + int[] intArray = {(int) value}; + Column intColumn = new IntColumn(1, Optional.empty(), intArray); + cachedColumns[0] = new RunLengthEncodedColumn(intColumn, count); break; case INT64: - cachedLong = (long) value; + long[] longArray = {(long) value}; + Column longColumn = new LongColumn(1, Optional.empty(), longArray); + cachedColumns[0] = new RunLengthEncodedColumn(longColumn, count); break; case FLOAT: - cachedFloat = (float) value; + float[] floatArray = {(float) value}; + Column floatColumn = new FloatColumn(1, Optional.empty(), floatArray); + cachedColumns[0] = new RunLengthEncodedColumn(floatColumn, count); break; case DOUBLE: - cachedDouble = (double) value; + double[] doubleArray = {(double) value}; + Column doubleColumn = new DoubleColumn(1, Optional.empty(), doubleArray); + cachedColumns[0] = new RunLengthEncodedColumn(doubleColumn, count); break; case TEXT: - cachedBinary = (Binary) value; + Binary[] binaryArray = {(Binary) value}; + Column binaryColumn = new BinaryColumn(1, Optional.empty(), binaryArray); + cachedColumns[0] = new RunLengthEncodedColumn(binaryColumn, count); break; case BOOLEAN: - cachedBoolean = (boolean) value; + boolean[] booleanArray = {(boolean) value}; + Column booleanColumn = new BooleanColumn(1, Optional.empty(), booleanArray); + cachedColumns[0] = new RunLengthEncodedColumn(booleanColumn, count); break; default: throw new QueryProcessException("Unsupported type: " + expression.getDataType()); @@ -82,62 +102,22 @@ public boolean isConstantPointReader() { } @Override - public YieldableState yield() { - return YieldableState.YIELDABLE; - } - - @Override - public boolean next() { - return true; - } - - @Override - public void readyForNext() { + public void consumedAll() { // Do nothing } @Override - public TSDataType getDataType() { - return expression.getDataType(); - } - - @Override - public long currentTime() throws IOException { - throw new UnsupportedOperationException(); + public Column[] current() throws IOException { + return cachedColumns; } @Override - public int currentInt() throws IOException { - return cachedInt; - } - - @Override - public long currentLong() throws IOException { - return cachedLong; - } - - @Override - public float currentFloat() throws IOException { - return cachedFloat; - } - - @Override - public double currentDouble() throws IOException { - return cachedDouble; - } - - @Override - public boolean currentBoolean() throws IOException { - return cachedBoolean; - } - - @Override - public Binary currentBinary() throws IOException { - return cachedBinary; + public YieldableState yield() { + return YieldableState.YIELDABLE; } @Override - public boolean isCurrentNull() { - return false; + public TSDataType[] getDataTypes() { + return new TSDataType[] {dataType}; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/IUDFInputDataSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/IUDFInputDataSet.java index f48507e8184b..33a384507595 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/IUDFInputDataSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/IUDFInputDataSet.java @@ -21,9 +21,9 @@ import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; -import java.io.IOException; import java.util.List; /** The input data set interface for a UDFPlan. */ @@ -32,20 +32,7 @@ public interface IUDFInputDataSet { /** returns the input data types, except the timestamp column(the last column). */ List getDataTypes(); - /** Whether the data set has next row. */ - boolean hasNextRowInObjects() throws IOException; - - /** Whether the data set has next row. */ - default YieldableState canYieldNextRowInObjects() throws Exception { - return hasNextRowInObjects() - ? YieldableState.YIELDABLE - : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - /** - * Get the next row for UDFPlan input. - * - *

The last element in the row is the timestamp. - */ - Object[] nextRowInObjects(); + YieldableState yield() throws Exception; + + Column[] currentBlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/QueryDataSetInputLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/QueryDataSetInputLayer.java index c504919feeb1..d0db0f2b58c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/QueryDataSetInputLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/QueryDataSetInputLayer.java @@ -20,309 +20,121 @@ package org.apache.iotdb.db.queryengine.transformation.dag.input; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.queryengine.transformation.dag.memory.SafetyLine; import org.apache.iotdb.db.queryengine.transformation.dag.memory.SafetyLine.SafetyPile; -import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowRecordList; +import org.apache.iotdb.db.queryengine.transformation.datastructure.iterator.RowListForwardIterator; +import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowList; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; - -import java.io.IOException; public class QueryDataSetInputLayer { - private IUDFInputDataSet queryDataSet; + private TsBlockInputDataSet queryDataSet; private TSDataType[] dataTypes; - private int timestampIndex; - private ElasticSerializableRowRecordList rowRecordList; + private ElasticSerializableRowList rowList; private SafetyLine safetyLine; public QueryDataSetInputLayer( - String queryId, float memoryBudgetInMB, IUDFInputDataSet queryDataSet) + String queryId, float memoryBudgetInMB, TsBlockInputDataSet queryDataSet) throws QueryProcessException { construct(queryId, memoryBudgetInMB, queryDataSet); } - private void construct(String queryId, float memoryBudgetInMB, IUDFInputDataSet queryDataSet) + private void construct(String queryId, float memoryBudgetInMB, TsBlockInputDataSet queryDataSet) throws QueryProcessException { this.queryDataSet = queryDataSet; dataTypes = queryDataSet.getDataTypes().toArray(new TSDataType[0]); - timestampIndex = dataTypes.length; - rowRecordList = - new ElasticSerializableRowRecordList( + rowList = + new ElasticSerializableRowList( dataTypes, queryId, memoryBudgetInMB, 1 + dataTypes.length / 2); safetyLine = new SafetyLine(); } public void updateRowRecordListEvictionUpperBound() { - rowRecordList.setEvictionUpperBound(safetyLine.getSafetyLine()); + rowList.setEvictionUpperBound(safetyLine.getSafetyLine()); } - public LayerPointReader constructTimePointReader() { - return new TimePointReader(); + public BlockColumnReader constructValueReader(int columnIndex) { + return new BlockColumnReader(columnIndex); } - public LayerPointReader constructValuePointReader(int columnIndex) { - return new ValuePointReader(columnIndex); + public BlockColumnReader constructTimeReader() { + return new BlockColumnReader(dataTypes.length); } - private abstract class AbstractLayerPointReader implements LayerPointReader { - - protected final SafetyPile safetyPile; - - protected int currentRowIndex; - - protected boolean hasCachedRowRecord; - protected Object[] cachedRowRecord; - - AbstractLayerPointReader() { - safetyPile = safetyLine.addSafetyPile(); - - hasCachedRowRecord = false; - cachedRowRecord = null; - currentRowIndex = -1; - } - - @Override - public final long currentTime() throws IOException { - return (long) cachedRowRecord[timestampIndex]; - } - - @Override - public final boolean isConstantPointReader() { - return false; - } + private class BlockColumnReader implements LayerReader { - @Override - public final void readyForNext() { - hasCachedRowRecord = false; - cachedRowRecord = null; + private final SafetyPile safetyPile = safetyLine.addSafetyPile(); - safetyPile.moveForwardTo(currentRowIndex + 1); - } - } - - private class ValuePointReader extends AbstractLayerPointReader { + private final int columnIndex; + private Column[] cachedColumns = null; + private int cacheConsumed = 0; - protected final int columnIndex; + private final RowListForwardIterator iterator = rowList.constructIterator(); - ValuePointReader(int columnIndex) { - super(); + BlockColumnReader(int columnIndex) { this.columnIndex = columnIndex; } @Override - public YieldableState yield() throws Exception { - if (hasCachedRowRecord) { + public final YieldableState yield() throws Exception { + // Look up code-level cache + if (cachedColumns != null && cacheConsumed < cachedColumns[0].getPositionCount()) { return YieldableState.YIELDABLE; } - for (int i = currentRowIndex + 1; i < rowRecordList.size(); ++i) { - Object[] rowRecordCandidate = rowRecordList.getRowRecord(i); - // If any field in the current row are null, we should treat this row as valid. - // Because in a GROUP BY time read, we must return every time window record even if there's - // no data. - // Under the situation, if hasCachedRowRecord is false, this row will be skipped and the - // result is not as our expected. - if (rowRecordCandidate[columnIndex] != null || rowRecordList.fieldsHasAnyNull(i)) { - hasCachedRowRecord = true; - cachedRowRecord = rowRecordCandidate; - currentRowIndex = i; - return YieldableState.YIELDABLE; - } - } - - YieldableState yieldableState; - while (YieldableState.YIELDABLE.equals( - yieldableState = queryDataSet.canYieldNextRowInObjects())) { - Object[] rowRecordCandidate = queryDataSet.nextRowInObjects(); - rowRecordList.put(rowRecordCandidate); - if (rowRecordCandidate[columnIndex] != null - || rowRecordList.fieldsHasAnyNull(rowRecordList.size() - 1)) { - hasCachedRowRecord = true; - cachedRowRecord = rowRecordCandidate; - currentRowIndex = rowRecordList.size() - 1; - return YieldableState.YIELDABLE; - } - } - return yieldableState; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - if (hasCachedRowRecord) { - return true; - } - - for (int i = currentRowIndex + 1; i < rowRecordList.size(); ++i) { - Object[] rowRecordCandidate = rowRecordList.getRowRecord(i); - // If any field in the current row are null, we should treat this row as valid. - // Because in a GROUP BY time read, we must return every time window record even if there's - // no data. - // Under the situation, if hasCachedRowRecord is false, this row will be skipped and the - // result is not as our expected. - if (rowRecordCandidate[columnIndex] != null || rowRecordList.fieldsHasAnyNull(i)) { - hasCachedRowRecord = true; - cachedRowRecord = rowRecordCandidate; - currentRowIndex = i; - break; - } - } - - if (!hasCachedRowRecord) { - while (queryDataSet.hasNextRowInObjects()) { - Object[] rowRecordCandidate = queryDataSet.nextRowInObjects(); - rowRecordList.put(rowRecordCandidate); - if (rowRecordCandidate[columnIndex] != null - || rowRecordList.fieldsHasAnyNull(rowRecordList.size() - 1)) { - hasCachedRowRecord = true; - cachedRowRecord = rowRecordCandidate; - currentRowIndex = rowRecordList.size() - 1; - break; - } - } - } - - return hasCachedRowRecord; - } - - @Override - public TSDataType getDataType() { - return dataTypes[columnIndex]; - } - - @Override - public int currentInt() { - return (int) cachedRowRecord[columnIndex]; - } + // Cache columns from row record list + if (iterator.hasNext()) { + iterator.next(); + cachedColumns = iterator.currentBlock(); - @Override - public long currentLong() { - return (long) cachedRowRecord[columnIndex]; - } - - @Override - public float currentFloat() { - return (float) cachedRowRecord[columnIndex]; - } - - @Override - public double currentDouble() { - return (double) cachedRowRecord[columnIndex]; - } - - @Override - public boolean currentBoolean() { - return (boolean) cachedRowRecord[columnIndex]; - } - - @Override - public boolean isCurrentNull() { - return cachedRowRecord[columnIndex] == null; - } - - @Override - public Binary currentBinary() { - return (Binary) cachedRowRecord[columnIndex]; - } - } - - private class TimePointReader extends AbstractLayerPointReader { - - @Override - public YieldableState yield() throws Exception { - if (hasCachedRowRecord) { return YieldableState.YIELDABLE; } - final int nextIndex = currentRowIndex + 1; - if (nextIndex < rowRecordList.size()) { - hasCachedRowRecord = true; - cachedRowRecord = rowRecordList.getRowRecord(nextIndex); - currentRowIndex = nextIndex; - return YieldableState.YIELDABLE; - } - - final YieldableState yieldableState = queryDataSet.canYieldNextRowInObjects(); - if (YieldableState.YIELDABLE == yieldableState) { - Object[] rowRecordCandidate = queryDataSet.nextRowInObjects(); - rowRecordList.put(rowRecordCandidate); - - hasCachedRowRecord = true; - cachedRowRecord = rowRecordCandidate; - currentRowIndex = rowRecordList.size() - 1; + // Cache columns from child operator + YieldableState yieldableState = queryDataSet.yield(); + if (YieldableState.YIELDABLE.equals(yieldableState)) { + Column[] columns = queryDataSet.currentBlock(); + rowList.put(columns); + iterator.next(); + cachedColumns = iterator.currentBlock(); + // No need to call `.consume()` like method in queryDataSet } return yieldableState; } @Override - public boolean next() throws QueryProcessException, IOException { - if (hasCachedRowRecord) { - return true; - } + public void consumedAll() { + int steps = cachedColumns[0].getPositionCount() - cacheConsumed; + safetyPile.moveForward(steps); - final int nextIndex = currentRowIndex + 1; - if (nextIndex < rowRecordList.size()) { - hasCachedRowRecord = true; - cachedRowRecord = rowRecordList.getRowRecord(nextIndex); - currentRowIndex = nextIndex; - return true; - } - - if (queryDataSet.hasNextRowInObjects()) { - Object[] rowRecordCandidate = queryDataSet.nextRowInObjects(); - rowRecordList.put(rowRecordCandidate); - - hasCachedRowRecord = true; - cachedRowRecord = rowRecordCandidate; - currentRowIndex = rowRecordList.size() - 1; - return true; - } - - return false; - } - - @Override - public TSDataType getDataType() { - return TSDataType.INT64; - } - - @Override - public int currentInt() throws IOException { - throw new UnsupportedOperationException(); + cacheConsumed = 0; + cachedColumns = null; } @Override - public long currentLong() throws IOException { - return (long) cachedRowRecord[timestampIndex]; + public Column[] current() { + Column valueColumn = cachedColumns[columnIndex]; + Column timeColumn = cachedColumns[cachedColumns.length - 1]; + return cacheConsumed == 0 + ? new Column[] {valueColumn, timeColumn} + : new Column[] { + valueColumn.subColumn(cacheConsumed), timeColumn.subColumn(cacheConsumed) + }; } @Override - public float currentFloat() throws IOException { - throw new UnsupportedOperationException(); + public TSDataType[] getDataTypes() { + return new TSDataType[] {dataTypes[columnIndex]}; } @Override - public double currentDouble() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public boolean currentBoolean() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isCurrentNull() throws IOException { + public final boolean isConstantPointReader() { return false; } - - @Override - public Binary currentBinary() throws IOException { - throw new UnsupportedOperationException(); - } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/TsBlockInputDataSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/TsBlockInputDataSet.java index 75338f62be60..5cc8b083210b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/TsBlockInputDataSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/TsBlockInputDataSet.java @@ -22,9 +22,9 @@ import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; -import org.apache.tsfile.read.common.block.TsBlock.TsBlockRowIterator; import java.util.List; @@ -32,8 +32,7 @@ public class TsBlockInputDataSet implements IUDFInputDataSet { private final Operator operator; private final List dataTypes; - - private TsBlockRowIterator tsBlockRowIterator; + private TsBlock tsBlock; public TsBlockInputDataSet(Operator operator, List dataTypes) { this.operator = operator; @@ -46,41 +45,29 @@ public List getDataTypes() { } @Override - public boolean hasNextRowInObjects() { - throw new UnsupportedOperationException(); - } - - @Override - public YieldableState canYieldNextRowInObjects() throws Exception { - if (tsBlockRowIterator == null) { + public YieldableState yield() throws Exception { + // Request from child operator if there is no TsBlock + if (tsBlock == null) { if (operator.isBlocked() != Operator.NOT_BLOCKED) { return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; } if (!operator.hasNextWithTimer()) { return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; } - final TsBlock tsBlock = operator.nextWithTimer(); + tsBlock = operator.nextWithTimer(); if (tsBlock == null) { return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; } - tsBlockRowIterator = tsBlock.getTsBlockRowIterator(); } - if (tsBlockRowIterator.hasNext()) { - return YieldableState.YIELDABLE; - } else { - tsBlockRowIterator = null; - if (operator.isBlocked() != Operator.NOT_BLOCKED) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - return operator.hasNextWithTimer() - ? YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA - : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } + return YieldableState.YIELDABLE; } @Override - public Object[] nextRowInObjects() { - return tsBlockRowIterator.next(); + public Column[] currentBlock() { + Column[] rows = tsBlock.getAllColumns(); + // Prepare for next TsBlock + tsBlock = null; + return rows; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/ConstantIntermediateLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/ConstantIntermediateLayer.java index f2d4c48cab38..72127aad5a56 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/ConstantIntermediateLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/ConstantIntermediateLayer.java @@ -21,8 +21,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; -import org.apache.iotdb.db.queryengine.transformation.api.LayerRowReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.LayerRowWindowReader; import org.apache.iotdb.db.queryengine.transformation.dag.input.ConstantInputReader; import org.apache.iotdb.udf.api.customizer.strategy.SessionTimeWindowAccessStrategy; @@ -33,24 +32,18 @@ /** IntermediateLayer for constants. */ public class ConstantIntermediateLayer extends IntermediateLayer { - private final LayerPointReader constantLayerPointReaderCache; + private final LayerReader constantLayerReader; public ConstantIntermediateLayer( ConstantOperand expression, String queryId, float memoryBudgetInMB) throws QueryProcessException { super(expression, queryId, memoryBudgetInMB); - constantLayerPointReaderCache = new ConstantInputReader(expression); + constantLayerReader = new ConstantInputReader(expression); } @Override - public LayerPointReader constructPointReader() { - return constantLayerPointReaderCache; - } - - @Override - public LayerRowReader constructRowReader() { - // Not allowed since the timestamp of a constant row is not defined. - throw new UnsupportedOperationException(); + public LayerReader constructReader() { + return constantLayerReader; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/IntermediateLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/IntermediateLayer.java index 26a866521ed4..149485ef3b5b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/IntermediateLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/IntermediateLayer.java @@ -21,8 +21,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.plan.expression.Expression; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; -import org.apache.iotdb.db.queryengine.transformation.api.LayerRowReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.LayerRowWindowReader; import org.apache.iotdb.udf.api.customizer.strategy.AccessStrategy; import org.apache.iotdb.udf.api.customizer.strategy.SessionTimeWindowAccessStrategy; @@ -48,9 +47,7 @@ protected IntermediateLayer(Expression expression, String queryId, float memoryB this.memoryBudgetInMB = memoryBudgetInMB; } - public abstract LayerPointReader constructPointReader(); - - public abstract LayerRowReader constructRowReader(); + public abstract LayerReader constructReader(); public final LayerRowWindowReader constructRowWindowReader( AccessStrategy strategy, float memoryBudgetInMB) throws QueryProcessException, IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputColumnIntermediateLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputColumnIntermediateLayer.java deleted file mode 100644 index 6fbc248b04c6..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputColumnIntermediateLayer.java +++ /dev/null @@ -1,797 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.intermediate; - -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.plan.expression.Expression; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; -import org.apache.iotdb.db.queryengine.transformation.api.LayerRowReader; -import org.apache.iotdb.db.queryengine.transformation.api.LayerRowWindowReader; -import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; -import org.apache.iotdb.db.queryengine.transformation.dag.adapter.ElasticSerializableRowRecordListBackedMultiColumnRow; -import org.apache.iotdb.db.queryengine.transformation.dag.adapter.ElasticSerializableRowRecordListBackedMultiColumnWindow; -import org.apache.iotdb.db.queryengine.transformation.dag.input.IUDFInputDataSet; -import org.apache.iotdb.db.queryengine.transformation.dag.util.InputRowUtils; -import org.apache.iotdb.db.queryengine.transformation.dag.util.LayerCacheUtils; -import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowRecordList; -import org.apache.iotdb.db.utils.datastructure.TimeSelector; -import org.apache.iotdb.udf.api.access.Row; -import org.apache.iotdb.udf.api.access.RowWindow; -import org.apache.iotdb.udf.api.customizer.strategy.SessionTimeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.StateWindowAccessStrategy; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.write.UnSupportedDataTypeException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -public class MultiInputColumnIntermediateLayer extends IntermediateLayer - implements IUDFInputDataSet { - - private static final Logger LOGGER = - LoggerFactory.getLogger(MultiInputColumnIntermediateLayer.class); - - private final LayerPointReader[] layerPointReaders; - private final TSDataType[] dataTypes; - private final TimeSelector timeHeap; - private final boolean[] shouldMoveNext; - - private boolean isFirstIteration = true; - private Object[] cachedRow = null; - - public MultiInputColumnIntermediateLayer( - Expression expression, - String queryId, - float memoryBudgetInMB, - List parentLayerPointReaders) { - super(expression, queryId, memoryBudgetInMB); - - layerPointReaders = parentLayerPointReaders.toArray(new LayerPointReader[0]); - - dataTypes = new TSDataType[layerPointReaders.length]; - for (int i = 0; i < layerPointReaders.length; ++i) { - dataTypes[i] = layerPointReaders[i].getDataType(); - } - - timeHeap = new TimeSelector(layerPointReaders.length << 1, true); - - shouldMoveNext = new boolean[dataTypes.length]; - } - - @Override - public List getDataTypes() { - return Arrays.asList(dataTypes); - } - - @Override - public boolean hasNextRowInObjects() throws IOException { - if (cachedRow != null) { - return true; - } - - if (isFirstIteration) { - for (LayerPointReader reader : layerPointReaders) { - if (reader.isConstantPointReader()) { - continue; - } - try { - if (reader.next()) { - timeHeap.add(reader.currentTime()); - } - } catch (Exception e) { - throw new IOException(e); - } - } - isFirstIteration = false; - } - - if (timeHeap.isEmpty()) { - return false; - } - - final long minTime = timeHeap.pollFirst(); - - final int columnLength = layerPointReaders.length; - cachedRow = new Object[columnLength + 1]; - cachedRow[columnLength] = minTime; - - try { - for (int i = 0; i < columnLength; ++i) { - final LayerPointReader reader = layerPointReaders[i]; - if (!reader.next() - || (!reader.isConstantPointReader() && reader.currentTime() != minTime)) { - continue; - } - - if (!reader.isCurrentNull()) { - switch (reader.getDataType()) { - case INT32: - cachedRow[i] = reader.currentInt(); - break; - case INT64: - cachedRow[i] = reader.currentLong(); - break; - case FLOAT: - cachedRow[i] = reader.currentFloat(); - break; - case DOUBLE: - cachedRow[i] = reader.currentDouble(); - break; - case BOOLEAN: - cachedRow[i] = reader.currentBoolean(); - break; - case TEXT: - cachedRow[i] = reader.currentBinary(); - break; - default: - throw new UnSupportedDataTypeException("Unsupported data type."); - } - } - - reader.readyForNext(); - - if (!reader.isConstantPointReader() && reader.next()) { - timeHeap.add(reader.currentTime()); - } - } - } catch (QueryProcessException e) { - throw new IOException(e.getMessage()); - } - - return true; - } - - @Override - public YieldableState canYieldNextRowInObjects() throws Exception { - if (cachedRow != null) { - return YieldableState.YIELDABLE; - } - - if (isFirstIteration) { - for (LayerPointReader reader : layerPointReaders) { - if (reader.isConstantPointReader()) { - continue; - } - try { - final YieldableState yieldableState = reader.yield(); - if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - if (yieldableState == YieldableState.YIELDABLE) { - timeHeap.add(reader.currentTime()); - } - } catch (Exception e) { - throw new IOException(e); - } - } - isFirstIteration = false; - } else { - for (int i = 0, columnLength = layerPointReaders.length; i < columnLength; ++i) { - if (shouldMoveNext[i]) { - layerPointReaders[i].readyForNext(); - shouldMoveNext[i] = false; - } - } - - for (LayerPointReader layerPointReader : layerPointReaders) { - try { - if (!layerPointReader.isConstantPointReader()) { - final YieldableState yieldableState = layerPointReader.yield(); - if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - if (yieldableState == YieldableState.YIELDABLE) { - timeHeap.add(layerPointReader.currentTime()); - } - } - } catch (Exception e) { - throw new IOException(e); - } - } - } - - if (timeHeap.isEmpty()) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - final long minTime = timeHeap.pollFirst(); - - final int columnLength = layerPointReaders.length; - final Object[] row = new Object[columnLength + 1]; - row[columnLength] = minTime; - - try { - for (int i = 0; i < columnLength; ++i) { - final LayerPointReader reader = layerPointReaders[i]; - - final YieldableState yieldableState = reader.yield(); - if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - for (int j = 0; j <= i; ++j) { - shouldMoveNext[j] = false; - } - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - if (yieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { - continue; - } - if (!reader.isConstantPointReader() && reader.currentTime() != minTime) { - continue; - } - - if (!reader.isCurrentNull()) { - switch (reader.getDataType()) { - case INT32: - row[i] = reader.currentInt(); - break; - case INT64: - row[i] = reader.currentLong(); - break; - case FLOAT: - row[i] = reader.currentFloat(); - break; - case DOUBLE: - row[i] = reader.currentDouble(); - break; - case BOOLEAN: - row[i] = reader.currentBoolean(); - break; - case TEXT: - row[i] = reader.currentBinary(); - break; - default: - throw new UnSupportedDataTypeException("Unsupported data type."); - } - } - - shouldMoveNext[i] = true; - } - - cachedRow = row; - } catch (QueryProcessException e) { - throw new IOException(e.getMessage()); - } - - return YieldableState.YIELDABLE; - } - - @Override - public Object[] nextRowInObjects() { - final Object[] returnedRow = cachedRow; - cachedRow = null; - return returnedRow; - } - - @Override - public LayerPointReader constructPointReader() { - throw new UnsupportedOperationException(); - } - - @Override - public LayerRowReader constructRowReader() { - - return new LayerRowReader() { - - private final ElasticSerializableRowRecordListBackedMultiColumnRow row = - new ElasticSerializableRowRecordListBackedMultiColumnRow(dataTypes); - - private boolean hasCached = false; - private boolean currentNull = false; - - @Override - public YieldableState yield() throws Exception { - if (hasCached) { - return YieldableState.YIELDABLE; - } - - final YieldableState yieldableState = canYieldNextRowInObjects(); - if (yieldableState != YieldableState.YIELDABLE) { - return yieldableState; - } - - Object[] rowRecords = nextRowInObjects(); - currentNull = InputRowUtils.isAllNull(rowRecords); - row.setRowRecord(rowRecords); - hasCached = true; - return YieldableState.YIELDABLE; - } - - @Override - public boolean next() throws IOException { - if (hasCached) { - return true; - } - - if (!hasNextRowInObjects()) { - return false; - } - Object[] rowRecords = nextRowInObjects(); - currentNull = InputRowUtils.isAllNull(rowRecords); - row.setRowRecord(rowRecords); - hasCached = true; - return true; - } - - @Override - public void readyForNext() { - hasCached = false; - currentNull = false; - } - - @Override - public TSDataType[] getDataTypes() { - return dataTypes; - } - - @Override - public long currentTime() { - return row.getTime(); - } - - @Override - public Row currentRow() { - return row; - } - - @Override - public boolean isCurrentNull() { - return currentNull; - } - }; - } - - @Override - protected LayerRowWindowReader constructRowSlidingSizeWindowReader( - SlidingSizeWindowAccessStrategy strategy, float memoryBudgetInMB) - throws QueryProcessException { - - final IUDFInputDataSet udfInputDataSet = this; - - return new LayerRowWindowReader() { - - private final int windowSize = strategy.getWindowSize(); - private final int slidingStep = strategy.getSlidingStep(); - - private final ElasticSerializableRowRecordList rowRecordList = - new ElasticSerializableRowRecordList( - dataTypes, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); - private final ElasticSerializableRowRecordListBackedMultiColumnWindow window = - new ElasticSerializableRowRecordListBackedMultiColumnWindow(rowRecordList); - - private boolean hasCached = false; - private int beginIndex = -slidingStep; - - @Override - public YieldableState yield() throws Exception { - if (hasCached) { - return YieldableState.YIELDABLE; - } - - beginIndex += slidingStep; - int endIndex = beginIndex + windowSize; - if (beginIndex < 0 || endIndex < 0) { - LOGGER.warn( - "LayerRowWindowReader index overflow. beginIndex: {}, endIndex: {}, windowSize: {}.", - beginIndex, - endIndex, - windowSize); - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - final int rowsToBeCollected = endIndex - rowRecordList.size(); - if (0 < rowsToBeCollected) { - final YieldableState yieldableState = - LayerCacheUtils.yieldRows(udfInputDataSet, rowRecordList, rowsToBeCollected); - - if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - beginIndex -= slidingStep; - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - - if (rowRecordList.size() <= beginIndex) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - window.seek( - beginIndex, - rowRecordList.size(), - rowRecordList.getTime(beginIndex), - rowRecordList.getTime(rowRecordList.size() - 1)); - } else { - window.seek( - beginIndex, - endIndex, - rowRecordList.getTime(beginIndex), - rowRecordList.getTime(endIndex - 1)); - } - - hasCached = true; - return YieldableState.YIELDABLE; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - if (hasCached) { - return true; - } - - beginIndex += slidingStep; - int endIndex = beginIndex + windowSize; - if (beginIndex < 0 || endIndex < 0) { - LOGGER.warn( - "LayerRowWindowReader index overflow. beginIndex: {}, endIndex: {}, windowSize: {}.", - beginIndex, - endIndex, - windowSize); - return false; - } - - int rowsToBeCollected = endIndex - rowRecordList.size(); - if (0 < rowsToBeCollected) { - LayerCacheUtils.cacheRows(udfInputDataSet, rowRecordList, rowsToBeCollected); - if (rowRecordList.size() <= beginIndex) { - return false; - } - - window.seek( - beginIndex, - rowRecordList.size(), - rowRecordList.getTime(beginIndex), - rowRecordList.getTime(rowRecordList.size() - 1)); - } else { - window.seek( - beginIndex, - endIndex, - rowRecordList.getTime(beginIndex), - rowRecordList.getTime(endIndex - 1)); - } - - hasCached = true; - return true; - } - - @Override - public void readyForNext() { - hasCached = false; - - rowRecordList.setEvictionUpperBound(beginIndex + 1); - } - - @Override - public TSDataType[] getDataTypes() { - return dataTypes; - } - - @Override - public RowWindow currentWindow() { - return window; - } - }; - } - - @Override - protected LayerRowWindowReader constructRowSlidingTimeWindowReader( - SlidingTimeWindowAccessStrategy strategy, float memoryBudgetInMB) - throws QueryProcessException { - - final long timeInterval = strategy.getTimeInterval(); - final long slidingStep = strategy.getSlidingStep(); - final long displayWindowEnd = strategy.getDisplayWindowEnd(); - - final IUDFInputDataSet udfInputDataSet = this; - final ElasticSerializableRowRecordList rowRecordList = - new ElasticSerializableRowRecordList( - dataTypes, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); - final ElasticSerializableRowRecordListBackedMultiColumnWindow window = - new ElasticSerializableRowRecordListBackedMultiColumnWindow(rowRecordList); - - return new LayerRowWindowReader() { - - private boolean isFirstIteration = true; - private boolean hasAtLeastOneRow = false; - - private boolean hasCached = false; - private long nextWindowTimeBegin = strategy.getDisplayWindowBegin(); - private int nextIndexBegin = 0; - - @Override - public YieldableState yield() throws Exception { - if (isFirstIteration) { - if (rowRecordList.size() == 0) { - final YieldableState yieldableState = - LayerCacheUtils.yieldRow(udfInputDataSet, rowRecordList); - if (yieldableState != YieldableState.YIELDABLE) { - return yieldableState; - } - if (nextWindowTimeBegin == Long.MIN_VALUE) { - // display window begin should be set to the same as the min timestamp of the query - // result set - nextWindowTimeBegin = rowRecordList.getTime(0); - } - } - hasAtLeastOneRow = rowRecordList.size() != 0; - isFirstIteration = false; - } - - if (hasCached) { - return YieldableState.YIELDABLE; - } - - if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); - while (rowRecordList.getTime(rowRecordList.size() - 1) < nextWindowTimeEnd) { - final YieldableState yieldableState = - LayerCacheUtils.yieldRow(udfInputDataSet, rowRecordList); - if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - if (yieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { - break; - } - } - - for (int i = nextIndexBegin; i < rowRecordList.size(); ++i) { - if (nextWindowTimeBegin <= rowRecordList.getTime(i)) { - nextIndexBegin = i; - break; - } - if (i == rowRecordList.size() - 1) { - nextIndexBegin = rowRecordList.size(); - } - } - - int nextIndexEnd = rowRecordList.size(); - for (int i = nextIndexBegin; i < rowRecordList.size(); ++i) { - if (nextWindowTimeEnd <= rowRecordList.getTime(i)) { - nextIndexEnd = i; - break; - } - } - if ((nextIndexEnd == nextIndexBegin) - && nextWindowTimeEnd < rowRecordList.getTime(rowRecordList.size() - 1)) { - window.setEmptyWindow(nextWindowTimeBegin, nextWindowTimeEnd); - return YieldableState.YIELDABLE; - } - window.seek( - nextIndexBegin, - nextIndexEnd, - nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); - - hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == rowRecordList.size()); - return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - if (isFirstIteration) { - if (rowRecordList.size() == 0 - && LayerCacheUtils.cacheRow(udfInputDataSet, rowRecordList) - && nextWindowTimeBegin == Long.MIN_VALUE) { - // display window begin should be set to the same as the min timestamp of the query - // result set - nextWindowTimeBegin = rowRecordList.getTime(0); - } - hasAtLeastOneRow = rowRecordList.size() != 0; - isFirstIteration = false; - } - - if (hasCached) { - return true; - } - - if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { - return false; - } - - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); - while (rowRecordList.getTime(rowRecordList.size() - 1) < nextWindowTimeEnd) { - if (!LayerCacheUtils.cacheRow(udfInputDataSet, rowRecordList)) { - break; - } - } - - for (int i = nextIndexBegin; i < rowRecordList.size(); ++i) { - if (nextWindowTimeBegin <= rowRecordList.getTime(i)) { - nextIndexBegin = i; - break; - } - if (i == rowRecordList.size() - 1) { - nextIndexBegin = rowRecordList.size(); - } - } - - int nextIndexEnd = rowRecordList.size(); - for (int i = nextIndexBegin; i < rowRecordList.size(); ++i) { - if (nextWindowTimeEnd <= rowRecordList.getTime(i)) { - nextIndexEnd = i; - break; - } - } - - if ((nextIndexEnd == nextIndexBegin) - && nextWindowTimeEnd < rowRecordList.getTime(rowRecordList.size() - 1)) { - window.setEmptyWindow(nextWindowTimeBegin, nextWindowTimeEnd); - return true; - } - - window.seek( - nextIndexBegin, - nextIndexEnd, - nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); - - hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == rowRecordList.size()); - return hasCached; - } - - @Override - public void readyForNext() { - hasCached = false; - nextWindowTimeBegin += slidingStep; - - rowRecordList.setEvictionUpperBound(nextIndexBegin + 1); - } - - @Override - public TSDataType[] getDataTypes() { - return dataTypes; - } - - @Override - public RowWindow currentWindow() { - return window; - } - }; - } - - @Override - protected LayerRowWindowReader constructRowSessionTimeWindowReader( - SessionTimeWindowAccessStrategy strategy, float memoryBudgetInMB) - throws QueryProcessException { - final long displayWindowBegin = strategy.getDisplayWindowBegin(); - final long displayWindowEnd = strategy.getDisplayWindowEnd(); - final long sessionTimeGap = strategy.getSessionTimeGap(); - - final IUDFInputDataSet udfInputDataSet = this; - final ElasticSerializableRowRecordList rowRecordList = - new ElasticSerializableRowRecordList( - dataTypes, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); - final ElasticSerializableRowRecordListBackedMultiColumnWindow window = - new ElasticSerializableRowRecordListBackedMultiColumnWindow(rowRecordList); - - return new LayerRowWindowReader() { - - private boolean isFirstIteration = true; - private boolean hasAtLeastOneRow = false; - - private long nextWindowTimeBegin = displayWindowBegin; - private long nextWindowTimeEnd = 0; - private int nextIndexBegin = 0; - private int nextIndexEnd = 1; - - @Override - public YieldableState yield() throws Exception { - if (isFirstIteration) { - if (rowRecordList.size() == 0) { - final YieldableState yieldableState = - LayerCacheUtils.yieldRow(udfInputDataSet, rowRecordList); - if (yieldableState != YieldableState.YIELDABLE) { - return yieldableState; - } - } - nextWindowTimeBegin = Math.max(displayWindowBegin, rowRecordList.getTime(0)); - hasAtLeastOneRow = rowRecordList.size() != 0; - isFirstIteration = false; - } - - if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - while (rowRecordList.getTime(rowRecordList.size() - 1) < displayWindowEnd) { - final YieldableState yieldableState = - LayerCacheUtils.yieldRow(udfInputDataSet, rowRecordList); - if (yieldableState == YieldableState.YIELDABLE) { - if (rowRecordList.getTime(rowRecordList.size() - 2) >= displayWindowBegin - && rowRecordList.getTime(rowRecordList.size() - 1) - - rowRecordList.getTime(rowRecordList.size() - 2) - > sessionTimeGap) { - nextIndexEnd = rowRecordList.size() - 1; - break; - } else { - nextIndexEnd++; - } - } else if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } else if (yieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { - nextIndexEnd = rowRecordList.size(); - break; - } - } - - nextWindowTimeEnd = rowRecordList.getTime(nextIndexEnd - 1); - - if (nextIndexBegin == nextIndexEnd) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - // Only if encounter user set the strategy's displayWindowBegin, which will go into the for - // loop to find the true index of the first window begin. - // For other situation, we will only go into if (nextWindowTimeBegin <= tvList.getTime(i)) - // once. - for (int i = nextIndexBegin; i < rowRecordList.size(); ++i) { - if (nextWindowTimeBegin <= rowRecordList.getTime(i)) { - nextIndexBegin = i; - break; - } - // The first window's beginning time is greater than all the timestamp of the query result - // set - if (i == rowRecordList.size() - 1) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - } - - window.seek(nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, nextWindowTimeEnd); - - return YieldableState.YIELDABLE; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - return false; - } - - @Override - public void readyForNext() throws IOException, QueryProcessException { - if (nextIndexEnd < rowRecordList.size()) { - nextWindowTimeBegin = rowRecordList.getTime(nextIndexEnd); - } - rowRecordList.setEvictionUpperBound(nextIndexBegin + 1); - nextIndexBegin = nextIndexEnd; - } - - @Override - public TSDataType[] getDataTypes() { - return dataTypes; - } - - @Override - public RowWindow currentWindow() { - return window; - } - }; - } - - @Override - protected LayerRowWindowReader constructRowStateWindowReader( - StateWindowAccessStrategy strategy, float memoryBudgetInMB) { - throw new UnsupportedOperationException( - "StateWindowAccessStrategy only support one input series for now."); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java new file mode 100644 index 000000000000..0012b9936210 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java @@ -0,0 +1,666 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.intermediate; + +import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.plan.expression.Expression; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerRowWindowReader; +import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; +import org.apache.iotdb.db.queryengine.transformation.dag.adapter.ElasticSerializableRowRecordListBackedMultiColumnWindow; +import org.apache.iotdb.db.queryengine.transformation.dag.input.IUDFInputDataSet; +import org.apache.iotdb.db.queryengine.transformation.dag.util.LayerCacheUtils; +import org.apache.iotdb.db.queryengine.transformation.datastructure.TVColumns; +import org.apache.iotdb.db.queryengine.transformation.datastructure.iterator.RowListForwardIterator; +import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowList; +import org.apache.iotdb.db.utils.datastructure.TimeSelector; +import org.apache.iotdb.udf.api.access.RowWindow; +import org.apache.iotdb.udf.api.customizer.strategy.SessionTimeWindowAccessStrategy; +import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; +import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; +import org.apache.iotdb.udf.api.customizer.strategy.StateWindowAccessStrategy; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +public class MultiInputLayer extends IntermediateLayer implements IUDFInputDataSet { + + private static final Logger LOGGER = LoggerFactory.getLogger(MultiInputLayer.class); + + private final LayerReader[] layerReaders; + private final TSDataType[] dataTypes; + private final TimeSelector timeHeap; + + private final TVColumns[] inputTVColumnsList; + private final int[] currentConsumedIndexes; + private final int[] nextConsumedIndexes; + + private final TsBlockBuilder tsBlockBuilder; + private TsBlock cachedTsBlock = null; + + public MultiInputLayer( + Expression expression, + String queryId, + float memoryBudgetInMB, + List parentLayerReaders) { + super(expression, queryId, memoryBudgetInMB); + + layerReaders = parentLayerReaders.toArray(new LayerReader[0]); + currentConsumedIndexes = new int[layerReaders.length]; + nextConsumedIndexes = new int[layerReaders.length]; + inputTVColumnsList = new TVColumns[layerReaders.length]; + + dataTypes = new TSDataType[layerReaders.length]; + for (int i = 0; i < layerReaders.length; ++i) { + dataTypes[i] = layerReaders[i].getDataTypes()[0]; + } + tsBlockBuilder = new TsBlockBuilder(Arrays.asList(dataTypes)); + + timeHeap = new TimeSelector(layerReaders.length << 1, true); + } + + @Override + public List getDataTypes() { + return Arrays.asList(dataTypes); + } + + @Override + public YieldableState yield() throws Exception { + tsBlockBuilder.reset(); + + // Fill input columns + YieldableState state = updateInputColumns(); + if (state != YieldableState.YIELDABLE) { + return state; + } + + // Choose minimum end time as this iteration's end time + long endTime = Long.MAX_VALUE; + for (TVColumns tvColumns : inputTVColumnsList) { + if (!tvColumns.isConstant()) { + long tvColumnsEndTime = tvColumns.getEndTime(); + endTime = Math.min(tvColumnsEndTime, endTime); + } + } + + // Construct row for given time from time heap + long currentTime = -1; + TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder(); + while (currentTime != endTime) { + currentTime = timeHeap.pollFirst(); + + timeBuilder.writeLong(currentTime); // Time + appendRowInBuilder(currentTime); // Values + tsBlockBuilder.declarePosition(); + + updateTimeHeap(); + } + + cachedTsBlock = tsBlockBuilder.build(); + return YieldableState.YIELDABLE; + } + + private YieldableState updateInputColumns() throws Exception { + for (int i = 0; i < layerReaders.length; i++) { + // Skip TVColumns that still remains some data + if (canSkipInputTVColumns(i)) { + continue; + } + // Prepare data for TVColumns without data + YieldableState state = layerReaders[i].yield(); + if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + return state; + } else if (state == YieldableState.YIELDABLE) { + Column[] columns = layerReaders[i].current(); + if (layerReaders[i].isConstantPointReader()) { + inputTVColumnsList[i] = new TVColumns(columns[0]); + } else { + inputTVColumnsList[i] = new TVColumns((TimeColumn) columns[1], columns[0]); + timeHeap.add(((TimeColumn) columns[1]).getStartTime()); + } + + currentConsumedIndexes[i] = 0; + layerReaders[i].consumedAll(); + } // Do nothing for YieldableState.NOT_YIELDABLE_NO_MORE_DATA + } + + return timeHeap.isEmpty() + ? YieldableState.NOT_YIELDABLE_NO_MORE_DATA + : YieldableState.YIELDABLE; + } + + private boolean canSkipInputTVColumns(int index) { + return inputTVColumnsList[index] != null && !hasConsumedAll(index); + } + + private boolean hasConsumedAll(int index) { + return inputTVColumnsList[index].getPositionCount() == currentConsumedIndexes[index]; + } + + private void appendRowInBuilder(long time) { + for (int i = 0; i < inputTVColumnsList.length; i++) { + ColumnBuilder builder = tsBlockBuilder.getColumnBuilder(i); + if (hasConsumedAll(i)) { + builder.appendNull(); + continue; + } + + TVColumns tvColumns = inputTVColumnsList[i]; + if (tvColumns.isConstant()) { + // TODO: maybe one constant is enough, notice this 0 + builder.write(tvColumns.getValueColumn(), 0); + continue; + } + + int currentIndex = currentConsumedIndexes[i]; + long currentTime = tvColumns.getTimeByIndex(currentIndex); + if (currentTime == time) { + if (tvColumns.getValueColumn().isNull(currentIndex)) { + builder.appendNull(); + } else { + builder.write(tvColumns.getValueColumn(), currentIndex); + } + nextConsumedIndexes[i] = currentIndex + 1; + } else { + builder.appendNull(); + } + } + } + + private void updateTimeHeap() { + for (int i = 0; i < inputTVColumnsList.length; i++) { + if (currentConsumedIndexes[i] != nextConsumedIndexes[i]) { + currentConsumedIndexes[i] = nextConsumedIndexes[i]; + // Add remaining time to time heap + if (!hasConsumedAll(i)) { + timeHeap.add(inputTVColumnsList[i].getTimeByIndex(currentConsumedIndexes[i])); + } + } + } + } + + @Override + public Column[] currentBlock() { + Column[] ret = cachedTsBlock.getAllColumns(); + cachedTsBlock = null; + return ret; + } + + @Override + public LayerReader constructReader() { + return new LayerReader() { + @Override + public boolean isConstantPointReader() { + return false; + } + + @Override + public void consumedAll() { + // Currently do nothing + } + + @Override + public Column[] current() { + return MultiInputLayer.this.currentBlock(); + } + + @Override + public TSDataType[] getDataTypes() { + return dataTypes; + } + + @Override + public YieldableState yield() throws Exception { + return MultiInputLayer.this.yield(); + } + }; + } + + @Override + protected LayerRowWindowReader constructRowSlidingSizeWindowReader( + SlidingSizeWindowAccessStrategy strategy, float memoryBudgetInMB) + throws QueryProcessException { + final IUDFInputDataSet udfInputDataSet = this; + + return new LayerRowWindowReader() { + + private final int windowSize = strategy.getWindowSize(); + private final int slidingStep = strategy.getSlidingStep(); + + private final ElasticSerializableRowList rowRecordList = + new ElasticSerializableRowList(dataTypes, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); + private final ElasticSerializableRowRecordListBackedMultiColumnWindow window = + new ElasticSerializableRowRecordListBackedMultiColumnWindow(rowRecordList); + + private boolean hasCached = false; + private int beginIndex = -slidingStep; + + @Override + public YieldableState yield() throws Exception { + if (hasCached) { + return YieldableState.YIELDABLE; + } + + beginIndex += slidingStep; + int endIndex = beginIndex + windowSize; + if (beginIndex < 0 || endIndex < 0) { + LOGGER.warn( + "LayerRowWindowReader index overflow. beginIndex: {}, endIndex: {}, windowSize: {}.", + beginIndex, + endIndex, + windowSize); + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + final int pointsToBeCollected = endIndex - rowRecordList.size(); + if (pointsToBeCollected > 0) { + final YieldableState yieldableState = + LayerCacheUtils.yieldRows(udfInputDataSet, rowRecordList, pointsToBeCollected); + if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + beginIndex -= slidingStep; + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } + + if (rowRecordList.size() <= beginIndex) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + // TVList's size may be less than endIndex + // When parent layer reader has no more data + endIndex = Math.min(endIndex, rowRecordList.size()); + } + + window.seek( + beginIndex, + endIndex, + rowRecordList.getTime(beginIndex), + rowRecordList.getTime(endIndex - 1)); + + hasCached = true; + return YieldableState.YIELDABLE; + } + + @Override + public void readyForNext() { + hasCached = false; + + rowRecordList.setEvictionUpperBound(beginIndex + 1); + } + + @Override + public TSDataType[] getDataTypes() { + return dataTypes; + } + + @Override + public RowWindow currentWindow() { + return window; + } + }; + } + + @Override + protected LayerRowWindowReader constructRowSlidingTimeWindowReader( + SlidingTimeWindowAccessStrategy strategy, float memoryBudgetInMB) + throws QueryProcessException { + + final long timeInterval = strategy.getTimeInterval(); + final long slidingStep = strategy.getSlidingStep(); + final long displayWindowEnd = strategy.getDisplayWindowEnd(); + + final IUDFInputDataSet udfInputDataSet = this; + final ElasticSerializableRowList rowRecordList = + new ElasticSerializableRowList(dataTypes, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); + final ElasticSerializableRowRecordListBackedMultiColumnWindow window = + new ElasticSerializableRowRecordListBackedMultiColumnWindow(rowRecordList); + + return new LayerRowWindowReader() { + + private boolean isFirstIteration = true; + private boolean hasAtLeastOneRow = false; + + private boolean hasCached = false; + private long nextWindowTimeBegin = strategy.getDisplayWindowBegin(); + private int nextIndexBegin = 0; + private int nextIndexEnd = 0; + private long currentEndTime = Long.MAX_VALUE; + + private final RowListForwardIterator beginIterator = rowRecordList.constructIterator(); + private TimeColumn cachedBeginTimeColumn; + private int cachedBeginConsumed; + + private TimeColumn cachedEndTimeColumn; + private int cachedEndConsumed; + + @Override + public YieldableState yield() throws Exception { + if (isFirstIteration) { + if (rowRecordList.size() == 0) { + final YieldableState state = udfInputDataSet.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + + Column[] columns = udfInputDataSet.currentBlock(); + TimeColumn times = (TimeColumn) columns[columns.length - 1]; + + rowRecordList.put(columns); + + cachedEndTimeColumn = times; + } + if (nextWindowTimeBegin == Long.MIN_VALUE) { + // display window begin should be set to the same as the min timestamp of the query + // result set + nextWindowTimeBegin = cachedEndTimeColumn.getStartTime(); + } + hasAtLeastOneRow = rowRecordList.size() != 0; + if (hasAtLeastOneRow) { + currentEndTime = cachedEndTimeColumn.getEndTime(); + } + isFirstIteration = false; + } + + if (hasCached) { + return YieldableState.YIELDABLE; + } + + if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); + while (currentEndTime < nextWindowTimeEnd) { + final YieldableState state = udfInputDataSet.yield(); + if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } + if (state == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { + break; + } + // Generate data + Column[] columns = udfInputDataSet.currentBlock(); + TimeColumn times = (TimeColumn) columns[columns.length - 1]; + // Put data into container + rowRecordList.put(columns); + currentEndTime = times.getEndTime(); + // Increase nextIndexEnd + nextIndexEnd += cachedEndTimeColumn.getPositionCount() - cachedEndConsumed; + // Update cache + cachedEndTimeColumn = times; + cachedEndConsumed = 0; + } + + // Set nextIndexEnd field + while (cachedEndConsumed < cachedEndTimeColumn.getPositionCount()) { + if (cachedEndTimeColumn.getLong(cachedEndConsumed) >= nextWindowTimeEnd) { + break; + } + cachedEndConsumed++; + nextIndexEnd++; + } + + // Set nextIndexBegin field + boolean findNextIndexBegin = false; + while (!findNextIndexBegin) { + while (cachedBeginTimeColumn != null + && cachedBeginConsumed < cachedBeginTimeColumn.getPositionCount()) { + if (cachedBeginTimeColumn.getLong(cachedBeginConsumed) >= nextWindowTimeBegin) { + findNextIndexBegin = true; + break; + } + cachedBeginConsumed++; + nextIndexBegin++; + } + + if (!findNextIndexBegin) { + if (beginIterator.hasNext()) { + beginIterator.next(); + + cachedBeginConsumed = 0; + Column[] columns = beginIterator.currentBlock(); + cachedBeginTimeColumn = (TimeColumn) columns[columns.length - 1]; + } else { + // No more data + // Set nextIndexBegin to list's size + findNextIndexBegin = true; + } + } + } + + if ((nextIndexEnd == nextIndexBegin) + && nextWindowTimeEnd < cachedEndTimeColumn.getEndTime()) { + window.setEmptyWindow(nextWindowTimeBegin, nextWindowTimeEnd); + return YieldableState.YIELDABLE; + } + + window.seek( + nextIndexBegin, + nextIndexEnd, + nextWindowTimeBegin, + nextWindowTimeBegin + timeInterval - 1); + + hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == rowRecordList.size()); + return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + @Override + public void readyForNext() { + hasCached = false; + nextWindowTimeBegin += slidingStep; + + rowRecordList.setEvictionUpperBound(nextIndexBegin + 1); + } + + @Override + public TSDataType[] getDataTypes() { + return dataTypes; + } + + @Override + public RowWindow currentWindow() { + return window; + } + }; + } + + @Override + protected LayerRowWindowReader constructRowSessionTimeWindowReader( + SessionTimeWindowAccessStrategy strategy, float memoryBudgetInMB) + throws QueryProcessException { + final long displayWindowBegin = strategy.getDisplayWindowBegin(); + final long displayWindowEnd = strategy.getDisplayWindowEnd(); + final long sessionTimeGap = strategy.getSessionTimeGap(); + + final IUDFInputDataSet udfInputDataSet = this; + final ElasticSerializableRowList rowRecordList = + new ElasticSerializableRowList(dataTypes, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); + final ElasticSerializableRowRecordListBackedMultiColumnWindow window = + new ElasticSerializableRowRecordListBackedMultiColumnWindow(rowRecordList); + + return new LayerRowWindowReader() { + private boolean isFirstIteration = true; + private boolean hasAtLeastOneRow = false; + + private long nextWindowTimeBegin = displayWindowBegin; + private long nextWindowTimeEnd = 0; + private int nextIndexBegin = 0; + private int nextIndexEnd = 0; + + private TimeColumn cachedTimes; + private int cachedConsumed; + + @Override + public YieldableState yield() throws Exception { + if (isFirstIteration) { + YieldableState state = yieldInFirstIteration(); + if (state != YieldableState.YIELDABLE) { + return state; + } + } + + if (!hasAtLeastOneRow || nextWindowTimeBegin >= displayWindowEnd) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + // Set nextIndexEnd + long curTime = cachedTimes.getLong(cachedConsumed); + if (cachedConsumed < cachedTimes.getPositionCount()) { + nextIndexEnd++; + cachedConsumed++; + } + boolean findWindow = false; + // Find target window or no more data to exit + while (!findWindow) { + while (cachedConsumed < cachedTimes.getPositionCount()) { + long nextTime = cachedTimes.getLong(cachedConsumed); + + if (curTime >= displayWindowEnd) { + nextIndexEnd--; + findWindow = true; + break; + } + + if (nextTime - curTime > sessionTimeGap) { + findWindow = true; + break; + } + nextIndexEnd++; + cachedConsumed++; + curTime = nextTime; + } + + if (!findWindow) { + if (cachedTimes.getEndTime() < displayWindowEnd) { + YieldableState state = yieldAndCache(); + if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } else if (state == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { + break; + } + } + } + } + // Set nextWindowTimeEnd + nextWindowTimeEnd = rowRecordList.getTime(nextIndexEnd - 1); + + if (nextIndexBegin == nextIndexEnd) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + window.seek(nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, nextWindowTimeEnd); + return YieldableState.YIELDABLE; + } + + private YieldableState yieldInFirstIteration() throws Exception { + // Yield initial data in first iteration + if (rowRecordList.size() == 0) { + YieldableState state = yieldAndCache(); + if (state != YieldableState.YIELDABLE) { + return state; + } + } + // Initialize essential information + nextWindowTimeBegin = Math.max(displayWindowBegin, cachedTimes.getStartTime()); + hasAtLeastOneRow = rowRecordList.size() != 0; + isFirstIteration = false; + + // Set initial nextIndexBegin + long currentEndTime = cachedTimes.getEndTime(); + // Find corresponding block + while (currentEndTime < nextWindowTimeBegin) { + // Consume all data + cachedConsumed = cachedTimes.getPositionCount(); + nextIndexBegin += cachedConsumed; + + YieldableState state = yieldAndCache(); + if (state != YieldableState.YIELDABLE) { + // Cannot find nextIndexBegin + // Set nextIndexEnd to nextIndexBegin and exit + nextIndexEnd = nextIndexBegin; + return state; + } + } + // Find nextIndexBegin in block + while (cachedConsumed < cachedTimes.getPositionCount()) { + if (cachedTimes.getLong(cachedConsumed) >= nextWindowTimeBegin) { + break; + } + cachedConsumed++; + nextIndexBegin++; + } + nextIndexEnd = nextIndexBegin; + + return YieldableState.YIELDABLE; + } + + private YieldableState yieldAndCache() throws Exception { + final YieldableState state = udfInputDataSet.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + Column[] columns = udfInputDataSet.currentBlock(); + TimeColumn times = (TimeColumn) columns[columns.length - 1]; + + rowRecordList.put(columns); + + cachedTimes = times; + cachedConsumed = 0; + + return YieldableState.YIELDABLE; + } + + @Override + public void readyForNext() throws IOException { + if (nextIndexEnd < rowRecordList.size()) { + nextWindowTimeBegin = rowRecordList.getTime(nextIndexEnd); + } + rowRecordList.setEvictionUpperBound(nextIndexBegin + 1); + nextIndexBegin = nextIndexEnd; + } + + @Override + public TSDataType[] getDataTypes() { + return dataTypes; + } + + @Override + public RowWindow currentWindow() { + return window; + } + }; + } + + @Override + protected LayerRowWindowReader constructRowStateWindowReader( + StateWindowAccessStrategy strategy, float memoryBudgetInMB) { + throw new UnsupportedOperationException( + "StateWindowAccessStrategy only support one input series for now."); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputColumnMultiReferenceIntermediateLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputColumnMultiReferenceIntermediateLayer.java deleted file mode 100644 index 1656b3ed2ca7..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputColumnMultiReferenceIntermediateLayer.java +++ /dev/null @@ -1,797 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.intermediate; - -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.plan.expression.Expression; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; -import org.apache.iotdb.db.queryengine.transformation.api.LayerRowReader; -import org.apache.iotdb.db.queryengine.transformation.api.LayerRowWindowReader; -import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; -import org.apache.iotdb.db.queryengine.transformation.dag.adapter.ElasticSerializableTVListBackedSingleColumnRow; -import org.apache.iotdb.db.queryengine.transformation.dag.adapter.ElasticSerializableTVListBackedSingleColumnWindow; -import org.apache.iotdb.db.queryengine.transformation.dag.memory.SafetyLine; -import org.apache.iotdb.db.queryengine.transformation.dag.memory.SafetyLine.SafetyPile; -import org.apache.iotdb.db.queryengine.transformation.dag.util.LayerCacheUtils; -import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.ElasticSerializableTVList; -import org.apache.iotdb.db.queryengine.transformation.datastructure.util.ValueRecorder; -import org.apache.iotdb.udf.api.access.Row; -import org.apache.iotdb.udf.api.access.RowWindow; -import org.apache.iotdb.udf.api.customizer.strategy.SessionTimeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.StateWindowAccessStrategy; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -public class SingleInputColumnMultiReferenceIntermediateLayer extends IntermediateLayer { - - private static final Logger LOGGER = - LoggerFactory.getLogger(SingleInputColumnMultiReferenceIntermediateLayer.class); - - private final LayerPointReader parentLayerPointReader; - private final TSDataType parentLayerPointReaderDataType; - private final boolean isParentLayerPointReaderConstant; - private final ElasticSerializableTVList tvList; - private final SafetyLine safetyLine; - - public SingleInputColumnMultiReferenceIntermediateLayer( - Expression expression, - String queryId, - float memoryBudgetInMB, - LayerPointReader parentLayerPointReader) { - super(expression, queryId, memoryBudgetInMB); - this.parentLayerPointReader = parentLayerPointReader; - - parentLayerPointReaderDataType = parentLayerPointReader.getDataType(); - isParentLayerPointReaderConstant = parentLayerPointReader.isConstantPointReader(); - tvList = - ElasticSerializableTVList.newElasticSerializableTVList( - parentLayerPointReaderDataType, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); - safetyLine = new SafetyLine(); - } - - @Override - public LayerPointReader constructPointReader() { - - return new LayerPointReader() { - - private final SafetyPile safetyPile = safetyLine.addSafetyPile(); - - private boolean hasCached = false; - private int currentPointIndex = -1; - - @Override - public boolean isConstantPointReader() { - return isParentLayerPointReaderConstant; - } - - @Override - public YieldableState yield() throws Exception { - if (hasCached) { - return YieldableState.YIELDABLE; - } - - if (currentPointIndex < tvList.size() - 1) { - ++currentPointIndex; - hasCached = true; - return YieldableState.YIELDABLE; - } - - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList); - if (yieldableState == YieldableState.YIELDABLE) { - ++currentPointIndex; - hasCached = true; - } - return yieldableState; - } - - @Override - public boolean next() throws QueryProcessException, IOException { - if (!hasCached - && (currentPointIndex < tvList.size() - 1 - || LayerCacheUtils.cachePoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList))) { - ++currentPointIndex; - hasCached = true; - } - - return hasCached; - } - - @Override - public void readyForNext() { - hasCached = false; - - safetyPile.moveForwardTo(currentPointIndex + 1); - tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); - } - - @Override - public TSDataType getDataType() { - return parentLayerPointReaderDataType; - } - - @Override - public long currentTime() throws IOException { - return tvList.getTime(currentPointIndex); - } - - @Override - public int currentInt() throws IOException { - return tvList.getInt(currentPointIndex); - } - - @Override - public long currentLong() throws IOException { - return tvList.getLong(currentPointIndex); - } - - @Override - public float currentFloat() throws IOException { - return tvList.getFloat(currentPointIndex); - } - - @Override - public double currentDouble() throws IOException { - return tvList.getDouble(currentPointIndex); - } - - @Override - public boolean currentBoolean() throws IOException { - return tvList.getBoolean(currentPointIndex); - } - - @Override - public Binary currentBinary() throws IOException { - return tvList.getBinary(currentPointIndex); - } - - @Override - public boolean isCurrentNull() throws IOException { - return tvList.isNull(currentPointIndex); - } - }; - } - - @Override - public LayerRowReader constructRowReader() { - - return new LayerRowReader() { - - private final SafetyPile safetyPile = safetyLine.addSafetyPile(); - private final ElasticSerializableTVListBackedSingleColumnRow row = - new ElasticSerializableTVListBackedSingleColumnRow(tvList, -1); - - private boolean hasCached = false; - private int currentRowIndex = -1; - - @Override - public YieldableState yield() throws Exception { - if (hasCached) { - return YieldableState.YIELDABLE; - } - - if (currentRowIndex < tvList.size() - 1) { - row.seek(++currentRowIndex); - hasCached = true; - return YieldableState.YIELDABLE; - } - - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList); - if (yieldableState == YieldableState.YIELDABLE) { - row.seek(++currentRowIndex); - hasCached = true; - } - return yieldableState; - } - - @Override - public boolean next() throws QueryProcessException, IOException { - if (!hasCached - && ((currentRowIndex < tvList.size() - 1) - || LayerCacheUtils.cachePoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList))) { - row.seek(++currentRowIndex); - hasCached = true; - } - - return hasCached; - } - - @Override - public void readyForNext() { - hasCached = false; - - safetyPile.moveForwardTo(currentRowIndex + 1); - tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); - } - - @Override - public TSDataType[] getDataTypes() { - return new TSDataType[] {parentLayerPointReaderDataType}; - } - - @Override - public long currentTime() throws IOException { - return row.getTime(); - } - - @Override - public Row currentRow() { - return row; - } - - @Override - public boolean isCurrentNull() { - return tvList.isNull(currentRowIndex); - } - }; - } - - @Override - protected LayerRowWindowReader constructRowSlidingSizeWindowReader( - SlidingSizeWindowAccessStrategy strategy, float memoryBudgetInMB) { - - return new LayerRowWindowReader() { - - private final int windowSize = strategy.getWindowSize(); - private final int slidingStep = strategy.getSlidingStep(); - - private final SafetyPile safetyPile = safetyLine.addSafetyPile(); - private final ElasticSerializableTVListBackedSingleColumnWindow window = - new ElasticSerializableTVListBackedSingleColumnWindow(tvList); - - private boolean hasCached = false; - private int beginIndex = -slidingStep; - - @Override - public YieldableState yield() throws Exception { - if (hasCached) { - return YieldableState.YIELDABLE; - } - - beginIndex += slidingStep; - final int endIndex = beginIndex + windowSize; - if (beginIndex < 0 || endIndex < 0) { - LOGGER.warn( - "LayerRowWindowReader index overflow. beginIndex: {}, endIndex: {}, windowSize: {}.", - beginIndex, - endIndex, - windowSize); - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - final int pointsToBeCollected = endIndex - tvList.size(); - if (0 < pointsToBeCollected) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoints( - parentLayerPointReaderDataType, - parentLayerPointReader, - tvList, - pointsToBeCollected); - if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - beginIndex -= slidingStep; - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - - if (tvList.size() <= beginIndex) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - window.seek( - beginIndex, - tvList.size(), - tvList.getTime(beginIndex), - tvList.getTime(tvList.size() - 1)); - } else { - window.seek( - beginIndex, endIndex, tvList.getTime(beginIndex), tvList.getTime(endIndex - 1)); - } - - hasCached = true; - return YieldableState.YIELDABLE; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - if (hasCached) { - return true; - } - - beginIndex += slidingStep; - int endIndex = beginIndex + windowSize; - if (beginIndex < 0 || endIndex < 0) { - LOGGER.warn( - "LayerRowWindowReader index overflow. beginIndex: {}, endIndex: {}, windowSize: {}.", - beginIndex, - endIndex, - windowSize); - return false; - } - - int pointsToBeCollected = endIndex - tvList.size(); - if (0 < pointsToBeCollected) { - LayerCacheUtils.cachePoints( - parentLayerPointReaderDataType, parentLayerPointReader, tvList, pointsToBeCollected); - if (tvList.size() <= beginIndex) { - return false; - } - - window.seek( - beginIndex, - tvList.size(), - tvList.getTime(beginIndex), - tvList.getTime(tvList.size() - 1)); - } else { - window.seek( - beginIndex, endIndex, tvList.getTime(beginIndex), tvList.getTime(endIndex - 1)); - } - - hasCached = true; - return true; - } - - @Override - public void readyForNext() { - hasCached = false; - - safetyPile.moveForwardTo(beginIndex + 1); - tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); - } - - @Override - public TSDataType[] getDataTypes() { - return new TSDataType[] {parentLayerPointReaderDataType}; - } - - @Override - public RowWindow currentWindow() { - return window; - } - }; - } - - @Override - protected LayerRowWindowReader constructRowSlidingTimeWindowReader( - SlidingTimeWindowAccessStrategy strategy, float memoryBudgetInMB) { - - final long timeInterval = strategy.getTimeInterval(); - final long slidingStep = strategy.getSlidingStep(); - final long displayWindowEnd = strategy.getDisplayWindowEnd(); - - final SafetyPile safetyPile = safetyLine.addSafetyPile(); - final ElasticSerializableTVListBackedSingleColumnWindow window = - new ElasticSerializableTVListBackedSingleColumnWindow(tvList); - - final long nextWindowTimeBeginGivenByStrategy = strategy.getDisplayWindowBegin(); - - return new LayerRowWindowReader() { - - private boolean isFirstIteration = true; - private boolean hasCached = false; - private long nextWindowTimeBegin = nextWindowTimeBeginGivenByStrategy; - private int nextIndexBegin = 0; - private boolean hasAtLeastOneRow; - - @Override - public YieldableState yield() throws Exception { - if (isFirstIteration) { - if (tvList.size() == 0) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList); - if (yieldableState != YieldableState.YIELDABLE) { - return yieldableState; - } - } - if (nextWindowTimeBeginGivenByStrategy == Long.MIN_VALUE) { - // display window begin should be set to the same as the min timestamp of the query - // result - // set - nextWindowTimeBegin = tvList.getTime(0); - } - hasAtLeastOneRow = tvList.size() != 0; - isFirstIteration = false; - } - - if (hasCached) { - return YieldableState.YIELDABLE; - } - if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); - while (tvList.getTime(tvList.size() - 1) < nextWindowTimeEnd) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList); - if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - if (yieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { - break; - } - } - - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeBegin <= tvList.getTime(i)) { - nextIndexBegin = i; - break; - } - if (i == tvList.size() - 1) { - nextIndexBegin = tvList.size(); - } - } - - int nextIndexEnd = tvList.size(); - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeEnd <= tvList.getTime(i)) { - nextIndexEnd = i; - break; - } - } - - if ((nextIndexEnd == nextIndexBegin) - && nextWindowTimeEnd < tvList.getTime(tvList.size() - 1)) { - window.setEmptyWindow(nextWindowTimeBegin, nextWindowTimeEnd); - return YieldableState.YIELDABLE; - } - - window.seek( - nextIndexBegin, - nextIndexEnd, - nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); - - hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == tvList.size()); - return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - if (isFirstIteration) { - if (tvList.size() == 0 - && LayerCacheUtils.cachePoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList) - && nextWindowTimeBeginGivenByStrategy == Long.MIN_VALUE) { - // display window begin should be set to the same as the min timestamp of the query - // result - // set - nextWindowTimeBegin = tvList.getTime(0); - } - hasAtLeastOneRow = tvList.size() != 0; - isFirstIteration = false; - } - if (hasCached) { - return true; - } - if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { - return false; - } - - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); - while (tvList.getTime(tvList.size() - 1) < nextWindowTimeEnd) { - if (!LayerCacheUtils.cachePoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList)) { - break; - } - } - - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeBegin <= tvList.getTime(i)) { - nextIndexBegin = i; - break; - } - if (i == tvList.size() - 1) { - nextIndexBegin = tvList.size(); - } - } - - int nextIndexEnd = tvList.size(); - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeEnd <= tvList.getTime(i)) { - nextIndexEnd = i; - break; - } - } - - if ((nextIndexEnd == nextIndexBegin) - && nextWindowTimeEnd < tvList.getTime(tvList.size() - 1)) { - window.setEmptyWindow(nextWindowTimeBegin, nextWindowTimeEnd); - return true; - } - - window.seek( - nextIndexBegin, - nextIndexEnd, - nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); - - hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == tvList.size()); - return hasCached; - } - - @Override - public void readyForNext() { - hasCached = false; - nextWindowTimeBegin += slidingStep; - - safetyPile.moveForwardTo(nextIndexBegin + 1); - tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); - } - - @Override - public TSDataType[] getDataTypes() { - return new TSDataType[] {parentLayerPointReaderDataType}; - } - - @Override - public RowWindow currentWindow() { - return window; - } - }; - } - - @Override - protected LayerRowWindowReader constructRowSessionTimeWindowReader( - SessionTimeWindowAccessStrategy strategy, float memoryBudgetInMB) { - final long displayWindowBegin = strategy.getDisplayWindowBegin(); - final long displayWindowEnd = strategy.getDisplayWindowEnd(); - final long sessionTimeGap = strategy.getSessionTimeGap(); - - final SafetyPile safetyPile = safetyLine.addSafetyPile(); - final ElasticSerializableTVListBackedSingleColumnWindow window = - new ElasticSerializableTVListBackedSingleColumnWindow(tvList); - - return new LayerRowWindowReader() { - - private boolean isFirstIteration = true; - private boolean hasAtLeastOneRow = false; - - private long nextWindowTimeBegin = displayWindowBegin; - private long nextWindowTimeEnd = 0; - private int nextIndexBegin = 0; - private int nextIndexEnd = 1; - - @Override - public YieldableState yield() throws Exception { - if (isFirstIteration) { - if (tvList.size() == 0) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList); - if (yieldableState != YieldableState.YIELDABLE) { - return yieldableState; - } - } - nextWindowTimeBegin = Math.max(displayWindowBegin, tvList.getTime(0)); - hasAtLeastOneRow = tvList.size() != 0; - isFirstIteration = false; - } - - if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - while (tvList.getTime(tvList.size() - 1) < displayWindowEnd) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList); - if (yieldableState == YieldableState.YIELDABLE) { - if (tvList.getTime(tvList.size() - 2) >= displayWindowBegin - && tvList.getTime(tvList.size() - 1) - tvList.getTime(tvList.size() - 2) - > sessionTimeGap) { - nextIndexEnd = tvList.size() - 1; - break; - } else { - nextIndexEnd++; - } - } else if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } else if (yieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { - nextIndexEnd = tvList.size(); - break; - } - } - - nextWindowTimeEnd = tvList.getTime(nextIndexEnd - 1); - - if (nextIndexBegin == nextIndexEnd) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - // Only if encounter user set the strategy's displayWindowBegin, which will go into the for - // loop to find the true index of the first window begin. - // For other situation, we will only go into if (nextWindowTimeBegin <= tvList.getTime(i)) - // once. - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeBegin <= tvList.getTime(i)) { - nextIndexBegin = i; - break; - } - // The first window's beginning time is greater than all the timestamp of the query result - // set - if (i == tvList.size() - 1) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - } - - window.seek(nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, nextWindowTimeEnd); - - return YieldableState.YIELDABLE; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - return false; - } - - @Override - public void readyForNext() throws IOException, QueryProcessException { - if (nextIndexEnd < tvList.size()) { - nextWindowTimeBegin = tvList.getTime(nextIndexEnd); - } - safetyPile.moveForwardTo(nextIndexBegin + 1); - tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); - nextIndexBegin = nextIndexEnd; - } - - @Override - public TSDataType[] getDataTypes() { - return new TSDataType[] {parentLayerPointReaderDataType}; - } - - @Override - public RowWindow currentWindow() { - return window; - } - }; - } - - @Override - protected LayerRowWindowReader constructRowStateWindowReader( - StateWindowAccessStrategy strategy, float memoryBudgetInMB) { - final long displayWindowBegin = strategy.getDisplayWindowBegin(); - final long displayWindowEnd = strategy.getDisplayWindowEnd(); - final double delta = strategy.getDelta(); - - final SafetyPile safetyPile = safetyLine.addSafetyPile(); - final ElasticSerializableTVListBackedSingleColumnWindow window = - new ElasticSerializableTVListBackedSingleColumnWindow(tvList); - - return new LayerRowWindowReader() { - - private boolean isFirstIteration = true; - private boolean hasAtLeastOneRow = false; - - private long nextWindowTimeBegin = displayWindowBegin; - private long nextWindowTimeEnd = 0; - private int nextIndexBegin = 0; - private int nextIndexEnd = 1; - - private ValueRecorder valueRecorder = new ValueRecorder(); - - @Override - public YieldableState yield() throws Exception { - if (isFirstIteration) { - if (tvList.size() == 0) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList); - if (yieldableState != YieldableState.YIELDABLE) { - return yieldableState; - } - } - nextWindowTimeBegin = Math.max(displayWindowBegin, tvList.getTime(0)); - hasAtLeastOneRow = tvList.size() != 0; - isFirstIteration = false; - } - - if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - while (tvList.getTime(tvList.size() - 1) < displayWindowEnd) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint( - parentLayerPointReaderDataType, parentLayerPointReader, tvList); - if (yieldableState == YieldableState.YIELDABLE) { - if (tvList.getTime(tvList.size() - 2) >= displayWindowBegin - && TransformUtils.splitWindowForStateWindow( - parentLayerPointReaderDataType, valueRecorder, delta, tvList)) { - nextIndexEnd = tvList.size() - 1; - break; - } else { - nextIndexEnd++; - } - } else if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } else if (yieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { - nextIndexEnd = tvList.size(); - break; - } - } - - nextWindowTimeEnd = tvList.getTime(nextIndexEnd - 1); - - if (nextIndexBegin == nextIndexEnd) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - // Only if encounter user set the strategy's displayWindowBegin, which will go into the for - // loop to find the true index of the first window begin. - // For other situation, we will only go into if (nextWindowTimeBegin <= tvList.getTime(i)) - // once. - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeBegin <= tvList.getTime(i)) { - nextIndexBegin = i; - break; - } - // The first window's beginning time is greater than all the timestamp of the query result - // set - if (i == tvList.size() - 1) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - } - - window.seek(nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, nextWindowTimeEnd); - - return YieldableState.YIELDABLE; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - return false; - } - - @Override - public void readyForNext() throws IOException, QueryProcessException { - if (nextIndexEnd < tvList.size()) { - nextWindowTimeBegin = tvList.getTime(nextIndexEnd); - } - safetyPile.moveForwardTo(nextIndexBegin + 1); - tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); - nextIndexBegin = nextIndexEnd; - } - - @Override - public TSDataType[] getDataTypes() { - return new TSDataType[] {parentLayerPointReaderDataType}; - } - - @Override - public RowWindow currentWindow() { - return window; - } - }; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputColumnSingleReferenceIntermediateLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputColumnSingleReferenceIntermediateLayer.java deleted file mode 100644 index ba36e92e73dc..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputColumnSingleReferenceIntermediateLayer.java +++ /dev/null @@ -1,668 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.intermediate; - -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.plan.expression.Expression; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; -import org.apache.iotdb.db.queryengine.transformation.api.LayerRowReader; -import org.apache.iotdb.db.queryengine.transformation.api.LayerRowWindowReader; -import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; -import org.apache.iotdb.db.queryengine.transformation.dag.adapter.ElasticSerializableTVListBackedSingleColumnWindow; -import org.apache.iotdb.db.queryengine.transformation.dag.adapter.LayerPointReaderBackedSingleColumnRow; -import org.apache.iotdb.db.queryengine.transformation.dag.util.LayerCacheUtils; -import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.ElasticSerializableTVList; -import org.apache.iotdb.db.queryengine.transformation.datastructure.util.ValueRecorder; -import org.apache.iotdb.udf.api.access.Row; -import org.apache.iotdb.udf.api.access.RowWindow; -import org.apache.iotdb.udf.api.customizer.strategy.SessionTimeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.StateWindowAccessStrategy; - -import org.apache.tsfile.enums.TSDataType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -public class SingleInputColumnSingleReferenceIntermediateLayer extends IntermediateLayer { - - private static final Logger LOGGER = - LoggerFactory.getLogger(SingleInputColumnSingleReferenceIntermediateLayer.class); - - private final LayerPointReader parentLayerPointReader; - private final TSDataType dataType; - - public SingleInputColumnSingleReferenceIntermediateLayer( - Expression expression, - String queryId, - float memoryBudgetInMB, - LayerPointReader parentLayerPointReader) { - super(expression, queryId, memoryBudgetInMB); - this.parentLayerPointReader = parentLayerPointReader; - dataType = parentLayerPointReader.getDataType(); - } - - @Override - public LayerPointReader constructPointReader() { - return parentLayerPointReader; - } - - @Override - public LayerRowReader constructRowReader() { - - return new LayerRowReader() { - - private final Row row = new LayerPointReaderBackedSingleColumnRow(parentLayerPointReader); - - private boolean hasCached = false; - private boolean isCurrentNull = false; - - @Override - public YieldableState yield() throws Exception { - if (!hasCached) { - final YieldableState yieldableState = parentLayerPointReader.yield(); - if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - hasCached = yieldableState == YieldableState.YIELDABLE; - if (hasCached) { - isCurrentNull = parentLayerPointReader.isCurrentNull(); - } - } - return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - if (!hasCached) { - hasCached = parentLayerPointReader.next(); - if (hasCached) { - isCurrentNull = parentLayerPointReader.isCurrentNull(); - } - } - return hasCached; - } - - @Override - public void readyForNext() { - hasCached = false; - isCurrentNull = false; - parentLayerPointReader.readyForNext(); - } - - @Override - public TSDataType[] getDataTypes() { - return new TSDataType[] {parentLayerPointReader.getDataType()}; - } - - @Override - public long currentTime() throws IOException { - return parentLayerPointReader.currentTime(); - } - - @Override - public Row currentRow() { - return row; - } - - @Override - public boolean isCurrentNull() { - return isCurrentNull; - } - }; - } - - @Override - protected LayerRowWindowReader constructRowSlidingSizeWindowReader( - SlidingSizeWindowAccessStrategy strategy, float memoryBudgetInMB) { - - return new LayerRowWindowReader() { - - private final int windowSize = strategy.getWindowSize(); - private final int slidingStep = strategy.getSlidingStep(); - - private final ElasticSerializableTVList tvList = - ElasticSerializableTVList.newElasticSerializableTVList( - dataType, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); - private final ElasticSerializableTVListBackedSingleColumnWindow window = - new ElasticSerializableTVListBackedSingleColumnWindow(tvList); - - private boolean hasCached = false; - private int beginIndex = -slidingStep; - - @Override - public YieldableState yield() throws Exception { - if (hasCached) { - return YieldableState.YIELDABLE; - } - - beginIndex += slidingStep; - int endIndex = beginIndex + windowSize; - if (beginIndex < 0 || endIndex < 0) { - LOGGER.warn( - "LayerRowWindowReader index overflow. beginIndex: {}, endIndex: {}, windowSize: {}.", - beginIndex, - endIndex, - windowSize); - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - int pointsToBeCollected = endIndex - tvList.size(); - if (0 < pointsToBeCollected) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoints( - dataType, parentLayerPointReader, tvList, pointsToBeCollected); - if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - beginIndex -= slidingStep; - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - - if (tvList.size() <= beginIndex) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - window.seek( - beginIndex, - tvList.size(), - tvList.getTime(beginIndex), - tvList.getTime(tvList.size() - 1)); - } else { - window.seek( - beginIndex, endIndex, tvList.getTime(beginIndex), tvList.getTime(endIndex - 1)); - } - - hasCached = true; - return YieldableState.YIELDABLE; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - if (hasCached) { - return true; - } - - beginIndex += slidingStep; - int endIndex = beginIndex + windowSize; - if (beginIndex < 0 || endIndex < 0) { - LOGGER.warn( - "LayerRowWindowReader index overflow. beginIndex: {}, endIndex: {}, windowSize: {}.", - beginIndex, - endIndex, - windowSize); - return false; - } - - int pointsToBeCollected = endIndex - tvList.size(); - if (0 < pointsToBeCollected) { - LayerCacheUtils.cachePoints( - dataType, parentLayerPointReader, tvList, pointsToBeCollected); - if (tvList.size() <= beginIndex) { - return false; - } - - window.seek( - beginIndex, - tvList.size(), - tvList.getTime(beginIndex), - tvList.getTime(tvList.size() - 1)); - } else { - window.seek( - beginIndex, endIndex, tvList.getTime(beginIndex), tvList.getTime(endIndex - 1)); - } - - hasCached = true; - return true; - } - - @Override - public void readyForNext() { - hasCached = false; - - tvList.setEvictionUpperBound(beginIndex + 1); - } - - @Override - public TSDataType[] getDataTypes() { - return new TSDataType[] {dataType}; - } - - @Override - public RowWindow currentWindow() { - return window; - } - }; - } - - @Override - protected LayerRowWindowReader constructRowSlidingTimeWindowReader( - SlidingTimeWindowAccessStrategy strategy, float memoryBudgetInMB) { - - final long timeInterval = strategy.getTimeInterval(); - final long slidingStep = strategy.getSlidingStep(); - final long displayWindowEnd = strategy.getDisplayWindowEnd(); - - final ElasticSerializableTVList tvList = - ElasticSerializableTVList.newElasticSerializableTVList( - dataType, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); - final ElasticSerializableTVListBackedSingleColumnWindow window = - new ElasticSerializableTVListBackedSingleColumnWindow(tvList); - - return new LayerRowWindowReader() { - - private boolean isFirstIteration = true; - private boolean hasAtLeastOneRow = false; - - private boolean hasCached = false; - private long nextWindowTimeBegin = strategy.getDisplayWindowBegin(); - private int nextIndexBegin = 0; - - @Override - public YieldableState yield() throws Exception { - if (isFirstIteration) { - if (tvList.size() == 0) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint(dataType, parentLayerPointReader, tvList); - if (yieldableState != YieldableState.YIELDABLE) { - return yieldableState; - } - } - if (nextWindowTimeBegin == Long.MIN_VALUE) { - // display window begin should be set to the same as the min timestamp of the query - // result set - nextWindowTimeBegin = tvList.getTime(0); - } - hasAtLeastOneRow = tvList.size() != 0; - isFirstIteration = false; - } - - if (hasCached) { - return YieldableState.YIELDABLE; - } - - if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); - while (tvList.getTime(tvList.size() - 1) < nextWindowTimeEnd) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint(dataType, parentLayerPointReader, tvList); - if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - if (yieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { - break; - } - } - - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeBegin <= tvList.getTime(i)) { - nextIndexBegin = i; - break; - } - if (i == tvList.size() - 1) { - nextIndexBegin = tvList.size(); - } - } - - int nextIndexEnd = tvList.size(); - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeEnd <= tvList.getTime(i)) { - nextIndexEnd = i; - break; - } - } - - if ((nextIndexEnd == nextIndexBegin) - && nextWindowTimeEnd < tvList.getTime(tvList.size() - 1)) { - window.setEmptyWindow(nextWindowTimeBegin, nextWindowTimeEnd); - return YieldableState.YIELDABLE; - } - - window.seek( - nextIndexBegin, - nextIndexEnd, - nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); - - hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == tvList.size()); - return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - if (isFirstIteration) { - if (tvList.size() == 0 - && LayerCacheUtils.cachePoint(dataType, parentLayerPointReader, tvList) - && nextWindowTimeBegin == Long.MIN_VALUE) { - // display window begin should be set to the same as the min timestamp of the query - // result set - nextWindowTimeBegin = tvList.getTime(0); - } - hasAtLeastOneRow = tvList.size() != 0; - isFirstIteration = false; - } - - if (hasCached) { - return true; - } - - if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { - return false; - } - - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); - while (tvList.getTime(tvList.size() - 1) < nextWindowTimeEnd) { - if (!LayerCacheUtils.cachePoint(dataType, parentLayerPointReader, tvList)) { - break; - } - } - - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeBegin <= tvList.getTime(i)) { - nextIndexBegin = i; - break; - } - if (i == tvList.size() - 1) { - nextIndexBegin = tvList.size(); - } - } - - int nextIndexEnd = tvList.size(); - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeEnd <= tvList.getTime(i)) { - nextIndexEnd = i; - break; - } - } - - if ((nextIndexEnd == nextIndexBegin) - && nextWindowTimeEnd < tvList.getTime(tvList.size() - 1)) { - window.setEmptyWindow(nextWindowTimeBegin, nextWindowTimeEnd); - return true; - } - - window.seek( - nextIndexBegin, - nextIndexEnd, - nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); - - hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == tvList.size()); - return hasCached; - } - - @Override - public void readyForNext() { - hasCached = false; - nextWindowTimeBegin += slidingStep; - - tvList.setEvictionUpperBound(nextIndexBegin + 1); - } - - @Override - public TSDataType[] getDataTypes() { - return new TSDataType[] {dataType}; - } - - @Override - public RowWindow currentWindow() { - return window; - } - }; - } - - @Override - protected LayerRowWindowReader constructRowSessionTimeWindowReader( - SessionTimeWindowAccessStrategy strategy, float memoryBudgetInMB) { - - final long displayWindowBegin = strategy.getDisplayWindowBegin(); - final long displayWindowEnd = strategy.getDisplayWindowEnd(); - final long sessionTimeGap = strategy.getSessionTimeGap(); - - final ElasticSerializableTVList tvList = - ElasticSerializableTVList.newElasticSerializableTVList( - dataType, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); - final ElasticSerializableTVListBackedSingleColumnWindow window = - new ElasticSerializableTVListBackedSingleColumnWindow(tvList); - - return new LayerRowWindowReader() { - - private boolean isFirstIteration = true; - private boolean hasAtLeastOneRow = false; - - private long nextWindowTimeBegin = displayWindowBegin; - private long nextWindowTimeEnd = 0; - private int nextIndexBegin = 0; - private int nextIndexEnd = 1; - - @Override - public YieldableState yield() throws Exception { - if (isFirstIteration) { - if (tvList.size() == 0) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint(dataType, parentLayerPointReader, tvList); - if (yieldableState != YieldableState.YIELDABLE) { - return yieldableState; - } - } - nextWindowTimeBegin = Math.max(displayWindowBegin, tvList.getTime(0)); - hasAtLeastOneRow = tvList.size() != 0; - isFirstIteration = false; - } - - if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - while (tvList.getTime(tvList.size() - 1) < displayWindowEnd) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint(dataType, parentLayerPointReader, tvList); - if (yieldableState == YieldableState.YIELDABLE) { - if (tvList.getTime(tvList.size() - 2) >= displayWindowBegin - && tvList.getTime(tvList.size() - 1) - tvList.getTime(tvList.size() - 2) - > sessionTimeGap) { - nextIndexEnd = tvList.size() - 1; - break; - } else { - nextIndexEnd++; - } - } else if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } else if (yieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { - nextIndexEnd = tvList.size(); - break; - } - } - - nextWindowTimeEnd = tvList.getTime(nextIndexEnd - 1); - - if (nextIndexBegin == nextIndexEnd) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - // Only if encounter user set the strategy's displayWindowBegin, which will go into the for - // loop to find the true index of the first window begin. - // For other situation, we will only go into if (nextWindowTimeBegin <= tvList.getTime(i)) - // once. - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeBegin <= tvList.getTime(i)) { - nextIndexBegin = i; - break; - } - // The first window's beginning time is greater than all the timestamp of the query result - // set - if (i == tvList.size() - 1) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - } - - window.seek(nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, nextWindowTimeEnd); - - return YieldableState.YIELDABLE; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - return false; - } - - @Override - public void readyForNext() throws IOException { - if (nextIndexEnd < tvList.size()) { - nextWindowTimeBegin = tvList.getTime(nextIndexEnd); - } - tvList.setEvictionUpperBound(nextIndexBegin + 1); - nextIndexBegin = nextIndexEnd; - } - - @Override - public TSDataType[] getDataTypes() { - return new TSDataType[] {dataType}; - } - - @Override - public RowWindow currentWindow() { - return window; - } - }; - } - - @Override - protected LayerRowWindowReader constructRowStateWindowReader( - StateWindowAccessStrategy strategy, float memoryBudgetInMB) { - - final long displayWindowBegin = strategy.getDisplayWindowBegin(); - final long displayWindowEnd = strategy.getDisplayWindowEnd(); - final double delta = strategy.getDelta(); - - final ElasticSerializableTVList tvList = - ElasticSerializableTVList.newElasticSerializableTVList( - dataType, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); - final ElasticSerializableTVListBackedSingleColumnWindow window = - new ElasticSerializableTVListBackedSingleColumnWindow(tvList); - - return new LayerRowWindowReader() { - - private boolean isFirstIteration = true; - private boolean hasAtLeastOneRow = false; - - private long nextWindowTimeBegin = displayWindowBegin; - private long nextWindowTimeEnd = 0; - private int nextIndexBegin = 0; - private int nextIndexEnd = 1; - - private ValueRecorder valueRecorder = new ValueRecorder(); - - @Override - public YieldableState yield() throws Exception { - if (isFirstIteration) { - if (tvList.size() == 0) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint(dataType, parentLayerPointReader, tvList); - if (yieldableState != YieldableState.YIELDABLE) { - return yieldableState; - } - } - nextWindowTimeBegin = Math.max(displayWindowBegin, tvList.getTime(0)); - hasAtLeastOneRow = tvList.size() != 0; - isFirstIteration = false; - } - - if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - while (tvList.getTime(tvList.size() - 1) < displayWindowEnd) { - final YieldableState yieldableState = - LayerCacheUtils.yieldPoint(dataType, parentLayerPointReader, tvList); - if (yieldableState == YieldableState.YIELDABLE) { - if (tvList.getTime(tvList.size() - 2) >= displayWindowBegin - && TransformUtils.splitWindowForStateWindow( - dataType, valueRecorder, delta, tvList)) { - nextIndexEnd = tvList.size() - 1; - break; - } else { - nextIndexEnd++; - } - } else if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } else if (yieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { - nextIndexEnd = tvList.size(); - break; - } - } - - nextWindowTimeEnd = tvList.getTime(nextIndexEnd - 1); - - if (nextIndexBegin == nextIndexEnd) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - - // Only if encounter user set the strategy's displayWindowBegin, which will go into the for - // loop to find the true index of the first window begin. - // For other situation, we will only go into if (nextWindowTimeBegin <= tvList.getTime(i)) - // once. - for (int i = nextIndexBegin; i < tvList.size(); ++i) { - if (nextWindowTimeBegin <= tvList.getTime(i)) { - nextIndexBegin = i; - break; - } - // The first window's beginning time is greater than all the timestamp of the query result - // set - if (i == tvList.size() - 1) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } - } - - window.seek(nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, nextWindowTimeEnd); - - return YieldableState.YIELDABLE; - } - - @Override - public boolean next() throws IOException, QueryProcessException { - return false; - } - - @Override - public void readyForNext() throws IOException { - if (nextIndexEnd < tvList.size()) { - nextWindowTimeBegin = tvList.getTime(nextIndexEnd); - } - tvList.setEvictionUpperBound(nextIndexBegin + 1); - nextIndexBegin = nextIndexEnd; - } - - @Override - public TSDataType[] getDataTypes() { - return new TSDataType[] {dataType}; - } - - @Override - public RowWindow currentWindow() { - return window; - } - }; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java new file mode 100644 index 000000000000..a57b9e108633 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java @@ -0,0 +1,724 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.intermediate; + +import org.apache.iotdb.db.queryengine.plan.expression.Expression; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerRowWindowReader; +import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; +import org.apache.iotdb.db.queryengine.transformation.dag.adapter.ElasticSerializableTVListBackedSingleColumnWindow; +import org.apache.iotdb.db.queryengine.transformation.dag.memory.SafetyLine; +import org.apache.iotdb.db.queryengine.transformation.dag.memory.SafetyLine.SafetyPile; +import org.apache.iotdb.db.queryengine.transformation.dag.util.LayerCacheUtils; +import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; +import org.apache.iotdb.db.queryengine.transformation.datastructure.iterator.TVListForwardIterator; +import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.ElasticSerializableTVList; +import org.apache.iotdb.db.queryengine.transformation.datastructure.util.ValueRecorder; +import org.apache.iotdb.udf.api.access.RowWindow; +import org.apache.iotdb.udf.api.customizer.strategy.SessionTimeWindowAccessStrategy; +import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; +import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; +import org.apache.iotdb.udf.api.customizer.strategy.StateWindowAccessStrategy; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class SingleInputMultiReferenceLayer extends IntermediateLayer { + + private static final Logger LOGGER = + LoggerFactory.getLogger(SingleInputMultiReferenceLayer.class); + + private final LayerReader parentLayerReader; + private final TSDataType parentLayerReaderDataType; + private final boolean isParentLayerReaderConstant; + private final ElasticSerializableTVList tvList; + private final SafetyLine safetyLine; + + public SingleInputMultiReferenceLayer( + Expression expression, + String queryId, + float memoryBudgetInMB, + LayerReader parentLayerReader) { + super(expression, queryId, memoryBudgetInMB); + this.parentLayerReader = parentLayerReader; + + parentLayerReaderDataType = this.parentLayerReader.getDataTypes()[0]; + isParentLayerReaderConstant = this.parentLayerReader.isConstantPointReader(); + tvList = + ElasticSerializableTVList.construct( + parentLayerReaderDataType, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); + safetyLine = new SafetyLine(); + } + + @Override + public LayerReader constructReader() { + return new LayerReader() { + private final SafetyPile safetyPile = safetyLine.addSafetyPile(); + + private TimeColumn cachedTimes = null; + private Column cachedValues = null; + private int cacheConsumed = 0; + + private final TVListForwardIterator iterator = tvList.constructIterator(); + + @Override + public boolean isConstantPointReader() { + return isParentLayerReaderConstant; + } + + @Override + public YieldableState yield() throws Exception { + // Column cached in reader is not yet consumed + if (cachedTimes != null && cacheConsumed < cachedTimes.getPositionCount()) { + return YieldableState.YIELDABLE; + } + + // TVList still has some cached columns + if (iterator.hasNext()) { + iterator.next(); + cachedTimes = iterator.currentTimes(); + cachedValues = iterator.currentValues(); + + return YieldableState.YIELDABLE; + } + + // No data cached, yield from parent layer reader + YieldableState state = LayerCacheUtils.yieldPoints(parentLayerReader, tvList); + if (state == YieldableState.YIELDABLE) { + iterator.next(); + cachedTimes = iterator.currentTimes(); + cachedValues = iterator.currentValues(); + } + return state; + } + + @Override + public void consumedAll() { + int steps = cachedTimes.getPositionCount() - cacheConsumed; + safetyPile.moveForward(steps); + tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); + + cacheConsumed = 0; + cachedTimes = null; + cachedValues = null; + } + + @Override + public Column[] current() { + return cacheConsumed == 0 + ? new Column[] {cachedValues, cachedTimes} + : new Column[] { + cachedValues.subColumn(cacheConsumed), cachedTimes.subColumn(cacheConsumed) + }; + } + + @Override + public TSDataType[] getDataTypes() { + return new TSDataType[] {parentLayerReaderDataType}; + } + }; + } + + @Override + protected LayerRowWindowReader constructRowSlidingSizeWindowReader( + SlidingSizeWindowAccessStrategy strategy, float memoryBudgetInMB) { + return new LayerRowWindowReader() { + + private final int windowSize = strategy.getWindowSize(); + private final int slidingStep = strategy.getSlidingStep(); + + private final SafetyPile safetyPile = safetyLine.addSafetyPile(); + private final ElasticSerializableTVListBackedSingleColumnWindow window = + new ElasticSerializableTVListBackedSingleColumnWindow(tvList); + + private boolean hasCached = false; + private int beginIndex = -slidingStep; + + @Override + public YieldableState yield() throws Exception { + if (hasCached) { + return YieldableState.YIELDABLE; + } + + beginIndex += slidingStep; + int endIndex = beginIndex + windowSize; + if (beginIndex < 0 || endIndex < 0) { + LOGGER.warn( + "LayerRowWindowReader index overflow. beginIndex: {}, endIndex: {}, windowSize: {}.", + beginIndex, + endIndex, + windowSize); + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + final int pointsToBeCollected = endIndex - tvList.size(); + if (pointsToBeCollected > 0) { + final YieldableState yieldableState = + LayerCacheUtils.yieldPoints(parentLayerReader, tvList, pointsToBeCollected); + if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + beginIndex -= slidingStep; + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } + + if (tvList.size() <= beginIndex) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + // TVList's size may be less than endIndex + // When parent layer reader has no more data + endIndex = Math.min(endIndex, tvList.size()); + } + + window.seek(beginIndex, endIndex, tvList.getTime(beginIndex), tvList.getTime(endIndex - 1)); + + hasCached = true; + return YieldableState.YIELDABLE; + } + + @Override + public void readyForNext() { + hasCached = false; + + safetyPile.moveForwardTo(beginIndex + 1); + tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); + } + + @Override + public TSDataType[] getDataTypes() { + return new TSDataType[] {parentLayerReaderDataType}; + } + + @Override + public RowWindow currentWindow() { + return window; + } + }; + } + + @Override + protected LayerRowWindowReader constructRowSlidingTimeWindowReader( + SlidingTimeWindowAccessStrategy strategy, float memoryBudgetInMB) { + + final long timeInterval = strategy.getTimeInterval(); + final long slidingStep = strategy.getSlidingStep(); + final long displayWindowEnd = strategy.getDisplayWindowEnd(); + + final SafetyPile safetyPile = safetyLine.addSafetyPile(); + final ElasticSerializableTVListBackedSingleColumnWindow window = + new ElasticSerializableTVListBackedSingleColumnWindow(tvList); + + final long nextWindowTimeBeginGivenByStrategy = strategy.getDisplayWindowBegin(); + + return new LayerRowWindowReader() { + + private boolean isFirstIteration = true; + private boolean hasAtLeastOneRow = false; + private boolean hasCached = false; + private long nextWindowTimeBegin = nextWindowTimeBeginGivenByStrategy; + private int nextIndexBegin = 0; + private int nextIndexEnd = 0; + private long currentEndTime = Long.MAX_VALUE; + + private final TVListForwardIterator beginIterator = tvList.constructIterator(); + private TimeColumn cachedBeginTimeColumn; + private int cachedBeginConsumed; + + private TimeColumn cachedEndTimeColumn; + private int cachedEndConsumed; + + @Override + public YieldableState yield() throws Exception { + if (isFirstIteration) { + if (tvList.size() == 0) { + final YieldableState state = parentLayerReader.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + + Column[] columns = parentLayerReader.current(); + TimeColumn times = (TimeColumn) columns[1]; + Column values = columns[0]; + + tvList.putColumn(times, values); + parentLayerReader.consumedAll(); + + cachedEndTimeColumn = times; + } + if (nextWindowTimeBegin == Long.MIN_VALUE) { + // display window begin should be set to the same as the min timestamp of the query + // result set + nextWindowTimeBegin = cachedEndTimeColumn.getStartTime(); + } + hasAtLeastOneRow = tvList.size() != 0; + if (hasAtLeastOneRow) { + currentEndTime = cachedEndTimeColumn.getEndTime(); + } + isFirstIteration = false; + } + + if (hasCached) { + return YieldableState.YIELDABLE; + } + + if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); + while (currentEndTime < nextWindowTimeEnd) { + final YieldableState state = parentLayerReader.yield(); + if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } + if (state == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { + break; + } + // Generate data + Column[] columns = parentLayerReader.current(); + TimeColumn times = (TimeColumn) columns[1]; + Column values = columns[0]; + // Put data into container + tvList.putColumn(times, values); + parentLayerReader.consumedAll(); + currentEndTime = times.getEndTime(); + // Increase nextIndexEnd + nextIndexEnd += cachedEndTimeColumn.getPositionCount() - cachedEndConsumed; + // Update cache + cachedEndTimeColumn = times; + cachedEndConsumed = 0; + } + + // Set nextIndexEnd field + while (cachedEndConsumed < cachedEndTimeColumn.getPositionCount()) { + if (cachedEndTimeColumn.getLong(cachedEndConsumed) >= nextWindowTimeEnd) { + break; + } + cachedEndConsumed++; + nextIndexEnd++; + } + + // Set nextIndexBegin field + boolean findNextIndexBegin = false; + while (!findNextIndexBegin) { + while (cachedBeginTimeColumn != null + && cachedBeginConsumed < cachedBeginTimeColumn.getPositionCount()) { + if (cachedBeginTimeColumn.getLong(cachedBeginConsumed) >= nextWindowTimeBegin) { + findNextIndexBegin = true; + break; + } + cachedBeginConsumed++; + nextIndexBegin++; + } + + if (!findNextIndexBegin) { + if (beginIterator.hasNext()) { + beginIterator.next(); + + cachedBeginConsumed = 0; + cachedBeginTimeColumn = beginIterator.currentTimes(); + } else { + // No more data + // Set nextIndexBegin to list's size + findNextIndexBegin = true; + } + } + } + + if ((nextIndexEnd == nextIndexBegin) + && nextWindowTimeEnd < cachedEndTimeColumn.getEndTime()) { + window.setEmptyWindow(nextWindowTimeBegin, nextWindowTimeEnd); + return YieldableState.YIELDABLE; + } + + window.seek( + nextIndexBegin, + nextIndexEnd, + nextWindowTimeBegin, + nextWindowTimeBegin + timeInterval - 1); + + hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == tvList.size()); + return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + @Override + public void readyForNext() { + hasCached = false; + nextWindowTimeBegin += slidingStep; + + safetyPile.moveForwardTo(nextIndexBegin + 1); + tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); + } + + @Override + public TSDataType[] getDataTypes() { + return new TSDataType[] {parentLayerReaderDataType}; + } + + @Override + public RowWindow currentWindow() { + return window; + } + }; + } + + @Override + protected LayerRowWindowReader constructRowSessionTimeWindowReader( + SessionTimeWindowAccessStrategy strategy, float memoryBudgetInMB) { + final long displayWindowBegin = strategy.getDisplayWindowBegin(); + final long displayWindowEnd = strategy.getDisplayWindowEnd(); + final long sessionTimeGap = strategy.getSessionTimeGap(); + + final SafetyPile safetyPile = safetyLine.addSafetyPile(); + final ElasticSerializableTVListBackedSingleColumnWindow window = + new ElasticSerializableTVListBackedSingleColumnWindow(tvList); + + return new LayerRowWindowReader() { + + private boolean isFirstIteration = true; + private boolean hasAtLeastOneRow = false; + + private long nextWindowTimeBegin = displayWindowBegin; + private long nextWindowTimeEnd = 0; + private int nextIndexBegin = 0; + private int nextIndexEnd = 0; + + private TimeColumn cachedTimes; + private int cachedConsumed; + + @Override + public YieldableState yield() throws Exception { + if (isFirstIteration) { + YieldableState state = yieldInFirstIteration(); + if (state != YieldableState.YIELDABLE) { + return state; + } + } + + if (!hasAtLeastOneRow || nextWindowTimeBegin >= displayWindowEnd) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + // Set nextIndexEnd + long curTime = cachedTimes.getLong(cachedConsumed); + if (cachedConsumed < cachedTimes.getPositionCount()) { + nextIndexEnd++; + cachedConsumed++; + } + boolean findWindow = false; + // Find target window or no more data to exit + while (!findWindow) { + while (cachedConsumed < cachedTimes.getPositionCount()) { + long nextTime = cachedTimes.getLong(cachedConsumed); + + if (curTime >= displayWindowEnd) { + nextIndexEnd--; + findWindow = true; + break; + } + + if (nextTime - curTime > sessionTimeGap) { + findWindow = true; + break; + } + nextIndexEnd++; + cachedConsumed++; + curTime = nextTime; + } + + if (!findWindow) { + if (cachedTimes.getEndTime() < displayWindowEnd) { + YieldableState state = yieldAndCache(); + if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } else if (state == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { + break; + } + } + } + } + // Set nextWindowTimeEnd + nextWindowTimeEnd = tvList.getTime(nextIndexEnd - 1); + + if (nextIndexBegin == nextIndexEnd) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + window.seek(nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, nextWindowTimeEnd); + return YieldableState.YIELDABLE; + } + + private YieldableState yieldInFirstIteration() throws Exception { + // Yield initial data in first iteration + if (tvList.size() == 0) { + YieldableState state = yieldAndCache(); + if (state != YieldableState.YIELDABLE) { + return state; + } + } + // Initialize essential information + nextWindowTimeBegin = Math.max(displayWindowBegin, cachedTimes.getStartTime()); + hasAtLeastOneRow = tvList.size() != 0; + isFirstIteration = false; + + // Set initial nextIndexBegin + long currentEndTime = cachedTimes.getEndTime(); + // Find corresponding block + while (currentEndTime < nextWindowTimeBegin) { + // Consume all data + cachedConsumed = cachedTimes.getPositionCount(); + nextIndexBegin += cachedConsumed; + + YieldableState state = yieldAndCache(); + if (state != YieldableState.YIELDABLE) { + // Cannot find nextIndexBegin + // Set nextIndexEnd to nextIndexBegin and exit + nextIndexEnd = nextIndexBegin; + return state; + } + } + // Find nextIndexBegin in block + while (cachedConsumed < cachedTimes.getPositionCount()) { + if (cachedTimes.getLong(cachedConsumed) >= nextWindowTimeBegin) { + break; + } + cachedConsumed++; + nextIndexBegin++; + } + nextIndexEnd = nextIndexBegin; + + return YieldableState.YIELDABLE; + } + + private YieldableState yieldAndCache() throws Exception { + final YieldableState state = parentLayerReader.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + Column[] columns = parentLayerReader.current(); + TimeColumn times = (TimeColumn) columns[1]; + Column values = columns[0]; + + tvList.putColumn(times, values); + parentLayerReader.consumedAll(); + + cachedTimes = times; + cachedConsumed = 0; + + return YieldableState.YIELDABLE; + } + + @Override + public void readyForNext() throws IOException { + if (nextIndexEnd < tvList.size()) { + nextWindowTimeBegin = tvList.getTime(nextIndexEnd); + } + safetyPile.moveForwardTo(nextIndexBegin + 1); + tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); + nextIndexBegin = nextIndexEnd; + } + + @Override + public TSDataType[] getDataTypes() { + return new TSDataType[] {parentLayerReaderDataType}; + } + + @Override + public RowWindow currentWindow() { + return window; + } + }; + } + + @Override + protected LayerRowWindowReader constructRowStateWindowReader( + StateWindowAccessStrategy strategy, float memoryBudgetInMB) { + final long displayWindowBegin = strategy.getDisplayWindowBegin(); + final long displayWindowEnd = strategy.getDisplayWindowEnd(); + final double delta = strategy.getDelta(); + + final SafetyPile safetyPile = safetyLine.addSafetyPile(); + final ElasticSerializableTVListBackedSingleColumnWindow window = + new ElasticSerializableTVListBackedSingleColumnWindow(tvList); + + return new LayerRowWindowReader() { + + private boolean isFirstIteration = true; + private boolean hasAtLeastOneRow = false; + + private long nextWindowTimeBegin = displayWindowBegin; + private long nextWindowTimeEnd = 0; + private int nextIndexBegin = 0; + private int nextIndexEnd = 0; + + private TimeColumn cachedTimes; + private Column cachedValues; + private int cachedConsumed; + + private final ValueRecorder valueRecorder = new ValueRecorder(); + + @Override + public YieldableState yield() throws Exception { + if (isFirstIteration) { + YieldableState state = yieldInFirstIteration(); + if (state != YieldableState.YIELDABLE) { + return state; + } + } + + if (!hasAtLeastOneRow || nextWindowTimeBegin >= displayWindowEnd) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + // Set nextIndexEnd + long curTime = cachedTimes.getLong(cachedConsumed); + if (cachedConsumed < cachedTimes.getPositionCount()) { + nextIndexEnd++; + cachedConsumed++; + } + boolean findWindow = false; + // Find target window or no more data to exit + while (!findWindow) { + while (cachedConsumed < cachedTimes.getPositionCount()) { + long nextTime = cachedTimes.getLong(cachedConsumed); + + if (curTime >= displayWindowEnd) { + nextIndexEnd--; + findWindow = true; + break; + } + + if (TransformUtils.splitWindowForStateWindow( + parentLayerReaderDataType, valueRecorder, delta, cachedValues, cachedConsumed)) { + findWindow = true; + break; + } + nextIndexEnd++; + cachedConsumed++; + curTime = nextTime; + } + + if (!findWindow) { + if (cachedTimes.getEndTime() < displayWindowEnd) { + YieldableState state = yieldAndCache(); + if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } else if (state == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { + break; + } + } + } + } + // Set nextWindowTimeEnd + nextWindowTimeEnd = tvList.getTime(nextIndexEnd - 1); + + if (nextIndexBegin == nextIndexEnd) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + window.seek(nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, nextWindowTimeEnd); + return YieldableState.YIELDABLE; + } + + private YieldableState yieldInFirstIteration() throws Exception { + // Yield initial data in first iteration + if (tvList.size() == 0) { + YieldableState state = yieldAndCache(); + if (state != YieldableState.YIELDABLE) { + return state; + } + } + // Initialize essential information + nextWindowTimeBegin = Math.max(displayWindowBegin, cachedTimes.getStartTime()); + hasAtLeastOneRow = tvList.size() != 0; + isFirstIteration = false; + + // Set initial nextIndexBegin + long currentEndTime = cachedTimes.getEndTime(); + // Find corresponding block + while (currentEndTime < nextWindowTimeBegin) { + // Consume all data + cachedConsumed = cachedTimes.getPositionCount(); + nextIndexBegin += cachedConsumed; + + YieldableState state = yieldAndCache(); + if (state != YieldableState.YIELDABLE) { + // Cannot find nextIndexBegin + // Set nextIndexEnd to nextIndexBegin and exit + nextIndexEnd = nextIndexBegin; + return state; + } + } + // Find nextIndexBegin in block + while (cachedConsumed < cachedTimes.getPositionCount()) { + if (cachedTimes.getLong(cachedConsumed) >= nextWindowTimeBegin) { + break; + } + cachedConsumed++; + nextIndexBegin++; + } + nextIndexEnd = nextIndexBegin; + + return YieldableState.YIELDABLE; + } + + private YieldableState yieldAndCache() throws Exception { + final YieldableState state = parentLayerReader.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + Column[] columns = parentLayerReader.current(); + TimeColumn times = (TimeColumn) columns[1]; + Column values = columns[0]; + + tvList.putColumn(times, values); + parentLayerReader.consumedAll(); + + cachedTimes = times; + cachedValues = values; + cachedConsumed = 0; + + return YieldableState.YIELDABLE; + } + + @Override + public void readyForNext() throws IOException { + if (nextIndexEnd < tvList.size()) { + nextWindowTimeBegin = tvList.getTime(nextIndexEnd); + } + safetyPile.moveForwardTo(nextIndexBegin + 1); + tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); + nextIndexBegin = nextIndexEnd; + } + + @Override + public TSDataType[] getDataTypes() { + return new TSDataType[] {parentLayerReaderDataType}; + } + + @Override + public RowWindow currentWindow() { + return window; + } + }; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java new file mode 100644 index 000000000000..f7e6d93f8435 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java @@ -0,0 +1,648 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.intermediate; + +import org.apache.iotdb.db.queryengine.plan.expression.Expression; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerRowWindowReader; +import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; +import org.apache.iotdb.db.queryengine.transformation.dag.adapter.ElasticSerializableTVListBackedSingleColumnWindow; +import org.apache.iotdb.db.queryengine.transformation.dag.util.LayerCacheUtils; +import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; +import org.apache.iotdb.db.queryengine.transformation.datastructure.iterator.TVListForwardIterator; +import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.ElasticSerializableTVList; +import org.apache.iotdb.db.queryengine.transformation.datastructure.util.ValueRecorder; +import org.apache.iotdb.udf.api.access.RowWindow; +import org.apache.iotdb.udf.api.customizer.strategy.SessionTimeWindowAccessStrategy; +import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; +import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; +import org.apache.iotdb.udf.api.customizer.strategy.StateWindowAccessStrategy; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class SingleInputSingleReferenceLayer extends IntermediateLayer { + + private static final Logger LOGGER = + LoggerFactory.getLogger(SingleInputSingleReferenceLayer.class); + + private final LayerReader parentLayerReader; + private final TSDataType dataType; + + public SingleInputSingleReferenceLayer( + Expression expression, + String queryId, + float memoryBudgetInMB, + LayerReader parentLayerReader) { + super(expression, queryId, memoryBudgetInMB); + this.parentLayerReader = parentLayerReader; + dataType = parentLayerReader.getDataTypes()[0]; + } + + @Override + public LayerReader constructReader() { + return parentLayerReader; + } + + @Override + protected LayerRowWindowReader constructRowSlidingSizeWindowReader( + SlidingSizeWindowAccessStrategy strategy, float memoryBudgetInMB) { + return new LayerRowWindowReader() { + private final int windowSize = strategy.getWindowSize(); + private final int slidingStep = strategy.getSlidingStep(); + + private final ElasticSerializableTVList tvList = + ElasticSerializableTVList.construct( + dataType, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); + private final ElasticSerializableTVListBackedSingleColumnWindow window = + new ElasticSerializableTVListBackedSingleColumnWindow(tvList); + + private boolean hasCached = false; + private int beginIndex = -slidingStep; + + @Override + public YieldableState yield() throws Exception { + if (hasCached) { + return YieldableState.YIELDABLE; + } + + beginIndex += slidingStep; + int endIndex = beginIndex + windowSize; + if (beginIndex < 0 || endIndex < 0) { + LOGGER.warn( + "LayerRowWindowReader index overflow. beginIndex: {}, endIndex: {}, windowSize: {}.", + beginIndex, + endIndex, + windowSize); + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + final int pointsToBeCollected = endIndex - tvList.size(); + if (pointsToBeCollected > 0) { + final YieldableState yieldableState = + LayerCacheUtils.yieldPoints(parentLayerReader, tvList, pointsToBeCollected); + if (yieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + beginIndex -= slidingStep; + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } + + if (tvList.size() <= beginIndex) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + // TVList's size may be less than endIndex + // When parent layer reader has no more data + endIndex = Math.min(endIndex, tvList.size()); + } + + window.seek(beginIndex, endIndex, tvList.getTime(beginIndex), tvList.getTime(endIndex - 1)); + + hasCached = true; + return YieldableState.YIELDABLE; + } + + @Override + public void readyForNext() { + hasCached = false; + + tvList.setEvictionUpperBound(beginIndex + 1); + } + + @Override + public TSDataType[] getDataTypes() { + return new TSDataType[] {dataType}; + } + + @Override + public RowWindow currentWindow() { + return window; + } + }; + } + + @Override + protected LayerRowWindowReader constructRowSlidingTimeWindowReader( + SlidingTimeWindowAccessStrategy strategy, float memoryBudgetInMB) { + final long timeInterval = strategy.getTimeInterval(); + final long slidingStep = strategy.getSlidingStep(); + final long displayWindowEnd = strategy.getDisplayWindowEnd(); + + final ElasticSerializableTVList tvList = + ElasticSerializableTVList.construct(dataType, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); + final ElasticSerializableTVListBackedSingleColumnWindow window = + new ElasticSerializableTVListBackedSingleColumnWindow(tvList); + + return new LayerRowWindowReader() { + private boolean isFirstIteration = true; + private boolean hasAtLeastOneRow = false; + + private boolean hasCached = false; + private long nextWindowTimeBegin = strategy.getDisplayWindowBegin(); + private int nextIndexBegin = 0; + private int nextIndexEnd = 0; + private long currentEndTime = Long.MAX_VALUE; + + private final TVListForwardIterator beginIterator = tvList.constructIterator(); + private TimeColumn cachedBeginTimeColumn; + private int cachedBeginConsumed; + + private TimeColumn cachedEndTimeColumn; + private int cachedEndConsumed; + + @Override + public YieldableState yield() throws Exception { + if (isFirstIteration) { + if (tvList.size() == 0) { + final YieldableState state = parentLayerReader.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + + Column[] columns = parentLayerReader.current(); + TimeColumn times = (TimeColumn) columns[1]; + Column values = columns[0]; + + tvList.putColumn(times, values); + parentLayerReader.consumedAll(); + + cachedEndTimeColumn = times; + } + if (nextWindowTimeBegin == Long.MIN_VALUE) { + // display window begin should be set to the same as the min timestamp of the query + // result set + nextWindowTimeBegin = cachedEndTimeColumn.getStartTime(); + } + hasAtLeastOneRow = tvList.size() != 0; + if (hasAtLeastOneRow) { + currentEndTime = cachedEndTimeColumn.getEndTime(); + } + isFirstIteration = false; + } + + if (hasCached) { + return YieldableState.YIELDABLE; + } + + if (!hasAtLeastOneRow || displayWindowEnd <= nextWindowTimeBegin) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); + while (currentEndTime < nextWindowTimeEnd) { + final YieldableState state = parentLayerReader.yield(); + if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } + if (state == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { + break; + } + // Generate data + Column[] columns = parentLayerReader.current(); + TimeColumn times = (TimeColumn) columns[1]; + Column values = columns[0]; + // Put data into container + tvList.putColumn(times, values); + parentLayerReader.consumedAll(); + currentEndTime = times.getEndTime(); + // Increase nextIndexEnd + nextIndexEnd += cachedEndTimeColumn.getPositionCount() - cachedEndConsumed; + // Update cache + cachedEndTimeColumn = times; + cachedEndConsumed = 0; + } + + // Set nextIndexEnd field + while (cachedEndConsumed < cachedEndTimeColumn.getPositionCount()) { + if (cachedEndTimeColumn.getLong(cachedEndConsumed) >= nextWindowTimeEnd) { + break; + } + cachedEndConsumed++; + nextIndexEnd++; + } + + // Set nextIndexBegin field + boolean findNextIndexBegin = false; + while (!findNextIndexBegin) { + while (cachedBeginTimeColumn != null + && cachedBeginConsumed < cachedBeginTimeColumn.getPositionCount()) { + if (cachedBeginTimeColumn.getLong(cachedBeginConsumed) >= nextWindowTimeBegin) { + findNextIndexBegin = true; + break; + } + cachedBeginConsumed++; + nextIndexBegin++; + } + + if (!findNextIndexBegin) { + if (beginIterator.hasNext()) { + beginIterator.next(); + + cachedBeginConsumed = 0; + cachedBeginTimeColumn = beginIterator.currentTimes(); + } else { + // No more data + // Set nextIndexBegin to list's size + findNextIndexBegin = true; + } + } + } + + if ((nextIndexEnd == nextIndexBegin) + && nextWindowTimeEnd < cachedEndTimeColumn.getEndTime()) { + window.setEmptyWindow(nextWindowTimeBegin, nextWindowTimeEnd); + return YieldableState.YIELDABLE; + } + + window.seek( + nextIndexBegin, + nextIndexEnd, + nextWindowTimeBegin, + nextWindowTimeBegin + timeInterval - 1); + + hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == tvList.size()); + return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + @Override + public void readyForNext() { + hasCached = false; + nextWindowTimeBegin += slidingStep; + + tvList.setEvictionUpperBound(nextIndexBegin + 1); + } + + @Override + public TSDataType[] getDataTypes() { + return new TSDataType[] {dataType}; + } + + @Override + public RowWindow currentWindow() { + return window; + } + }; + } + + @Override + protected LayerRowWindowReader constructRowSessionTimeWindowReader( + SessionTimeWindowAccessStrategy strategy, float memoryBudgetInMB) { + + final long displayWindowBegin = strategy.getDisplayWindowBegin(); + final long displayWindowEnd = strategy.getDisplayWindowEnd(); + final long sessionTimeGap = strategy.getSessionTimeGap(); + + final ElasticSerializableTVList tvList = + ElasticSerializableTVList.construct(dataType, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); + final ElasticSerializableTVListBackedSingleColumnWindow window = + new ElasticSerializableTVListBackedSingleColumnWindow(tvList); + + return new LayerRowWindowReader() { + + private boolean isFirstIteration = true; + private boolean hasAtLeastOneRow = false; + + private long nextWindowTimeBegin = displayWindowBegin; + private long nextWindowTimeEnd = 0; + private int nextIndexBegin = 0; + private int nextIndexEnd = 0; + + private TimeColumn cachedTimes; + private int cachedConsumed; + + @Override + public YieldableState yield() throws Exception { + if (isFirstIteration) { + YieldableState state = yieldInFirstIteration(); + if (state != YieldableState.YIELDABLE) { + return state; + } + } + + if (!hasAtLeastOneRow || nextWindowTimeBegin >= displayWindowEnd) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + // Set nextIndexEnd + long curTime = cachedTimes.getLong(cachedConsumed); + if (cachedConsumed < cachedTimes.getPositionCount()) { + nextIndexEnd++; + cachedConsumed++; + } + boolean findWindow = false; + // Find target window or no more data to exit + while (!findWindow) { + while (cachedConsumed < cachedTimes.getPositionCount()) { + long nextTime = cachedTimes.getLong(cachedConsumed); + + if (curTime >= displayWindowEnd) { + nextIndexEnd--; + findWindow = true; + break; + } + + if (nextTime - curTime > sessionTimeGap) { + findWindow = true; + break; + } + nextIndexEnd++; + cachedConsumed++; + curTime = nextTime; + } + + if (!findWindow) { + if (cachedTimes.getEndTime() < displayWindowEnd) { + YieldableState state = yieldAndCache(); + if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } else if (state == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { + break; + } + } + } + } + // Set nextWindowTimeEnd + nextWindowTimeEnd = tvList.getTime(nextIndexEnd - 1); + + if (nextIndexBegin == nextIndexEnd) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + window.seek(nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, nextWindowTimeEnd); + return YieldableState.YIELDABLE; + } + + private YieldableState yieldInFirstIteration() throws Exception { + // Yield initial data in first iteration + if (tvList.size() == 0) { + YieldableState state = yieldAndCache(); + if (state != YieldableState.YIELDABLE) { + return state; + } + } + // Initialize essential information + nextWindowTimeBegin = Math.max(displayWindowBegin, cachedTimes.getStartTime()); + hasAtLeastOneRow = tvList.size() != 0; + isFirstIteration = false; + + // Set initial nextIndexBegin + long currentEndTime = cachedTimes.getEndTime(); + // Find corresponding block + while (currentEndTime < nextWindowTimeBegin) { + // Consume all data + cachedConsumed = cachedTimes.getPositionCount(); + nextIndexBegin += cachedConsumed; + + YieldableState state = yieldAndCache(); + if (state != YieldableState.YIELDABLE) { + // Cannot find nextIndexBegin + // Set nextIndexEnd to nextIndexBegin and exit + nextIndexEnd = nextIndexBegin; + return state; + } + } + // Find nextIndexBegin in block + while (cachedConsumed < cachedTimes.getPositionCount()) { + if (cachedTimes.getLong(cachedConsumed) >= nextWindowTimeBegin) { + break; + } + cachedConsumed++; + nextIndexBegin++; + } + nextIndexEnd = nextIndexBegin; + + return YieldableState.YIELDABLE; + } + + private YieldableState yieldAndCache() throws Exception { + final YieldableState state = parentLayerReader.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + Column[] columns = parentLayerReader.current(); + TimeColumn times = (TimeColumn) columns[1]; + Column values = columns[0]; + + tvList.putColumn(times, values); + parentLayerReader.consumedAll(); + + cachedTimes = times; + cachedConsumed = 0; + + return YieldableState.YIELDABLE; + } + + @Override + public void readyForNext() throws IOException { + if (nextIndexEnd < tvList.size()) { + nextWindowTimeBegin = tvList.getTime(nextIndexEnd); + } + tvList.setEvictionUpperBound(nextIndexBegin + 1); + nextIndexBegin = nextIndexEnd; + } + + @Override + public TSDataType[] getDataTypes() { + return new TSDataType[] {dataType}; + } + + @Override + public RowWindow currentWindow() { + return window; + } + }; + } + + @Override + protected LayerRowWindowReader constructRowStateWindowReader( + StateWindowAccessStrategy strategy, float memoryBudgetInMB) { + + final long displayWindowBegin = strategy.getDisplayWindowBegin(); + final long displayWindowEnd = strategy.getDisplayWindowEnd(); + final double delta = strategy.getDelta(); + + final ElasticSerializableTVList tvList = + ElasticSerializableTVList.construct(dataType, queryId, memoryBudgetInMB, CACHE_BLOCK_SIZE); + final ElasticSerializableTVListBackedSingleColumnWindow window = + new ElasticSerializableTVListBackedSingleColumnWindow(tvList); + + return new LayerRowWindowReader() { + + private boolean isFirstIteration = true; + private boolean hasAtLeastOneRow = false; + + private long nextWindowTimeBegin = displayWindowBegin; + private long nextWindowTimeEnd = 0; + private int nextIndexBegin = 0; + private int nextIndexEnd = 0; + + private TimeColumn cachedTimes; + private Column cachedValues; + private int cachedConsumed; + + private final ValueRecorder valueRecorder = new ValueRecorder(); + + @Override + public YieldableState yield() throws Exception { + if (isFirstIteration) { + YieldableState state = yieldInFirstIteration(); + if (state != YieldableState.YIELDABLE) { + return state; + } + } + + if (!hasAtLeastOneRow || nextWindowTimeBegin >= displayWindowEnd) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + + // Set nextIndexEnd + long curTime = cachedTimes.getLong(cachedConsumed); + if (cachedConsumed < cachedTimes.getPositionCount()) { + nextIndexEnd++; + cachedConsumed++; + } + boolean findWindow = false; + // Find target window or no more data to exit + while (!findWindow) { + while (cachedConsumed < cachedTimes.getPositionCount()) { + long nextTime = cachedTimes.getLong(cachedConsumed); + + if (curTime >= displayWindowEnd) { + nextIndexEnd--; + findWindow = true; + break; + } + + if (TransformUtils.splitWindowForStateWindow( + dataType, valueRecorder, delta, cachedValues, cachedConsumed)) { + findWindow = true; + break; + } + nextIndexEnd++; + cachedConsumed++; + curTime = nextTime; + } + + if (!findWindow) { + if (cachedTimes.getEndTime() < displayWindowEnd) { + YieldableState state = yieldAndCache(); + if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } else if (state == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { + break; + } + } + } + } + // Set nextWindowTimeEnd + nextWindowTimeEnd = tvList.getTime(nextIndexEnd - 1); + + if (nextIndexBegin == nextIndexEnd) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + window.seek(nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, nextWindowTimeEnd); + return YieldableState.YIELDABLE; + } + + private YieldableState yieldInFirstIteration() throws Exception { + // Yield initial data in first iteration + if (tvList.size() == 0) { + YieldableState state = yieldAndCache(); + if (state != YieldableState.YIELDABLE) { + return state; + } + } + // Initialize essential information + nextWindowTimeBegin = Math.max(displayWindowBegin, cachedTimes.getStartTime()); + hasAtLeastOneRow = tvList.size() != 0; + isFirstIteration = false; + + // Set initial nextIndexBegin + long currentEndTime = cachedTimes.getEndTime(); + // Find corresponding block + while (currentEndTime < nextWindowTimeBegin) { + // Consume all data + cachedConsumed = cachedTimes.getPositionCount(); + nextIndexBegin += cachedConsumed; + + YieldableState state = yieldAndCache(); + if (state != YieldableState.YIELDABLE) { + // Cannot find nextIndexBegin + // Set nextIndexEnd to nextIndexBegin and exit + nextIndexEnd = nextIndexBegin; + return state; + } + } + // Find nextIndexBegin in block + while (cachedConsumed < cachedTimes.getPositionCount()) { + if (cachedTimes.getLong(cachedConsumed) >= nextWindowTimeBegin) { + break; + } + cachedConsumed++; + nextIndexBegin++; + } + nextIndexEnd = nextIndexBegin; + + return YieldableState.YIELDABLE; + } + + private YieldableState yieldAndCache() throws Exception { + final YieldableState state = parentLayerReader.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + Column[] columns = parentLayerReader.current(); + TimeColumn times = (TimeColumn) columns[1]; + Column values = columns[0]; + + tvList.putColumn(times, values); + parentLayerReader.consumedAll(); + + cachedTimes = times; + cachedValues = values; + cachedConsumed = 0; + + return YieldableState.YIELDABLE; + } + + @Override + public void readyForNext() throws IOException { + if (nextIndexEnd < tvList.size()) { + nextWindowTimeBegin = tvList.getTime(nextIndexEnd); + } + tvList.setEvictionUpperBound(nextIndexBegin + 1); + nextIndexBegin = nextIndexEnd; + } + + @Override + public TSDataType[] getDataTypes() { + return new TSDataType[] {dataType}; + } + + @Override + public RowWindow currentWindow() { + return window; + } + }; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/memory/SafetyLine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/memory/SafetyLine.java index f036d1fcb49e..e29038f061d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/memory/SafetyLine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/memory/SafetyLine.java @@ -80,5 +80,9 @@ public SafetyPile(int safetyPileIndex) { public void moveForwardTo(int safetyPilePosition) { safetyPiles[safetyPileIndex] = safetyPilePosition; } + + public void moveForward(int steps) { + safetyPiles[safetyPileIndex] += steps; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/Transformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/Transformer.java index 69c15696fab9..73e4905b78f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/Transformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/Transformer.java @@ -19,105 +19,51 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; -import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.block.column.Column; -import java.io.IOException; +public abstract class Transformer implements LayerReader { -public abstract class Transformer implements LayerPointReader { + protected Column[] cachedColumns; + protected int cacheConsumed; - protected boolean hasCachedValue; - - protected long cachedTime; - - protected int cachedInt; - protected long cachedLong; - protected float cachedFloat; - protected double cachedDouble; - protected boolean cachedBoolean; - protected Binary cachedBinary; - protected boolean currentNull; - - protected Transformer() { - hasCachedValue = false; - } - - @Override - public final boolean next() throws QueryProcessException, IOException { - if (!hasCachedValue) { - hasCachedValue = cacheValue(); - } - return hasCachedValue; - } - - /** if this method returns true, at least one of the cached field should be set. */ - protected abstract boolean cacheValue() throws QueryProcessException, IOException; + protected Transformer() {} @Override public final YieldableState yield() throws Exception { - if (hasCachedValue) { + if (cachedColumns != null && cacheConsumed < cachedColumns[0].getPositionCount()) { return YieldableState.YIELDABLE; } - final YieldableState yieldableState = yieldValue(); - if (YieldableState.YIELDABLE == yieldableState) { - hasCachedValue = true; - } - return yieldableState; - } - - /** - * if this method returns YieldableState.YIELDABLE, at least one of the cached field should be - * set. - */ - protected abstract YieldableState yieldValue() throws Exception; - - @Override - public final void readyForNext() { - hasCachedValue = false; - currentNull = false; + // Put concrete logic into yieldValue function call + return yieldValue(); } @Override - public final long currentTime() { - return cachedTime; + public void consumedAll() { + invalidCache(); } @Override - public final int currentInt() { - return cachedInt; - } - - @Override - public final long currentLong() { - return cachedLong; - } - - @Override - public final float currentFloat() { - return cachedFloat; - } + public Column[] current() { + if (cacheConsumed == 0) { + return cachedColumns; + } - @Override - public final double currentDouble() { - return cachedDouble; - } + Column[] ret = new Column[cachedColumns.length]; + for (int i = 0; i < cachedColumns.length; i++) { + ret[i] = cachedColumns[i].subColumn(cacheConsumed); + } - @Override - public final boolean currentBoolean() { - return cachedBoolean; + return ret; } - @Override - public final Binary currentBinary() { - return cachedBinary; + private void invalidCache() { + cacheConsumed = 0; + cachedColumns = null; } - @Override - public final boolean isCurrentNull() { - return currentNull; - } + protected abstract YieldableState yieldValue() throws Exception; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticAdditionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticAdditionTransformer.java index b42e93a2cd60..389d1f44e024 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticAdditionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticAdditionTransformer.java @@ -19,13 +19,12 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; public class ArithmeticAdditionTransformer extends ArithmeticBinaryTransformer { - public ArithmeticAdditionTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); + public ArithmeticAdditionTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticBinaryTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticBinaryTransformer.java index 013d757f0f00..971d91a712f0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticBinaryTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticBinaryTransformer.java @@ -20,43 +20,44 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.UnSupportedDataTypeException; -import java.io.IOException; +import static org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils.castValueToDouble; public abstract class ArithmeticBinaryTransformer extends BinaryTransformer { - protected ArithmeticBinaryTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); + protected ArithmeticBinaryTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override protected void checkType() { - if (leftPointReaderDataType == TSDataType.BOOLEAN - || rightPointReaderDataType == TSDataType.BOOLEAN) { + if (leftReaderDataType == TSDataType.BOOLEAN || rightReaderDataType == TSDataType.BOOLEAN) { throw new UnSupportedDataTypeException(TSDataType.BOOLEAN.name()); } - if (leftPointReaderDataType == TSDataType.TEXT || rightPointReaderDataType == TSDataType.TEXT) { + if (leftReaderDataType == TSDataType.TEXT || rightReaderDataType == TSDataType.TEXT) { throw new UnSupportedDataTypeException(TSDataType.TEXT.name()); } } @Override - protected void transformAndCache() throws QueryProcessException, IOException { - cachedDouble = - evaluate( - castCurrentValueToDoubleOperand(leftPointReader, leftPointReaderDataType), - castCurrentValueToDoubleOperand(rightPointReader, rightPointReaderDataType)); + protected void transformAndCache( + Column leftValues, int leftIndex, Column rightValues, int rightIndex, ColumnBuilder builder) + throws QueryProcessException { + double leftValue = castValueToDouble(leftValues, leftReaderDataType, leftIndex); + double rightValue = castValueToDouble(rightValues, rightReaderDataType, rightIndex); + builder.writeDouble(evaluate(leftValue, rightValue)); } protected abstract double evaluate(double leftOperand, double rightOperand); @Override - public TSDataType getDataType() { - return TSDataType.DOUBLE; + public TSDataType[] getDataTypes() { + return new TSDataType[] {TSDataType.DOUBLE}; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticDivisionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticDivisionTransformer.java index 35cee2f816ba..4d64f75ba4fb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticDivisionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticDivisionTransformer.java @@ -19,13 +19,12 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; public class ArithmeticDivisionTransformer extends ArithmeticBinaryTransformer { - public ArithmeticDivisionTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); + public ArithmeticDivisionTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticModuloTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticModuloTransformer.java index f57787bd5858..e3c92560f14d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticModuloTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticModuloTransformer.java @@ -19,13 +19,12 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; public class ArithmeticModuloTransformer extends ArithmeticBinaryTransformer { - public ArithmeticModuloTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); + public ArithmeticModuloTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticMultiplicationTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticMultiplicationTransformer.java index c71ca73361fd..c0ecaa1c28b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticMultiplicationTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticMultiplicationTransformer.java @@ -19,13 +19,12 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; public class ArithmeticMultiplicationTransformer extends ArithmeticBinaryTransformer { - public ArithmeticMultiplicationTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); + public ArithmeticMultiplicationTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticSubtractionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticSubtractionTransformer.java index 76e6aecfc11e..416e668c9045 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticSubtractionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/ArithmeticSubtractionTransformer.java @@ -19,13 +19,12 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; public class ArithmeticSubtractionTransformer extends ArithmeticBinaryTransformer { - public ArithmeticSubtractionTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); + public ArithmeticSubtractionTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/BinaryTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/BinaryTransformer.java index 307d2bcdece0..e1819be19f5b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/BinaryTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/BinaryTransformer.java @@ -20,35 +20,45 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer; +import org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import java.io.IOException; public abstract class BinaryTransformer extends Transformer { - protected final LayerPointReader leftPointReader; - protected final LayerPointReader rightPointReader; + protected final LayerReader leftReader; + protected final LayerReader rightReader; - protected final TSDataType leftPointReaderDataType; - protected final TSDataType rightPointReaderDataType; + protected final TSDataType leftReaderDataType; + protected final TSDataType rightReaderDataType; - protected final boolean isLeftPointReaderConstant; - protected final boolean isRightPointReaderConstant; + protected final boolean isLeftReaderConstant; + protected final boolean isRightReaderConstant; + + protected Column[] leftColumns; + protected Column[] rightColumns; + + protected int leftConsumed; + protected int rightConsumed; protected final boolean isCurrentConstant; - protected BinaryTransformer(LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - this.leftPointReader = leftPointReader; - this.rightPointReader = rightPointReader; - leftPointReaderDataType = leftPointReader.getDataType(); - rightPointReaderDataType = rightPointReader.getDataType(); - isLeftPointReaderConstant = leftPointReader.isConstantPointReader(); - isRightPointReaderConstant = rightPointReader.isConstantPointReader(); - isCurrentConstant = isLeftPointReaderConstant && isRightPointReaderConstant; + protected BinaryTransformer(LayerReader leftReader, LayerReader rightReader) { + this.leftReader = leftReader; + this.rightReader = rightReader; + leftReaderDataType = leftReader.getDataTypes()[0]; + rightReaderDataType = rightReader.getDataTypes()[0]; + isLeftReaderConstant = leftReader.isConstantPointReader(); + isRightReaderConstant = rightReader.isConstantPointReader(); + isCurrentConstant = isLeftReaderConstant && isRightReaderConstant; checkType(); } @@ -61,157 +71,144 @@ public boolean isConstantPointReader() { @Override public YieldableState yieldValue() throws Exception { - final YieldableState leftYieldableState = leftPointReader.yield(); - final YieldableState rightYieldableState = rightPointReader.yield(); - - if (YieldableState.NOT_YIELDABLE_NO_MORE_DATA.equals(leftYieldableState) - || YieldableState.NOT_YIELDABLE_NO_MORE_DATA.equals(rightYieldableState)) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + // Generate data + if (leftColumns == null) { + YieldableState state = leftReader.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + leftColumns = leftReader.current(); } - - if (YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA.equals(leftYieldableState) - || YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA.equals(rightYieldableState)) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + if (rightColumns == null) { + YieldableState state = rightReader.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + rightColumns = rightReader.current(); } - final YieldableState timeYieldState = yieldTime(); - if (!YieldableState.YIELDABLE.equals(timeYieldState)) { - return timeYieldState; + // Constant folding, or more precisely, constant caching + if (isCurrentConstant && cachedColumns != null) { + return YieldableState.YIELDABLE; } - if (leftPointReader.isCurrentNull() || rightPointReader.isCurrentNull()) { - currentNull = true; - } else { - transformAndCache(); - } + // Merge and transform + int leftCount = leftColumns[0].getPositionCount(); + int rightCount = rightColumns[0].getPositionCount(); + int leftRemains = leftCount - leftConsumed; + int rightRemains = rightCount - rightConsumed; + int expectedEntries = Math.min(leftRemains, rightRemains); + cachedColumns = mergeAndTransformColumns(expectedEntries); - leftPointReader.readyForNext(); - rightPointReader.readyForNext(); return YieldableState.YIELDABLE; } - private YieldableState yieldTime() throws Exception { - if (isCurrentConstant) { - return YieldableState.YIELDABLE; - } - if (isLeftPointReaderConstant) { - cachedTime = rightPointReader.currentTime(); - return YieldableState.YIELDABLE; - } - if (isRightPointReaderConstant) { - cachedTime = leftPointReader.currentTime(); - return YieldableState.YIELDABLE; - } - - long leftTime = leftPointReader.currentTime(); - long rightTime = rightPointReader.currentTime(); + protected Column[] mergeAndTransformColumns(int count) throws QueryProcessException, IOException { + TSDataType outputType = getDataTypes()[0]; + ColumnBuilder timeBuilder = new TimeColumnBuilder(null, count); + ColumnBuilder valueBuilder = TypeUtils.initColumnBuilder(outputType, count); - while (leftTime != rightTime) { - if (leftTime < rightTime) { - leftPointReader.readyForNext(); - final YieldableState leftYieldState = leftPointReader.yield(); - if (!YieldableState.YIELDABLE.equals(leftYieldState)) { - return leftYieldState; - } - leftTime = leftPointReader.currentTime(); - } else { - rightPointReader.readyForNext(); - final YieldableState rightYieldState = rightPointReader.yield(); - if (!YieldableState.YIELDABLE.equals(rightYieldState)) { - return rightYieldState; - } - rightTime = rightPointReader.currentTime(); - } + if (isLeftReaderConstant || isRightReaderConstant) { + return handleConstantColumns(valueBuilder); } - // leftTime == rightTime - cachedTime = leftTime; - return YieldableState.YIELDABLE; + return handleNonConstantColumns(timeBuilder, valueBuilder); } - @Override - protected boolean cacheValue() throws QueryProcessException, IOException { - if (!leftPointReader.next() || !rightPointReader.next()) { - return false; - } + private Column[] handleNonConstantColumns(ColumnBuilder timeBuilder, ColumnBuilder valueBuilder) + throws QueryProcessException, IOException { + Column leftTimes = leftColumns[1], leftValues = leftColumns[0]; + Column rightTimes = rightColumns[1], rightValues = rightColumns[0]; - if (!cacheTime()) { - return false; - } + int leftEnd = leftTimes.getPositionCount(); + int rightEnd = rightTimes.getPositionCount(); - if (leftPointReader.isCurrentNull() || rightPointReader.isCurrentNull()) { - currentNull = true; - } else { - transformAndCache(); - } + // Combine two columns by merge sort + while (leftConsumed < leftEnd && rightConsumed < rightEnd) { + long leftTime = leftTimes.getLong(leftConsumed); + long rightTime = rightTimes.getLong(rightConsumed); - leftPointReader.readyForNext(); - rightPointReader.readyForNext(); - return true; - } + if (leftTime != rightTime) { + if (leftTime < rightTime) { + leftConsumed++; + } else { + rightConsumed++; + } + } else { + timeBuilder.writeLong(leftTime); + if (leftValues.isNull(leftConsumed) || rightValues.isNull(rightConsumed)) { + valueBuilder.appendNull(); + } else { + transformAndCache(leftValues, leftConsumed, rightValues, rightConsumed, valueBuilder); + } - protected abstract void transformAndCache() throws QueryProcessException, IOException; - - /** - * finds the smallest, unconsumed timestamp that exists in both {@code leftPointReader} and {@code - * rightPointReader} and then caches the timestamp in {@code cachedTime}. - * - * @return true if there has a timestamp that meets the requirements - */ - private boolean cacheTime() throws IOException, QueryProcessException { - if (isCurrentConstant) { - return true; + leftConsumed++; + rightConsumed++; + } } - if (isLeftPointReaderConstant) { - cachedTime = rightPointReader.currentTime(); - return true; + + // Clean up + if (leftConsumed == leftEnd) { + leftColumns = null; + leftConsumed = 0; + leftReader.consumedAll(); } - if (isRightPointReaderConstant) { - cachedTime = leftPointReader.currentTime(); - return true; + if (rightConsumed == rightEnd) { + rightColumns = null; + rightConsumed = 0; + rightReader.consumedAll(); } - long leftTime = leftPointReader.currentTime(); - long rightTime = rightPointReader.currentTime(); + Column times = timeBuilder.build(); + Column values = valueBuilder.build(); + return new Column[] {values, times}; + } - while (leftTime != rightTime) { - if (leftTime < rightTime) { - leftPointReader.readyForNext(); - if (!leftPointReader.next()) { - return false; + private Column[] handleConstantColumns(ColumnBuilder valueBuilder) + throws QueryProcessException, IOException { + if (isLeftReaderConstant && isRightReaderConstant) { + transformAndCache(leftColumns[0], 0, rightColumns[0], 0, valueBuilder); + Column constants = valueBuilder.build(); + return new Column[] {constants}; + } else if (isLeftReaderConstant) { + for (int i = 0; i < rightColumns[0].getPositionCount(); i++) { + if (rightColumns[0].isNull(i)) { + valueBuilder.appendNull(); + } else { + transformAndCache(leftColumns[0], 0, rightColumns[0], i, valueBuilder); } - leftTime = leftPointReader.currentTime(); - } else { - rightPointReader.readyForNext(); - if (!rightPointReader.next()) { - return false; + } + Column times = rightColumns[1]; + Column values = valueBuilder.build(); + + // Clean up + rightColumns = null; + rightReader.consumedAll(); + + return new Column[] {values, times}; + } else if (isRightReaderConstant) { + for (int i = 0; i < leftColumns[0].getPositionCount(); i++) { + if (leftColumns[0].isNull(i)) { + valueBuilder.appendNull(); + } else { + transformAndCache(leftColumns[0], i, rightColumns[0], 0, valueBuilder); } - rightTime = rightPointReader.currentTime(); } - } + Column times = leftColumns[1]; + Column values = valueBuilder.build(); - // leftTime == rightTime - cachedTime = leftTime; - return true; - } + // Clean up + leftColumns = null; + leftReader.consumedAll(); - protected static double castCurrentValueToDoubleOperand( - LayerPointReader layerPointReader, TSDataType layerPointReaderDataType) - throws IOException, QueryProcessException { - switch (layerPointReaderDataType) { - case INT32: - return layerPointReader.currentInt(); - case INT64: - return layerPointReader.currentLong(); - case FLOAT: - return layerPointReader.currentFloat(); - case DOUBLE: - return layerPointReader.currentDouble(); - case BOOLEAN: - return layerPointReader.currentBoolean() ? 1.0d : 0.0d; - default: - throw new QueryProcessException( - "Unsupported data type: " + layerPointReader.getDataType().toString()); + return new Column[] {values, times}; } + + // Unreachable + return null; } + + protected abstract void transformAndCache( + Column leftValues, int leftIndex, Column rightValues, int rightIndex, ColumnBuilder builder) + throws QueryProcessException, IOException; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareBinaryTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareBinaryTransformer.java index ebe380059d13..7175c311df4b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareBinaryTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareBinaryTransformer.java @@ -20,8 +20,10 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -32,17 +34,17 @@ public abstract class CompareBinaryTransformer extends BinaryTransformer { @FunctionalInterface protected interface Evaluator { - boolean evaluate() throws QueryProcessException, IOException; + boolean evaluate(Column leftValues, int leftIndex, Column rightValues, int rightIndex) + throws QueryProcessException, IOException; } protected final Evaluator evaluator; - protected CompareBinaryTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) + protected CompareBinaryTransformer(LayerReader leftReader, LayerReader rightReader) throws UnSupportedDataTypeException { - super(leftPointReader, rightPointReader); + super(leftReader, rightReader); evaluator = - TSDataType.TEXT.equals(leftPointReaderDataType) + TSDataType.TEXT.equals(leftReaderDataType) ? constructTextEvaluator() : constructNumberEvaluator(); } @@ -53,27 +55,28 @@ protected CompareBinaryTransformer( @Override protected final void checkType() { - if (leftPointReaderDataType.equals(rightPointReaderDataType)) { + if (leftReaderDataType.equals(rightReaderDataType)) { return; } - if (leftPointReaderDataType.equals(TSDataType.BOOLEAN) - || rightPointReaderDataType.equals(TSDataType.BOOLEAN)) { + if (leftReaderDataType.equals(TSDataType.BOOLEAN) + || rightReaderDataType.equals(TSDataType.BOOLEAN)) { throw new UnSupportedDataTypeException(TSDataType.BOOLEAN.toString()); } - if (leftPointReaderDataType.equals(TSDataType.TEXT) - || rightPointReaderDataType.equals(TSDataType.TEXT)) { + if (leftReaderDataType.equals(TSDataType.TEXT) || rightReaderDataType.equals(TSDataType.TEXT)) { throw new UnSupportedDataTypeException(TSDataType.TEXT.toString()); } } @Override - protected final void transformAndCache() throws QueryProcessException, IOException { - cachedBoolean = evaluator.evaluate(); + protected final void transformAndCache( + Column leftValues, int leftIndex, Column rightValues, int rightIndex, ColumnBuilder builder) + throws QueryProcessException, IOException { + builder.writeBoolean(evaluator.evaluate(leftValues, leftIndex, rightValues, rightIndex)); } @Override - public TSDataType getDataType() { - return TSDataType.BOOLEAN; + public TSDataType[] getDataTypes() { + return new TSDataType[] {TSDataType.BOOLEAN}; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareEqualToTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareEqualToTransformer.java index cc1914e10b0c..9ae12add71c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareEqualToTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareEqualToTransformer.java @@ -19,29 +19,31 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; -public class CompareEqualToTransformer extends CompareBinaryTransformer { +import org.apache.tsfile.block.column.Column; + +import static org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils.castValueToDouble; - public CompareEqualToTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); +public class CompareEqualToTransformer extends CompareBinaryTransformer { + public CompareEqualToTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override protected Evaluator constructNumberEvaluator() { - return () -> + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> Double.compare( - castCurrentValueToDoubleOperand(leftPointReader, leftPointReaderDataType), - castCurrentValueToDoubleOperand(rightPointReader, rightPointReaderDataType)) + castValueToDouble(leftValues, leftReaderDataType, leftIndex), + castValueToDouble(rightValues, rightReaderDataType, rightIndex)) == 0; } @Override protected Evaluator constructTextEvaluator() { - return () -> - TransformUtils.compare(leftPointReader.currentBinary(), rightPointReader.currentBinary()) + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> + TransformUtils.compare(leftValues.getBinary(leftIndex), rightValues.getBinary(rightIndex)) == 0; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareGreaterEqualTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareGreaterEqualTransformer.java index 223a08132c6b..218a82c7f30e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareGreaterEqualTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareGreaterEqualTransformer.java @@ -19,29 +19,31 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; -public class CompareGreaterEqualTransformer extends CompareBinaryTransformer { +import org.apache.tsfile.block.column.Column; + +import static org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils.castValueToDouble; - public CompareGreaterEqualTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); +public class CompareGreaterEqualTransformer extends CompareBinaryTransformer { + public CompareGreaterEqualTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override protected Evaluator constructNumberEvaluator() { - return () -> + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> Double.compare( - castCurrentValueToDoubleOperand(leftPointReader, leftPointReaderDataType), - castCurrentValueToDoubleOperand(rightPointReader, rightPointReaderDataType)) + castValueToDouble(leftValues, leftReaderDataType, leftIndex), + castValueToDouble(rightValues, rightReaderDataType, rightIndex)) >= 0; } @Override protected Evaluator constructTextEvaluator() { - return () -> - TransformUtils.compare(leftPointReader.currentBinary(), rightPointReader.currentBinary()) + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> + TransformUtils.compare(leftValues.getBinary(leftIndex), rightValues.getBinary(rightIndex)) >= 0; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareGreaterThanTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareGreaterThanTransformer.java index 3eeadd8dff9c..a25560965c11 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareGreaterThanTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareGreaterThanTransformer.java @@ -19,29 +19,32 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; +import org.apache.tsfile.block.column.Column; + +import static org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils.castValueToDouble; + public class CompareGreaterThanTransformer extends CompareBinaryTransformer { - public CompareGreaterThanTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); + public CompareGreaterThanTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override protected Evaluator constructNumberEvaluator() { - return () -> + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> Double.compare( - castCurrentValueToDoubleOperand(leftPointReader, leftPointReaderDataType), - castCurrentValueToDoubleOperand(rightPointReader, rightPointReaderDataType)) + castValueToDouble(leftValues, leftReaderDataType, leftIndex), + castValueToDouble(rightValues, rightReaderDataType, rightIndex)) > 0; } @Override protected Evaluator constructTextEvaluator() { - return () -> - TransformUtils.compare(leftPointReader.currentBinary(), rightPointReader.currentBinary()) + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> + TransformUtils.compare(leftValues.getBinary(leftIndex), rightValues.getBinary(rightIndex)) > 0; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareLessEqualTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareLessEqualTransformer.java index 07b0856b8525..9f2783c6d44a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareLessEqualTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareLessEqualTransformer.java @@ -19,29 +19,31 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; -public class CompareLessEqualTransformer extends CompareBinaryTransformer { +import org.apache.tsfile.block.column.Column; + +import static org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils.castValueToDouble; - public CompareLessEqualTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); +public class CompareLessEqualTransformer extends CompareBinaryTransformer { + public CompareLessEqualTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override protected Evaluator constructNumberEvaluator() { - return () -> + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> Double.compare( - castCurrentValueToDoubleOperand(leftPointReader, leftPointReaderDataType), - castCurrentValueToDoubleOperand(rightPointReader, rightPointReaderDataType)) + castValueToDouble(leftValues, leftReaderDataType, leftIndex), + castValueToDouble(rightValues, rightReaderDataType, rightIndex)) <= 0; } @Override protected Evaluator constructTextEvaluator() { - return () -> - TransformUtils.compare(leftPointReader.currentBinary(), rightPointReader.currentBinary()) + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> + TransformUtils.compare(leftValues.getBinary(leftIndex), rightValues.getBinary(rightIndex)) <= 0; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareLessThanTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareLessThanTransformer.java index b191d8f23ca2..ceebcb332654 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareLessThanTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareLessThanTransformer.java @@ -19,29 +19,32 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; +import org.apache.tsfile.block.column.Column; + +import static org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils.castValueToDouble; + public class CompareLessThanTransformer extends CompareBinaryTransformer { - public CompareLessThanTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); + public CompareLessThanTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override protected Evaluator constructNumberEvaluator() { - return () -> + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> Double.compare( - castCurrentValueToDoubleOperand(leftPointReader, leftPointReaderDataType), - castCurrentValueToDoubleOperand(rightPointReader, rightPointReaderDataType)) + castValueToDouble(leftValues, leftReaderDataType, leftIndex), + castValueToDouble(rightValues, rightReaderDataType, rightIndex)) < 0; } @Override protected Evaluator constructTextEvaluator() { - return () -> - TransformUtils.compare(leftPointReader.currentBinary(), rightPointReader.currentBinary()) + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> + TransformUtils.compare(leftValues.getBinary(leftIndex), rightValues.getBinary(rightIndex)) < 0; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareNonEqualTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareNonEqualTransformer.java index 419f3f766225..2bcd252d1633 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareNonEqualTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/CompareNonEqualTransformer.java @@ -19,32 +19,33 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.write.UnSupportedDataTypeException; -public class CompareNonEqualTransformer extends CompareBinaryTransformer { +import static org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils.castValueToDouble; - public CompareNonEqualTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) +public class CompareNonEqualTransformer extends CompareBinaryTransformer { + public CompareNonEqualTransformer(LayerReader leftReader, LayerReader rightReader) throws UnSupportedDataTypeException { - super(leftPointReader, rightPointReader); + super(leftReader, rightReader); } @Override protected Evaluator constructNumberEvaluator() { - return () -> + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> Double.compare( - castCurrentValueToDoubleOperand(leftPointReader, leftPointReaderDataType), - castCurrentValueToDoubleOperand(rightPointReader, rightPointReaderDataType)) + castValueToDouble(leftValues, leftReaderDataType, leftIndex), + castValueToDouble(rightValues, rightReaderDataType, rightIndex)) != 0; } @Override protected Evaluator constructTextEvaluator() { - return () -> - TransformUtils.compare(leftPointReader.currentBinary(), rightPointReader.currentBinary()) + return (Column leftValues, int leftIndex, Column rightValues, int rightIndex) -> + TransformUtils.compare(leftValues.getBinary(leftIndex), rightValues.getBinary(rightIndex)) != 0; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicAndTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicAndTransformer.java index 4162556eb0d9..e11e45ba54a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicAndTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicAndTransformer.java @@ -19,12 +19,12 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; public class LogicAndTransformer extends LogicBinaryTransformer { - public LogicAndTransformer(LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); + public LogicAndTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicBinaryTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicBinaryTransformer.java index f8209dc7170f..7426dac8056d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicBinaryTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicBinaryTransformer.java @@ -19,153 +19,214 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.tsfile.write.UnSupportedDataTypeException; -import java.io.IOException; +import java.util.Optional; public abstract class LogicBinaryTransformer extends BinaryTransformer { + private final int count = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber(); - protected LogicBinaryTransformer( - LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); + private boolean isLeftDone; + private boolean isRightDone; + + protected LogicBinaryTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override protected void checkType() { - if (leftPointReaderDataType != TSDataType.BOOLEAN - || rightPointReaderDataType != TSDataType.BOOLEAN) { + if (leftReaderDataType != TSDataType.BOOLEAN || rightReaderDataType != TSDataType.BOOLEAN) { throw new UnSupportedDataTypeException("Unsupported data type: " + TSDataType.BOOLEAN); } } @Override public YieldableState yieldValue() throws Exception { - final YieldableState leftYieldableState = leftPointReader.yield(); - final YieldableState rightYieldableState = rightPointReader.yield(); - - if (leftYieldableState == YieldableState.YIELDABLE - && rightYieldableState == YieldableState.YIELDABLE) { - cacheValue(leftPointReader, rightPointReader); - return YieldableState.YIELDABLE; + // Generate data + if (leftColumns == null) { + YieldableState state = leftReader.yield(); + + if (state == YieldableState.YIELDABLE) { + leftColumns = leftReader.current(); + } else if (state == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { + isLeftDone = true; + if (isRightDone) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + // When another column has no data + // Threat it as all false values + boolean[] allFalse = new boolean[1]; + Column allFalseColumn = + new RunLengthEncodedColumn(new BooleanColumn(1, Optional.empty(), allFalse), count); + leftColumns = new Column[] {allFalseColumn}; + } else { + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } } - - if (leftYieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA - || rightYieldableState == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } - - if (leftYieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA - && rightYieldableState == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + if (rightColumns == null) { + YieldableState state = rightReader.yield(); + + if (state == YieldableState.YIELDABLE) { + rightColumns = rightReader.current(); + } else if (state == YieldableState.NOT_YIELDABLE_NO_MORE_DATA) { + isRightDone = true; + if (isLeftDone) { + return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + } + // When another column has no data + // Threat it as all false values + boolean[] allFalse = new boolean[1]; + Column allFalseColumn = + new RunLengthEncodedColumn(new BooleanColumn(1, Optional.empty(), allFalse), count); + rightColumns = new Column[] {allFalseColumn}; + } else { + return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; + } } - if (leftYieldableState == YieldableState.YIELDABLE && !isLeftPointReaderConstant) { - cacheValue(leftPointReader); - return YieldableState.YIELDABLE; - } - if (rightYieldableState == YieldableState.YIELDABLE && !isRightPointReaderConstant) { - cacheValue(rightPointReader); + // Constant folding, or more precisely, constant caching + if (isCurrentConstant && cachedColumns != null) { return YieldableState.YIELDABLE; } - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; + // Merge and transform + int leftCount = leftColumns[0].getPositionCount(); + int rightCount = rightColumns[0].getPositionCount(); + int leftRemains = leftCount - leftConsumed; + int rightRemains = rightCount - rightConsumed; + int expectedEntries = Math.min(leftRemains, rightRemains); + cachedColumns = mergeAndTransformColumns(expectedEntries); + + return YieldableState.YIELDABLE; } @Override - protected boolean cacheValue() throws QueryProcessException, IOException { - final boolean leftHasNext = leftPointReader.next(); - final boolean rightHasNext = rightPointReader.next(); - - if (leftHasNext && rightHasNext) { - return cacheValue(leftPointReader, rightPointReader); - } + protected Column[] mergeAndTransformColumns(int count) { + ColumnBuilder timeBuilder = new TimeColumnBuilder(null, count); + ColumnBuilder valueBuilder = new BooleanColumnBuilder(null, count); - if (!leftHasNext && !rightHasNext) { - return false; + if (isLeftReaderConstant || isRightReaderConstant) { + return handleConstantColumns(valueBuilder); } - if (leftHasNext && !isLeftPointReaderConstant) { - return cacheValue(leftPointReader); - } - if (rightHasNext && !isRightPointReaderConstant) { - return cacheValue(rightPointReader); - } - - return false; - } - - private boolean cacheValue(LayerPointReader reader) throws IOException { - cachedTime = reader.currentTime(); - cachedBoolean = !reader.isCurrentNull() && evaluate(false, reader.currentBoolean()); - reader.readyForNext(); - return true; + return handleNonConstantColumns(timeBuilder, valueBuilder); } - private boolean cacheValue(LayerPointReader leftPointReader, LayerPointReader rightPointReader) - throws IOException { - final boolean leftBoolean = - !leftPointReader.isCurrentNull() && leftPointReader.currentBoolean(); - final boolean rightBoolean = - !rightPointReader.isCurrentNull() && rightPointReader.currentBoolean(); - - if (isCurrentConstant) { - cachedBoolean = evaluate(leftBoolean, rightBoolean); - return true; + private Column[] handleNonConstantColumns(ColumnBuilder timeBuilder, ColumnBuilder valueBuilder) { + Column leftTimes = leftColumns[1], leftValues = leftColumns[0]; + Column rightTimes = rightColumns[1], rightValues = rightColumns[0]; + + int leftEnd = leftTimes.getPositionCount(); + int rightEnd = rightTimes.getPositionCount(); + + while (leftConsumed < leftEnd && rightConsumed < rightEnd) { + long leftTime = leftTimes.getLong(leftConsumed); + long rightTime = rightTimes.getLong(rightConsumed); + + if (leftTime != rightTime) { + if (leftTime < rightTime) { + leftConsumed++; + } else { + rightConsumed++; + } + } else { + boolean leftValue = !leftValues.isNull(leftConsumed) && leftValues.getBoolean(leftConsumed); + boolean rightValue = + !rightValues.isNull(rightConsumed) && rightValues.getBoolean(rightConsumed); + boolean result = evaluate(leftValue, rightValue); + valueBuilder.writeBoolean(result); + + leftConsumed++; + rightConsumed++; + } } - if (isLeftPointReaderConstant) { - cachedTime = rightPointReader.currentTime(); - cachedBoolean = evaluate(leftBoolean, rightBoolean); - rightPointReader.readyForNext(); - return true; + // Clean up + if (leftConsumed == leftEnd) { + leftConsumed = 0; + if (!isLeftDone) { + leftColumns = null; + leftReader.consumedAll(); + } } - - if (isRightPointReaderConstant) { - cachedTime = leftPointReader.currentTime(); - cachedBoolean = evaluate(leftBoolean, rightBoolean); - leftPointReader.readyForNext(); - return true; + if (rightConsumed == rightEnd) { + rightConsumed = 0; + if (!isRightDone) { + rightColumns = null; + rightReader.consumedAll(); + } } - final long leftTime = leftPointReader.currentTime(); - final long rightTime = rightPointReader.currentTime(); - - if (leftTime < rightTime) { - cachedTime = leftTime; - cachedBoolean = evaluate(leftBoolean, false); - leftPointReader.readyForNext(); - return true; - } + Column times = timeBuilder.build(); + Column values = valueBuilder.build(); + return new Column[] {values, times}; + } - if (rightTime < leftTime) { - cachedTime = rightTime; - cachedBoolean = evaluate(false, rightBoolean); - rightPointReader.readyForNext(); - return true; + private Column[] handleConstantColumns(ColumnBuilder valueBuilder) { + if (isLeftReaderConstant && isRightReaderConstant) { + boolean leftValue = leftColumns[0].getBoolean(0); + boolean rightValue = rightColumns[0].getBoolean(0); + boolean result = evaluate(leftValue, rightValue); + valueBuilder.writeBoolean(result); + + return new Column[] {valueBuilder.build()}; + } else if (isLeftReaderConstant) { + for (int i = 0; i < rightColumns[0].getPositionCount(); i++) { + boolean leftValue = leftColumns[0].getBoolean(0); + boolean rightValue = !rightColumns[0].isNull(i) && rightColumns[0].getBoolean(i); + boolean result = evaluate(leftValue, rightValue); + valueBuilder.writeBoolean(result); + } + Column times = rightColumns[1]; + Column values = valueBuilder.build(); + + // Clean up + rightColumns = null; + rightReader.consumedAll(); + + return new Column[] {values, times}; + } else if (isRightReaderConstant) { + for (int i = 0; i < leftColumns[0].getPositionCount(); i++) { + boolean leftValue = !leftColumns[0].isNull(0) && leftColumns[0].getBoolean(0); + boolean rightValue = rightColumns[0].getBoolean(0); + boolean result = evaluate(leftValue, rightValue); + valueBuilder.writeBoolean(result); + } + Column times = leftColumns[1]; + Column values = valueBuilder.build(); + + // Clean up + leftColumns = null; + leftReader.consumedAll(); + + return new Column[] {values, times}; } - // == rightTime - cachedTime = leftTime; - cachedBoolean = evaluate(leftBoolean, rightBoolean); - leftPointReader.readyForNext(); - rightPointReader.readyForNext(); - return true; + // Unreachable + return null; } protected abstract boolean evaluate(boolean leftOperand, boolean rightOperand); @Override - protected void transformAndCache() throws QueryProcessException, IOException { + protected void transformAndCache( + Column leftValues, int leftIndex, Column rightValues, int rightIndex, ColumnBuilder builder) { throw new UnsupportedOperationException(); } @Override - public TSDataType getDataType() { - return TSDataType.BOOLEAN; + public TSDataType[] getDataTypes() { + return new TSDataType[] {TSDataType.BOOLEAN}; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicOrTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicOrTransformer.java index 66faec0a8370..f55703345083 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicOrTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/binary/LogicOrTransformer.java @@ -19,12 +19,12 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.binary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; public class LogicOrTransformer extends LogicBinaryTransformer { - public LogicOrTransformer(LayerPointReader leftPointReader, LayerPointReader rightPointReader) { - super(leftPointReader, rightPointReader); + public LogicOrTransformer(LayerReader leftReader, LayerReader rightReader) { + super(leftReader, rightReader); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/MappableUDFQueryRowTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/MappableUDFQueryRowTransformer.java index 6bb3ed512dd0..43d04643a201 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/MappableUDFQueryRowTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/MappableUDFQueryRowTransformer.java @@ -19,109 +19,51 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.multi; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerRowReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.queryengine.transformation.dag.udf.UDTFExecutor; -import org.apache.iotdb.udf.api.access.Row; +import org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.write.UnSupportedDataTypeException; - -import java.io.IOException; -import java.util.Objects; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; public class MappableUDFQueryRowTransformer extends UDFQueryTransformer { - protected final LayerRowReader layerRowReader; - private final boolean isLayerRowReaderConstant; + protected final LayerReader layerReader; - public MappableUDFQueryRowTransformer(LayerRowReader layerRowReader, UDTFExecutor executor) { + public MappableUDFQueryRowTransformer(LayerReader layerReader, UDTFExecutor executor) { super(executor); - this.layerRowReader = layerRowReader; - this.isLayerRowReaderConstant = false; - } - - @Override - protected boolean cacheValue() throws QueryProcessException, IOException { - if (!layerRowReader.next()) { - return false; - } - - if (!isLayerRowReaderConstant) { - cachedTime = layerRowReader.currentTime(); - } - - if (layerRowReader.isCurrentNull()) { - currentNull = true; - } else { - Row row = layerRowReader.currentRow(); - execute(row); - } - - layerRowReader.readyForNext(); - return true; + this.layerReader = layerReader; } @Override protected YieldableState yieldValue() throws Exception { - final YieldableState yieldableState = layerRowReader.yield(); + final YieldableState yieldableState = layerReader.yield(); if (!YieldableState.YIELDABLE.equals(yieldableState)) { return yieldableState; } - if (!isLayerRowReaderConstant) { - cachedTime = layerRowReader.currentTime(); - } - - if (layerRowReader.isCurrentNull()) { - currentNull = true; - } else { - Row row = layerRowReader.currentRow(); - execute(row); - } + Column[] columns = layerReader.current(); + cachedColumns = execute(columns); - layerRowReader.readyForNext(); + layerReader.consumedAll(); return YieldableState.YIELDABLE; } - private void execute(Row row) { - executor.execute(row); - Object currentValue = executor.getCurrentValue(); - if (Objects.isNull(currentValue)) { - currentNull = true; - return; - } - switch (tsDataType) { - case INT32: - case DATE: - cachedInt = (int) currentValue; - break; - case INT64: - case TIMESTAMP: - cachedLong = (long) currentValue; - break; - case FLOAT: - cachedFloat = (float) currentValue; - break; - case DOUBLE: - cachedDouble = (double) currentValue; - break; - case BOOLEAN: - cachedBoolean = (boolean) currentValue; - break; - case TEXT: - case BLOB: - case STRING: - cachedBinary = (Binary) currentValue; - break; - default: - throw new UnSupportedDataTypeException(tsDataType.toString()); - } + private Column[] execute(Column[] columns) { + int count = columns[0].getPositionCount(); + ColumnBuilder valueColumnBuilder = TypeUtils.initColumnBuilder(tsDataType, count); + + executor.execute(columns, valueColumnBuilder); + + int timeColumnIndex = columns.length - 1; + Column timeColumn = columns[timeColumnIndex]; + Column valueColumn = valueColumnBuilder.build(); + return new Column[] {valueColumn, timeColumn}; } @Override public boolean isConstantPointReader() { - return isLayerRowReaderConstant; + return false; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryRowTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryRowTransformer.java index dfd35d722c09..d219332d0ad3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryRowTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryRowTransformer.java @@ -19,46 +19,44 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.multi; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerRowReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.queryengine.transformation.dag.udf.UDTFExecutor; +import org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils; -import java.io.IOException; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; public class UDFQueryRowTransformer extends UniversalUDFQueryTransformer { - protected final LayerRowReader layerRowReader; + protected final LayerReader layerReader; - public UDFQueryRowTransformer(LayerRowReader layerRowReader, UDTFExecutor executor) { + public UDFQueryRowTransformer(LayerReader layerReader, UDTFExecutor executor) { super(executor); - this.layerRowReader = layerRowReader; + this.layerReader = layerReader; } @Override protected YieldableState tryExecuteUDFOnce() throws Exception { - final YieldableState yieldableState = layerRowReader.yield(); + final YieldableState yieldableState = layerReader.yield(); if (yieldableState != YieldableState.YIELDABLE) { return yieldableState; } - if (layerRowReader.isCurrentNull()) { - currentNull = true; - } - executor.execute(layerRowReader.currentRow(), currentNull); - layerRowReader.readyForNext(); + + Column[] columns = layerReader.current(); + cachedColumns = execute(columns); + + layerReader.consumedAll(); return YieldableState.YIELDABLE; } - @Override - protected boolean executeUDFOnce() throws IOException, QueryProcessException { - if (!layerRowReader.next()) { - return false; - } - if (layerRowReader.isCurrentNull()) { - currentNull = true; - } - executor.execute(layerRowReader.currentRow(), currentNull); - layerRowReader.readyForNext(); - return true; + private Column[] execute(Column[] columns) throws Exception { + int count = columns[0].getPositionCount(); + TimeColumnBuilder timeColumnBuilder = new TimeColumnBuilder(null, count); + ColumnBuilder valueColumnBuilder = TypeUtils.initColumnBuilder(tsDataType, count); + + executor.execute(columns, timeColumnBuilder, valueColumnBuilder); + return executor.getCurrentBlock(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryRowWindowTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryRowWindowTransformer.java index 1f90032cb1c9..b0b9a438c776 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryRowWindowTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryRowWindowTransformer.java @@ -19,12 +19,13 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.multi; -import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.transformation.api.LayerRowWindowReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.queryengine.transformation.dag.udf.UDTFExecutor; +import org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils; -import java.io.IOException; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; public class UDFQueryRowWindowTransformer extends UniversalUDFQueryTransformer { @@ -42,18 +43,12 @@ protected YieldableState tryExecuteUDFOnce() throws Exception { if (yieldableState != YieldableState.YIELDABLE) { return yieldableState; } - executor.execute(layerRowWindowReader.currentWindow()); - layerRowWindowReader.readyForNext(); - return YieldableState.YIELDABLE; - } - @Override - protected boolean executeUDFOnce() throws QueryProcessException, IOException { - if (!layerRowWindowReader.next()) { - return false; - } - executor.execute(layerRowWindowReader.currentWindow()); + TimeColumnBuilder timeColumnBuilder = new TimeColumnBuilder(null, 1); + ColumnBuilder valueColumnBuilder = TypeUtils.initColumnBuilder(tsDataType, 1); + + executor.execute(layerRowWindowReader.currentWindow(), timeColumnBuilder, valueColumnBuilder); layerRowWindowReader.readyForNext(); - return true; + return YieldableState.YIELDABLE; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryTransformer.java index fa3a108d2d30..7ba137fe356f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UDFQueryTransformer.java @@ -22,8 +22,11 @@ import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer; import org.apache.iotdb.db.queryengine.transformation.dag.udf.UDTFExecutor; +import org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; public abstract class UDFQueryTransformer extends Transformer { @@ -45,13 +48,16 @@ protected final boolean terminate() { if (terminated) { return false; } - executor.terminate(); + // Some UDTF still generate new data in terminate method + TimeColumnBuilder timeColumnBuilder = new TimeColumnBuilder(null, 1); + ColumnBuilder valueColumnBuilder = TypeUtils.initColumnBuilder(tsDataType, 1); + executor.terminate(timeColumnBuilder, valueColumnBuilder); terminated = true; return true; } @Override - public final TSDataType getDataType() { - return tsDataType; + public final TSDataType[] getDataTypes() { + return new TSDataType[] {tsDataType}; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UniversalUDFQueryTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UniversalUDFQueryTransformer.java index f5d92d08d99a..69e879656ddc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UniversalUDFQueryTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/multi/UniversalUDFQueryTransformer.java @@ -19,24 +19,23 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.multi; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.queryengine.transformation.dag.udf.UDTFExecutor; +import org.apache.iotdb.db.queryengine.transformation.datastructure.iterator.TVListForwardIterator; +import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.ElasticSerializableTVList; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.block.column.Column; import java.io.IOException; public abstract class UniversalUDFQueryTransformer extends UDFQueryTransformer { - - protected final LayerPointReader layerPointReader; - protected final boolean isLayerPointReaderConstant; + protected final ElasticSerializableTVList outputStorage; + protected final TVListForwardIterator outputLayerIterator; protected UniversalUDFQueryTransformer(UDTFExecutor executor) { super(executor); - layerPointReader = executor.getCollector().constructPointReaderUsingTrivialEvictionStrategy(); - isLayerPointReaderConstant = layerPointReader.isConstantPointReader(); + outputStorage = executor.getOutputStorage(); + outputLayerIterator = outputStorage.constructIterator(); } @Override @@ -53,61 +52,25 @@ protected final YieldableState yieldValue() throws Exception { return YieldableState.YIELDABLE; } - @Override - protected final boolean cacheValue() throws QueryProcessException, IOException { - while (!cacheValueFromUDFOutput()) { - if (!executeUDFOnce() && !terminate()) { - return false; - } - } - return true; - } - protected abstract YieldableState tryExecuteUDFOnce() throws Exception; - protected abstract boolean executeUDFOnce() throws QueryProcessException, IOException; - - protected final boolean cacheValueFromUDFOutput() throws QueryProcessException, IOException { - if (!layerPointReader.next()) { + protected final boolean cacheValueFromUDFOutput() throws IOException { + if (!outputLayerIterator.hasNext()) { return false; } - cachedTime = layerPointReader.currentTime(); - if (layerPointReader.isCurrentNull()) { - currentNull = true; - } else { - switch (tsDataType) { - case INT32: - case DATE: - cachedInt = layerPointReader.currentInt(); - break; - case INT64: - case TIMESTAMP: - cachedLong = layerPointReader.currentLong(); - break; - case FLOAT: - cachedFloat = layerPointReader.currentFloat(); - break; - case DOUBLE: - cachedDouble = layerPointReader.currentDouble(); - break; - case BOOLEAN: - cachedBoolean = layerPointReader.currentBoolean(); - break; - case TEXT: - case BLOB: - case STRING: - cachedBinary = layerPointReader.currentBinary(); - break; - default: - throw new UnSupportedDataTypeException(tsDataType.toString()); - } - } - layerPointReader.readyForNext(); + outputLayerIterator.next(); + + Column values = outputLayerIterator.currentValues(); + Column times = outputLayerIterator.currentTimes(); + cachedColumns = new Column[] {values, times}; + + outputStorage.setEvictionUpperBound(outputLayerIterator.getEndPointIndex()); + return true; } @Override public final boolean isConstantPointReader() { - return isLayerPointReaderConstant; + return false; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/BetweenTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/BetweenTransformer.java index 4329a0b1f9a8..1102976bf73f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/BetweenTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/BetweenTransformer.java @@ -21,45 +21,58 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.ternary; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.util.TransformUtils; +import org.apache.tsfile.block.column.Column; + +import static org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils.castValueToDouble; + public class BetweenTransformer extends CompareTernaryTransformer { boolean isNotBetween; public BetweenTransformer( - LayerPointReader firstPointReader, - LayerPointReader secondPointReader, - LayerPointReader thirdPointReader, + LayerReader firstReader, + LayerReader secondReader, + LayerReader thirdReader, boolean isNotBetween) { - super(firstPointReader, secondPointReader, thirdPointReader); + super(firstReader, secondReader, thirdReader); this.isNotBetween = isNotBetween; } @Override protected Evaluator constructNumberEvaluator() { - return () -> + return (Column firstValues, + int firstIndex, + Column secondValues, + int secondIndex, + Column thirdValues, + int thirdIndex) -> ((Double.compare( - castCurrentValueToDoubleOperand(firstPointReader, firstPointReaderDataType), - castCurrentValueToDoubleOperand( - secondPointReader, secondPointReaderDataType)) + castValueToDouble(firstValues, firstReaderDataType, firstIndex), + castValueToDouble(secondValues, secondReaderDataType, secondIndex)) >= 0) && (Double.compare( - castCurrentValueToDoubleOperand(firstPointReader, firstPointReaderDataType), - castCurrentValueToDoubleOperand(thirdPointReader, thirdPointReaderDataType)) + castValueToDouble(firstValues, firstReaderDataType, firstIndex), + castValueToDouble(thirdValues, thirdReaderDataType, thirdIndex)) <= 0)) ^ isNotBetween; } @Override protected Evaluator constructTextEvaluator() { - return () -> + return (Column firstValues, + int firstIndex, + Column secondValues, + int secondIndex, + Column thirdValues, + int thirdIndex) -> ((TransformUtils.compare( - firstPointReader.currentBinary(), secondPointReader.currentBinary()) + firstValues.getBinary(firstIndex), secondValues.getBinary(secondIndex)) >= 0) && (TransformUtils.compare( - firstPointReader.currentBinary(), thirdPointReader.currentBinary()) + firstValues.getBinary(firstIndex), thirdValues.getBinary(thirdIndex)) <= 0)) ^ isNotBetween; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/CompareTernaryTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/CompareTernaryTransformer.java index c52447428857..6e4515878332 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/CompareTernaryTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/CompareTernaryTransformer.java @@ -22,8 +22,10 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.ternary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -34,19 +36,24 @@ public abstract class CompareTernaryTransformer extends TernaryTransformer { @FunctionalInterface public interface Evaluator { - boolean evaluate() throws QueryProcessException, IOException; + boolean evaluate( + Column firstValues, + int firstIndex, + Column secondValues, + int secondIndex, + Column thirdValues, + int thirdIndex) + throws QueryProcessException, IOException; } protected final Evaluator evaluator; protected CompareTernaryTransformer( - LayerPointReader firstPointReader, - LayerPointReader secondPointReader, - LayerPointReader thirdPointReader) + LayerReader firstReader, LayerReader secondReader, LayerReader thirdReader) throws UnSupportedDataTypeException { - super(firstPointReader, secondPointReader, thirdPointReader); + super(firstReader, secondReader, thirdReader); evaluator = - TSDataType.TEXT.equals(firstPointReaderDataType) + TSDataType.TEXT.equals(firstReaderDataType) ? constructTextEvaluator() : constructNumberEvaluator(); } @@ -57,31 +64,41 @@ protected CompareTernaryTransformer( @Override protected final void checkType() { - if ((firstPointReaderDataType).equals(secondPointReaderDataType) - && (firstPointReaderDataType).equals(thirdPointReaderDataType)) { + if ((firstReaderDataType).equals(secondReaderDataType) + && (firstReaderDataType).equals(thirdReaderDataType)) { return; } - if (firstPointReaderDataType.equals(TSDataType.BOOLEAN) - || secondPointReaderDataType.equals(TSDataType.BOOLEAN) - || thirdPointReaderDataType.equals(TSDataType.BOOLEAN)) { + if (firstReaderDataType.equals(TSDataType.BOOLEAN) + || secondReaderDataType.equals(TSDataType.BOOLEAN) + || thirdReaderDataType.equals(TSDataType.BOOLEAN)) { throw new UnSupportedDataTypeException(TSDataType.BOOLEAN.toString()); } - if (firstPointReaderDataType.equals(TSDataType.TEXT) - || secondPointReaderDataType.equals(TSDataType.TEXT) - || thirdPointReaderDataType.equals(TSDataType.TEXT)) { + if (firstReaderDataType.equals(TSDataType.TEXT) + || secondReaderDataType.equals(TSDataType.TEXT) + || thirdReaderDataType.equals(TSDataType.TEXT)) { throw new UnSupportedDataTypeException(TSDataType.TEXT.toString()); } } @Override - protected final void transformAndCache() throws QueryProcessException, IOException { - cachedBoolean = evaluator.evaluate(); + protected final void transformAndCache( + Column firstValues, + int firstIndex, + Column secondValues, + int secondIndex, + Column thirdValues, + int thirdIndex, + ColumnBuilder builder) + throws QueryProcessException, IOException { + builder.writeBoolean( + evaluator.evaluate( + firstValues, firstIndex, secondValues, secondIndex, thirdValues, thirdIndex)); } @Override - public TSDataType getDataType() { - return TSDataType.BOOLEAN; + public TSDataType[] getDataTypes() { + return new TSDataType[] {TSDataType.BOOLEAN}; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/TernaryTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/TernaryTransformer.java index a0ee4f215edc..e2d8980a2c88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/TernaryTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/ternary/TernaryTransformer.java @@ -22,245 +22,223 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.ternary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer; +import org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import java.io.IOException; public abstract class TernaryTransformer extends Transformer { - protected final LayerPointReader firstPointReader; - protected final LayerPointReader secondPointReader; - protected final LayerPointReader thirdPointReader; + protected final LayerReader firstReader; + protected final LayerReader secondReader; + protected final LayerReader thirdReader; - protected final TSDataType firstPointReaderDataType; - protected final TSDataType secondPointReaderDataType; - protected final TSDataType thirdPointReaderDataType; + protected final TSDataType firstReaderDataType; + protected final TSDataType secondReaderDataType; + protected final TSDataType thirdReaderDataType; - protected final boolean isFirstPointReaderConstant; - protected final boolean isSecondPointReaderConstant; - protected final boolean isThirdPointReaderConstant; + protected final boolean isFirstReaderConstant; + protected final boolean isSecondReaderConstant; + protected final boolean isThirdReaderConstant; protected final boolean isCurrentConstant; - @Override - protected YieldableState yieldValue() throws Exception { - final YieldableState firstYieldableState = firstPointReader.yield(); - final YieldableState secondYieldableState = secondPointReader.yield(); - final YieldableState thirdYieldableState = thirdPointReader.yield(); + protected Column[] firstColumns; + protected Column[] secondColumns; + protected Column[] thirdColumns; - if (YieldableState.NOT_YIELDABLE_NO_MORE_DATA.equals(firstYieldableState) - || YieldableState.NOT_YIELDABLE_NO_MORE_DATA.equals(secondYieldableState) - || YieldableState.NOT_YIELDABLE_NO_MORE_DATA.equals(thirdYieldableState)) { - return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; - } + protected int firstConsumed; + protected int secondConsumed; + protected int thirdConsumed; - if (YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA.equals(firstYieldableState) - || YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA.equals(secondYieldableState) - || YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA.equals(thirdYieldableState)) { - return YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA; - } + protected TernaryTransformer( + LayerReader firstReader, LayerReader secondReader, LayerReader thirdReader) { + this.firstReader = firstReader; + this.secondReader = secondReader; + this.thirdReader = thirdReader; + this.firstReaderDataType = firstReader.getDataTypes()[0]; + this.secondReaderDataType = secondReader.getDataTypes()[0]; + this.thirdReaderDataType = thirdReader.getDataTypes()[0]; + this.isFirstReaderConstant = firstReader.isConstantPointReader(); + this.isSecondReaderConstant = secondReader.isConstantPointReader(); + this.isThirdReaderConstant = thirdReader.isConstantPointReader(); + this.isCurrentConstant = + isFirstReaderConstant && isSecondReaderConstant && isThirdReaderConstant; + checkType(); + } - final YieldableState timeYieldState = yieldTime(); - if (!YieldableState.YIELDABLE.equals(timeYieldState)) { - return timeYieldState; + @Override + public YieldableState yieldValue() throws Exception { + // Generate data + if (firstColumns == null) { + YieldableState state = firstReader.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + firstColumns = firstReader.current(); } - - if (firstPointReader.isCurrentNull() - || secondPointReader.isCurrentNull() - || thirdPointReader.isCurrentNull()) { - currentNull = true; - } else { - transformAndCache(); + if (secondColumns == null) { + YieldableState state = secondReader.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + secondColumns = secondReader.current(); } - - firstPointReader.readyForNext(); - secondPointReader.readyForNext(); - thirdPointReader.readyForNext(); - return YieldableState.YIELDABLE; - } - - private YieldableState yieldTime() throws Exception { - if (isCurrentConstant) { - return YieldableState.YIELDABLE; + if (thirdColumns == null) { + YieldableState state = thirdReader.yield(); + if (state != YieldableState.YIELDABLE) { + return state; + } + thirdColumns = thirdReader.current(); } - long firstTime = isFirstPointReaderConstant ? Long.MIN_VALUE : firstPointReader.currentTime(); - long secondTime = - isSecondPointReaderConstant ? Long.MIN_VALUE : secondPointReader.currentTime(); - long thirdTime = isThirdPointReaderConstant ? Long.MIN_VALUE : thirdPointReader.currentTime(); + int firstCount = firstColumns[0].getPositionCount(); + int secondCount = secondColumns[0].getPositionCount(); + int thirdCount = thirdColumns[0].getPositionCount(); + int firstRemains = firstCount - firstConsumed; + int secondRemains = secondCount - secondConsumed; + int thirdRemains = thirdCount - thirdConsumed; - while (firstTime != secondTime || firstTime != thirdTime) { // the logic is similar to MergeSort - if (firstTime < secondTime) { - if (isFirstPointReaderConstant) { - firstTime = secondTime; - } else { - firstPointReader.readyForNext(); - final YieldableState firstYieldableState = firstPointReader.yield(); - if (!YieldableState.YIELDABLE.equals(firstYieldableState)) { - return firstYieldableState; - } - firstTime = firstPointReader.currentTime(); - } - } else if (secondTime < thirdTime) { - if (isSecondPointReaderConstant) { - secondTime = thirdTime; - } else { - secondPointReader.readyForNext(); - final YieldableState secondYieldableState = secondPointReader.yield(); - if (!YieldableState.YIELDABLE.equals(secondYieldableState)) { - return secondYieldableState; - } - secondTime = secondPointReader.currentTime(); - } - } else { - if (isThirdPointReaderConstant) { - thirdTime = firstTime; - } else { - thirdPointReader.readyForNext(); - final YieldableState thirdYieldableState = thirdPointReader.yield(); - if (!YieldableState.YIELDABLE.equals(thirdYieldableState)) { - return thirdYieldableState; - } - thirdTime = thirdPointReader.currentTime(); - } - } - } + int expectedEntries = Math.min(Math.min(firstRemains, secondRemains), thirdRemains); + cachedColumns = mergeAndTransformColumns(expectedEntries); - if (firstTime != Long.MIN_VALUE) { - cachedTime = firstTime; - } return YieldableState.YIELDABLE; } - protected TernaryTransformer( - LayerPointReader firstPointReader, - LayerPointReader secondPointReader, - LayerPointReader thirdPointReader) { - this.firstPointReader = firstPointReader; - this.secondPointReader = secondPointReader; - this.thirdPointReader = thirdPointReader; - this.firstPointReaderDataType = firstPointReader.getDataType(); - this.secondPointReaderDataType = secondPointReader.getDataType(); - this.thirdPointReaderDataType = thirdPointReader.getDataType(); - this.isFirstPointReaderConstant = firstPointReader.isConstantPointReader(); - this.isSecondPointReaderConstant = secondPointReader.isConstantPointReader(); - this.isThirdPointReaderConstant = thirdPointReader.isConstantPointReader(); - this.isCurrentConstant = - isFirstPointReaderConstant && isSecondPointReaderConstant && isThirdPointReaderConstant; - checkType(); - } - @Override public boolean isConstantPointReader() { - return firstPointReader.isConstantPointReader() - && secondPointReader.isConstantPointReader() - && thirdPointReader.isConstantPointReader(); + return firstReader.isConstantPointReader() + && secondReader.isConstantPointReader() + && thirdReader.isConstantPointReader(); } - @Override - protected boolean cacheValue() throws QueryProcessException, IOException { - if (!firstPointReader.next() || !secondPointReader.next() || !thirdPointReader.next()) { - return false; - } + protected Column[] mergeAndTransformColumns(int count) throws QueryProcessException, IOException { + TSDataType outputType = getDataTypes()[0]; + ColumnBuilder timeBuilder = new TimeColumnBuilder(null, count); + ColumnBuilder valueBuilder = TypeUtils.initColumnBuilder(outputType, count); + + int firstEnd = firstColumns[0].getPositionCount(); + int secondEnd = secondColumns[0].getPositionCount(); + int thirdEnd = thirdColumns[0].getPositionCount(); + + while (firstConsumed < firstEnd && secondConsumed < secondEnd && thirdConsumed < thirdEnd) { + long time = findFirstSameTime(); + + if (firstConsumed < firstEnd && secondConsumed < secondEnd && thirdConsumed < thirdEnd) { + if (time != Long.MIN_VALUE) { + timeBuilder.writeLong(time); + if (firstColumns[0].isNull(firstConsumed) + || secondColumns[0].isNull(secondConsumed) + || thirdColumns[0].isNull(thirdConsumed)) { + valueBuilder.appendNull(); + } else { + transformAndCache( + firstColumns[0], + firstConsumed, + secondColumns[0], + secondConsumed, + thirdColumns[0], + thirdConsumed, + valueBuilder); + } + } - if (!cacheTime()) { - return false; + firstConsumed++; + secondConsumed++; + thirdConsumed++; + } } - if (firstPointReader.isCurrentNull() - || secondPointReader.isCurrentNull() - || thirdPointReader.isCurrentNull()) { - currentNull = true; - } else { - transformAndCache(); + // Clean up + if (firstConsumed == firstEnd) { + firstColumns = null; + firstConsumed = 0; + firstReader.consumedAll(); + } + if (secondConsumed == secondEnd) { + secondColumns = null; + secondConsumed = 0; + secondReader.consumedAll(); + } + if (thirdConsumed == thirdEnd) { + thirdColumns = null; + thirdConsumed = 0; + thirdReader.consumedAll(); } - firstPointReader.readyForNext(); - secondPointReader.readyForNext(); - thirdPointReader.readyForNext(); - return true; + Column times = timeBuilder.build(); + Column values = valueBuilder.build(); + return new Column[] {values, times}; } - protected abstract void transformAndCache() throws QueryProcessException, IOException; + private long findFirstSameTime() { + int firstEnd = firstColumns[0].getPositionCount(); + int secondEnd = secondColumns[0].getPositionCount(); + int thirdEnd = thirdColumns[0].getPositionCount(); - protected abstract void checkType(); + long firstTime = getTime(firstReader, firstColumns, firstConsumed); + long secondTime = getTime(secondReader, secondColumns, secondConsumed); + long thirdTime = getTime(thirdReader, thirdColumns, thirdConsumed); - /** - * finds the smallest, unconsumed, same timestamp that exists in {@code firstPointReader}, {@code - * secondPointReader} and {@code thirdPointReader}and then caches the timestamp in {@code - * cachedTime}. - * - * @return true if there has a timestamp that meets the requirements - */ - private boolean cacheTime() throws IOException, QueryProcessException { - boolean isFirstConstant = firstPointReader.isConstantPointReader(); - boolean isSecondConstant = secondPointReader.isConstantPointReader(); - boolean isThirdConstant = thirdPointReader.isConstantPointReader(); - long firstTime = isFirstConstant ? Long.MIN_VALUE : firstPointReader.currentTime(); - long secondTime = isSecondConstant ? Long.MIN_VALUE : secondPointReader.currentTime(); - long thirdTime = isThirdConstant ? Long.MIN_VALUE : secondPointReader.currentTime(); - // Long.MIN_VALUE is used to determine whether (isFirstConstant and isSecondConstant and - // isThirdConstant) is true - while (firstTime != secondTime || firstTime != thirdTime) { // the logic is similar to MergeSort + while (firstTime != secondTime || secondTime != thirdTime) { if (firstTime < secondTime) { - if (isFirstConstant) { + if (isFirstReaderConstant) { firstTime = secondTime; } else { - firstPointReader.readyForNext(); - if (!firstPointReader.next()) { - return false; + firstConsumed++; + if (firstConsumed < firstEnd) { + firstTime = getTime(firstReader, firstColumns, firstConsumed); + } else { + break; } - firstTime = firstPointReader.currentTime(); } } else if (secondTime < thirdTime) { - if (isSecondConstant) { + if (isSecondReaderConstant) { secondTime = thirdTime; } else { - secondPointReader.readyForNext(); - if (!secondPointReader.next()) { - return false; + secondConsumed++; + if (secondConsumed < secondEnd) { + secondTime = getTime(secondReader, secondColumns, secondConsumed); + } else { + break; } - secondTime = secondPointReader.currentTime(); } } else { - if (isThirdConstant) { + if (isThirdReaderConstant) { thirdTime = firstTime; } else { - thirdPointReader.readyForNext(); - if (!thirdPointReader.next()) { - return false; + thirdConsumed++; + if (thirdConsumed < thirdEnd) { + thirdTime = getTime(thirdReader, thirdColumns, thirdConsumed); + } else { + break; } - thirdTime = secondPointReader.currentTime(); } } } - if (firstTime != Long.MIN_VALUE) { - cachedTime = firstTime; - } - return true; + return firstTime; } - protected static double castCurrentValueToDoubleOperand( - LayerPointReader layerPointReader, TSDataType layerPointReaderDataType) - throws IOException, QueryProcessException { - switch (layerPointReaderDataType) { - case INT32: - return layerPointReader.currentInt(); - case INT64: - return layerPointReader.currentLong(); - case FLOAT: - return layerPointReader.currentFloat(); - case DOUBLE: - return layerPointReader.currentDouble(); - case BOOLEAN: - return layerPointReader.currentBoolean() ? 1.0d : 0.0d; - default: - throw new QueryProcessException( - "Unsupported data type: " + layerPointReader.getDataType().toString()); - } + private long getTime(LayerReader reader, Column[] columns, int index) { + return reader.isConstantPointReader() ? Long.MIN_VALUE : columns[1].getLong(index); } + + protected abstract void transformAndCache( + Column firstValues, + int firstIndex, + Column secondValues, + int secondIndex, + Column thirdValues, + int thirdIndex, + ColumnBuilder builder) + throws QueryProcessException, IOException; + + protected abstract void checkType(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java index dd87a8a5681f..5ecd69142d62 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java @@ -20,41 +20,99 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import java.io.IOException; public class ArithmeticNegationTransformer extends UnaryTransformer { - public ArithmeticNegationTransformer(LayerPointReader layerPointReader) { - super(layerPointReader); + public ArithmeticNegationTransformer(LayerReader layerReader) { + super(layerReader); } @Override - public TSDataType getDataType() { - return layerPointReaderDataType; + public TSDataType[] getDataTypes() { + return new TSDataType[] {layerReaderDataType}; } @Override - protected void transformAndCache() throws QueryProcessException, IOException { - switch (layerPointReaderDataType) { + protected void transform(Column[] columns, ColumnBuilder builder) + throws QueryProcessException, IOException { + switch (layerReaderDataType) { case INT32: - cachedInt = -layerPointReader.currentInt(); - break; + transformInt(columns, builder); + return; case INT64: - cachedLong = -layerPointReader.currentLong(); - break; + transformLong(columns, builder); + return; case FLOAT: - cachedFloat = -layerPointReader.currentFloat(); - break; + transformFloat(columns, builder); + return; case DOUBLE: - cachedDouble = -layerPointReader.currentDouble(); - break; + transformDouble(columns, builder); + return; default: - throw new QueryProcessException( - "Unsupported data type: " + layerPointReader.getDataType().toString()); + throw new QueryProcessException("Unsupported data type: " + layerReaderDataType); + } + } + + private void transformInt(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + int[] values = columns[0].getInts(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeInt(-values[i]); + } else { + builder.appendNull(); + } + } + } + + private void transformLong(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + long[] values = columns[0].getLongs(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeLong(-values[i]); + } else { + builder.appendNull(); + } + } + } + + private void transformFloat(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + float[] values = columns[0].getFloats(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeFloat(-values[i]); + } else { + builder.appendNull(); + } + } + } + + private void transformDouble(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + double[] values = columns[0].getDoubles(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeDouble(-values[i]); + } else { + builder.appendNull(); + } } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java index 7e479aed852f..7630112726de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java @@ -20,8 +20,10 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; @@ -41,14 +43,14 @@ public class InTransformer extends UnaryTransformer { private Set booleanSet; private Set stringSet; - public InTransformer(LayerPointReader layerPointReader, boolean isNotIn, Set values) { - super(layerPointReader); + public InTransformer(LayerReader layerReader, boolean isNotIn, Set values) { + super(layerReader); satisfy = isNotIn ? new NotInSatisfy() : new InSatisfy(); initTypedSet(values); } private void initTypedSet(Set values) { - switch (layerPointReaderDataType) { + switch (layerReaderDataType) { case INT32: case DATE: intSet = new HashSet<>(); @@ -86,48 +88,132 @@ private void initTypedSet(Set values) { stringSet = values; break; default: - throw new UnsupportedOperationException( - "unsupported data type: " + layerPointReader.getDataType()); + throw new UnsupportedOperationException("unsupported data type: " + layerReaderDataType); } } @Override - public TSDataType getDataType() { - return TSDataType.BOOLEAN; + public TSDataType[] getDataTypes() { + return new TSDataType[] {TSDataType.BOOLEAN}; } @Override - protected void transformAndCache() throws QueryProcessException, IOException { - switch (layerPointReaderDataType) { + protected void transform(Column[] columns, ColumnBuilder builder) + throws QueryProcessException, IOException { + switch (layerReaderDataType) { case INT32: case DATE: - int intValue = layerPointReader.currentInt(); - cachedBoolean = satisfy.of(intValue); - break; + transformInt(columns, builder); + return; case INT64: case TIMESTAMP: - long longValue = layerPointReader.currentLong(); - cachedBoolean = satisfy.of(longValue); - break; + transformLong(columns, builder); + return; case FLOAT: - float floatValue = layerPointReader.currentFloat(); - cachedBoolean = satisfy.of(floatValue); - break; + transformFloat(columns, builder); + return; case DOUBLE: - double doubleValue = layerPointReader.currentDouble(); - cachedBoolean = satisfy.of(doubleValue); - break; + transformDouble(columns, builder); + return; case BOOLEAN: - boolean booleanValue = layerPointReader.currentBoolean(); - cachedBoolean = satisfy.of(booleanValue); - break; + transformBoolean(columns, builder); + return; case TEXT: case STRING: - Binary binaryValue = layerPointReader.currentBinary(); - cachedBoolean = satisfy.of(binaryValue.getStringValue(TSFileConfig.STRING_CHARSET)); - break; + transformBinary(columns, builder); + return; default: - throw new QueryProcessException("unsupported data type: " + layerPointReader.getDataType()); + throw new QueryProcessException("unsupported data type: " + layerReaderDataType); + } + } + + private void transformInt(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + int[] values = columns[0].getInts(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + boolean res = satisfy.of(values[i]); + builder.writeBoolean(res); + } else { + builder.appendNull(); + } + } + } + + private void transformLong(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + long[] values = columns[0].getLongs(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + boolean res = satisfy.of(values[i]); + builder.writeBoolean(res); + } else { + builder.appendNull(); + } + } + } + + private void transformFloat(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + float[] values = columns[0].getFloats(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + boolean res = satisfy.of(values[i]); + builder.writeBoolean(res); + } else { + builder.appendNull(); + } + } + } + + private void transformDouble(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + double[] values = columns[0].getDoubles(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + boolean res = satisfy.of(values[i]); + builder.writeBoolean(res); + } else { + builder.appendNull(); + } + } + } + + private void transformBoolean(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + boolean[] values = columns[0].getBooleans(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + boolean res = satisfy.of(values[i]); + builder.writeBoolean(res); + } else { + builder.appendNull(); + } + } + } + + private void transformBinary(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + Binary[] values = columns[0].getBinaries(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + boolean res = satisfy.of(values[i].getStringValue(TSFileConfig.STRING_CHARSET)); + builder.writeBoolean(res); + } else { + builder.appendNull(); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/IsNullTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/IsNullTransformer.java index f88ea436c422..aa6da69532a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/IsNullTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/IsNullTransformer.java @@ -20,9 +20,10 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; -import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import java.io.IOException; @@ -30,31 +31,14 @@ public class IsNullTransformer extends UnaryTransformer { private final boolean isNot; - public IsNullTransformer(LayerPointReader layerPointReader, boolean isNot) { - super(layerPointReader); + public IsNullTransformer(LayerReader layerReader, boolean isNot) { + super(layerReader); this.isNot = isNot; } @Override - public TSDataType getDataType() { - return TSDataType.BOOLEAN; - } - - @Override - public final YieldableState yieldValue() throws Exception { - final YieldableState yieldableState = layerPointReader.yield(); - if (!YieldableState.YIELDABLE.equals(yieldableState)) { - return yieldableState; - } - - if (!isLayerPointReaderConstant) { - cachedTime = layerPointReader.currentTime(); - } - - transformAndCache(); - - layerPointReader.readyForNext(); - return YieldableState.YIELDABLE; + public TSDataType[] getDataTypes() { + return new TSDataType[] {TSDataType.BOOLEAN}; } /* @@ -65,7 +49,12 @@ public final YieldableState yieldValue() throws Exception { * So we need use '^' here. */ @Override - protected void transformAndCache() throws QueryProcessException, IOException { - cachedBoolean = layerPointReader.isCurrentNull() ^ isNot; + protected void transform(Column[] columns, ColumnBuilder builder) + throws QueryProcessException, IOException { + int count = columns[0].getPositionCount(); + boolean[] isNulls = columns[0].isNull(); + for (int i = 0; i < count; i++) { + builder.writeBoolean(isNulls[i] ^ isNot); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/LogicNotTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/LogicNotTransformer.java index 65a432e41191..50f9a9e55736 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/LogicNotTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/LogicNotTransformer.java @@ -20,8 +20,10 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -29,22 +31,34 @@ public class LogicNotTransformer extends UnaryTransformer { - public LogicNotTransformer(LayerPointReader layerPointReader) { - super(layerPointReader); + public LogicNotTransformer(LayerReader layerReader) { + super(layerReader); - if (layerPointReaderDataType != TSDataType.BOOLEAN) { + if (layerReaderDataType != TSDataType.BOOLEAN) { throw new UnSupportedDataTypeException( - "Unsupported data type: " + layerPointReader.getDataType().toString()); + "Unsupported data type: " + layerReader.getDataTypes()[0].toString()); } } @Override - public TSDataType getDataType() { - return TSDataType.BOOLEAN; + public TSDataType[] getDataTypes() { + return new TSDataType[] {TSDataType.BOOLEAN}; } @Override - protected void transformAndCache() throws QueryProcessException, IOException { - cachedBoolean = !layerPointReader.currentBoolean(); + protected void transform(Column[] columns, ColumnBuilder builder) + throws QueryProcessException, IOException { + int count = columns[0].getPositionCount(); + boolean[] values = columns[0].getBooleans(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + boolean res = !values[i]; + builder.writeBoolean(res); + } else { + builder.appendNull(); + } + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/RegularTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/RegularTransformer.java index 959f0259e45b..61cfb15d82c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/RegularTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/RegularTransformer.java @@ -20,8 +20,10 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; @@ -34,24 +36,35 @@ public class RegularTransformer extends UnaryTransformer { private final Pattern pattern; - public RegularTransformer(LayerPointReader layerPointReader, Pattern pattern) { - super(layerPointReader); + public RegularTransformer(LayerReader layerReader, Pattern pattern) { + super(layerReader); this.pattern = pattern; - if (layerPointReaderDataType != TSDataType.TEXT) { - throw new UnSupportedDataTypeException( - "Unsupported data type: " + layerPointReader.getDataType().toString()); + if (layerReaderDataType != TSDataType.TEXT) { + throw new UnSupportedDataTypeException("Unsupported data type: " + layerReaderDataType); } } @Override - public TSDataType getDataType() { - return TSDataType.BOOLEAN; + public TSDataType[] getDataTypes() { + return new TSDataType[] {TSDataType.BOOLEAN}; } @Override - protected void transformAndCache() throws QueryProcessException, IOException { - Binary binary = layerPointReader.currentBinary(); - cachedBoolean = pattern.matcher(binary.getStringValue(TSFileConfig.STRING_CHARSET)).find(); + protected void transform(Column[] columns, ColumnBuilder builder) + throws QueryProcessException, IOException { + int count = columns[0].getPositionCount(); + Binary[] binaries = columns[0].getBinaries(); + boolean[] isNulls = columns[0].isNull(); + + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + boolean res = + pattern.matcher(binaries[i].getStringValue(TSFileConfig.STRING_CHARSET)).find(); + builder.writeBoolean(res); + } else { + builder.appendNull(); + } + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/TransparentTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/TransparentTransformer.java index dc5e4215e5f5..da73f54bb64c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/TransparentTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/TransparentTransformer.java @@ -20,8 +20,9 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import java.io.IOException; @@ -34,39 +35,17 @@ *

It's mainly used for a UDF with aggregation query as its parameters. */ public class TransparentTransformer extends UnaryTransformer { - - public TransparentTransformer(LayerPointReader layerPointReader) { - super(layerPointReader); + public TransparentTransformer(LayerReader layerReader) { + super(layerReader); } @Override - public TSDataType getDataType() { - return layerPointReaderDataType; + public TSDataType[] getDataTypes() { + return new TSDataType[] {layerReaderDataType}; } @Override - protected void transformAndCache() throws QueryProcessException, IOException { - switch (layerPointReaderDataType) { - case BOOLEAN: - cachedBoolean = layerPointReader.currentBoolean(); - break; - case DOUBLE: - cachedDouble = layerPointReader.currentDouble(); - break; - case FLOAT: - cachedFloat = layerPointReader.currentFloat(); - break; - case INT32: - cachedInt = layerPointReader.currentInt(); - break; - case INT64: - cachedLong = layerPointReader.currentLong(); - break; - case TEXT: - cachedBinary = layerPointReader.currentBinary(); - break; - default: - throw new QueryProcessException("unsupported data type: " + layerPointReader.getDataType()); - } + protected Column[] transform(Column[] columns) throws QueryProcessException, IOException { + return columns; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/UnaryTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/UnaryTransformer.java index 01c394fc2a3f..f9a246ba3073 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/UnaryTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/UnaryTransformer.java @@ -20,71 +20,63 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer; +import org.apache.iotdb.db.queryengine.transformation.dag.util.TypeUtils; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import java.io.IOException; public abstract class UnaryTransformer extends Transformer { - protected final LayerPointReader layerPointReader; - protected final TSDataType layerPointReaderDataType; - protected final boolean isLayerPointReaderConstant; + protected final LayerReader layerReader; + protected final TSDataType layerReaderDataType; + protected final boolean isLayerReaderConstant; - protected UnaryTransformer(LayerPointReader layerPointReader) { - this.layerPointReader = layerPointReader; - layerPointReaderDataType = layerPointReader.getDataType(); - isLayerPointReaderConstant = layerPointReader.isConstantPointReader(); + protected UnaryTransformer(LayerReader layerReader) { + this.layerReader = layerReader; + layerReaderDataType = layerReader.getDataTypes()[0]; + isLayerReaderConstant = layerReader.isConstantPointReader(); } @Override public final boolean isConstantPointReader() { - return isLayerPointReaderConstant; + return isLayerReaderConstant; } @Override public YieldableState yieldValue() throws Exception { - final YieldableState yieldableState = layerPointReader.yield(); - if (!YieldableState.YIELDABLE.equals(yieldableState)) { - return yieldableState; + final YieldableState state = layerReader.yield(); + if (!YieldableState.YIELDABLE.equals(state)) { + return state; } - if (!isLayerPointReaderConstant) { - cachedTime = layerPointReader.currentTime(); - } - - if (layerPointReader.isCurrentNull()) { - currentNull = true; - } else { - transformAndCache(); - } + Column[] columns = layerReader.current(); + cachedColumns = transform(columns); + layerReader.consumedAll(); - layerPointReader.readyForNext(); return YieldableState.YIELDABLE; } - @Override - protected final boolean cacheValue() throws QueryProcessException, IOException { - if (!layerPointReader.next()) { - return false; - } - - if (!isLayerPointReaderConstant) { - cachedTime = layerPointReader.currentTime(); - } + protected Column[] transform(Column[] columns) throws QueryProcessException, IOException { + // Prepare ColumnBuilder with given type + int count = columns[0].getPositionCount(); + TSDataType dataType = getDataTypes()[0]; + ColumnBuilder builder = TypeUtils.initColumnBuilder(dataType, count); - if (layerPointReader.isCurrentNull()) { - currentNull = true; - } else { - transformAndCache(); - } + // Transform data + transform(columns, builder); - layerPointReader.readyForNext(); - return true; + // Build cache + Column valueColumn = builder.build(); + Column timeColumn = columns[1]; + return new Column[] {valueColumn, timeColumn}; } - protected abstract void transformAndCache() throws QueryProcessException, IOException; + protected void transform(Column[] columns, ColumnBuilder builder) + throws QueryProcessException, IOException {} } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java index aca3010a69ea..9c6ba87da66a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java @@ -21,11 +21,19 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.plan.expression.multi.builtin.helper.CastFunctionHelper; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.UnaryTransformer; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; +import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; +import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; @@ -34,198 +42,459 @@ public class CastFunctionTransformer extends UnaryTransformer { private final TSDataType targetDataType; - public CastFunctionTransformer(LayerPointReader layerPointReader, TSDataType targetDataType) { - super(layerPointReader); + public CastFunctionTransformer(LayerReader layerReader, TSDataType targetDataType) { + super(layerReader); this.targetDataType = targetDataType; } @Override - public TSDataType getDataType() { - return targetDataType; + public TSDataType[] getDataTypes() { + return new TSDataType[] {targetDataType}; } @Override - protected void transformAndCache() throws QueryProcessException, IOException { - switch (layerPointReaderDataType) { + protected Column[] transform(Column[] columns) throws QueryProcessException, IOException { + switch (layerReaderDataType) { case INT32: - cast(layerPointReader.currentInt()); - return; + return castInts(columns); case INT64: - cast(layerPointReader.currentLong()); - return; + return castLongs(columns); case FLOAT: - cast(layerPointReader.currentFloat()); - return; + return castFloats(columns); case DOUBLE: - cast(layerPointReader.currentDouble()); - return; + return castDoubles(columns); case BOOLEAN: - cast(layerPointReader.currentBoolean()); - return; + return castBooleans(columns); case TEXT: - cast(layerPointReader.currentBinary()); - return; + return castBinaries(columns); default: throw new UnsupportedOperationException( - String.format("Unsupported source dataType: %s", layerPointReaderDataType)); + String.format("Unsupported source dataType: %s", layerReaderDataType)); } } - private void cast(int value) { + private Column[] castInts(Column[] columns) { + if (targetDataType == TSDataType.INT32) { + return columns; + } + + int count = columns[0].getPositionCount(); + int[] values = columns[0].getInts(); + boolean[] isNulls = columns[0].isNull(); + ColumnBuilder builder; switch (targetDataType) { - case INT32: - cachedInt = value; - return; case INT64: - cachedLong = value; - return; + builder = new LongColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeLong(values[i]); + } else { + builder.appendNull(); + } + } + break; case FLOAT: - cachedFloat = value; - return; + builder = new FloatColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeFloat(values[i]); + } else { + builder.appendNull(); + } + } + break; case DOUBLE: - cachedDouble = value; - return; + builder = new DoubleColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeDouble(values[i]); + } else { + builder.appendNull(); + } + } + break; case BOOLEAN: - cachedBoolean = (value != 0); - return; + builder = new BooleanColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeBoolean(values[i] != 0); + } else { + builder.appendNull(); + } + } + break; case TEXT: - cachedBinary = BytesUtils.valueOf(String.valueOf(value)); - return; + builder = new BinaryColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeBinary(BytesUtils.valueOf(String.valueOf(values[i]))); + } else { + builder.appendNull(); + } + } + break; default: throw new UnsupportedOperationException( - String.format("Unsupported target dataType: %s", layerPointReaderDataType)); + String.format("Unsupported target dataType: %s", layerReaderDataType)); } + + Column valueColumn = builder.build(); + Column timeColumn = columns[1]; + return new Column[] {valueColumn, timeColumn}; } - private void cast(long value) { + private Column[] castLongs(Column[] columns) { + if (targetDataType == TSDataType.INT64) { + return columns; + } + + int count = columns[0].getPositionCount(); + long[] values = columns[0].getLongs(); + boolean[] isNulls = columns[0].isNull(); + ColumnBuilder builder; switch (targetDataType) { case INT32: - cachedInt = CastFunctionHelper.castLongToInt(value); - return; - case INT64: - cachedLong = value; - return; + builder = new IntColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeInt(CastFunctionHelper.castLongToInt(values[i])); + } else { + builder.appendNull(); + } + } + break; case FLOAT: - cachedFloat = value; - return; + builder = new FloatColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeFloat(values[i]); + } else { + builder.appendNull(); + } + } + break; case DOUBLE: - cachedDouble = value; - return; + builder = new DoubleColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeDouble(values[i]); + } else { + builder.appendNull(); + } + } + break; case BOOLEAN: - cachedBoolean = (value != 0L); - return; + builder = new BooleanColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeBoolean(values[i] != 0L); + } else { + builder.appendNull(); + } + } + break; case TEXT: - cachedBinary = BytesUtils.valueOf(String.valueOf(value)); - return; + builder = new BinaryColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeBinary(BytesUtils.valueOf(String.valueOf(values[i]))); + } else { + builder.appendNull(); + } + } + break; default: throw new UnsupportedOperationException( - String.format("Unsupported target dataType: %s", layerPointReaderDataType)); + String.format("Unsupported target dataType: %s", layerReaderDataType)); } + + Column valueColumn = builder.build(); + Column timeColumn = columns[1]; + return new Column[] {valueColumn, timeColumn}; } - private void cast(float value) { + private Column[] castFloats(Column[] columns) { + if (targetDataType == TSDataType.FLOAT) { + return columns; + } + + int count = columns[0].getPositionCount(); + float[] values = columns[0].getFloats(); + boolean[] isNulls = columns[0].isNull(); + ColumnBuilder builder; switch (targetDataType) { case INT32: - cachedInt = CastFunctionHelper.castFloatToInt(value); - return; + builder = new IntColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeInt(CastFunctionHelper.castFloatToInt(values[i])); + } else { + builder.appendNull(); + } + } + break; case INT64: - cachedLong = CastFunctionHelper.castFloatToLong(value); - return; - case FLOAT: - cachedFloat = value; - return; + builder = new LongColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeLong(CastFunctionHelper.castFloatToLong(values[i])); + } else { + builder.appendNull(); + } + } + break; case DOUBLE: - cachedDouble = value; - return; + builder = new DoubleColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeDouble(values[i]); + } else { + builder.appendNull(); + } + } + break; case BOOLEAN: - cachedBoolean = (value != 0f); - return; + builder = new BooleanColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeBoolean(values[i] != 0f); + } else { + builder.appendNull(); + } + } + break; case TEXT: - cachedBinary = BytesUtils.valueOf(String.valueOf(value)); - return; + builder = new BinaryColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeBinary(BytesUtils.valueOf(String.valueOf(values[i]))); + } else { + builder.appendNull(); + } + } + break; default: throw new UnsupportedOperationException( - String.format("Unsupported target dataType: %s", layerPointReaderDataType)); + String.format("Unsupported target dataType: %s", layerReaderDataType)); } + + Column valueColumn = builder.build(); + Column timeColumn = columns[1]; + return new Column[] {valueColumn, timeColumn}; } - private void cast(double value) { + private Column[] castDoubles(Column[] columns) { + if (targetDataType == TSDataType.DOUBLE) { + return columns; + } + + int count = columns[0].getPositionCount(); + double[] values = columns[0].getDoubles(); + boolean[] isNulls = columns[0].isNull(); + ColumnBuilder builder; switch (targetDataType) { case INT32: - cachedInt = CastFunctionHelper.castDoubleToInt(value); - return; + builder = new IntColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeInt(CastFunctionHelper.castDoubleToInt(values[i])); + } else { + builder.appendNull(); + } + } + break; case INT64: - cachedLong = CastFunctionHelper.castDoubleToLong(value); - return; + builder = new LongColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeLong(CastFunctionHelper.castDoubleToLong(values[i])); + } else { + builder.appendNull(); + } + } + break; case FLOAT: - cachedFloat = CastFunctionHelper.castDoubleToFloat(value); - return; - case DOUBLE: - cachedDouble = value; - return; + builder = new FloatColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeFloat(CastFunctionHelper.castDoubleToFloat(values[i])); + } else { + builder.appendNull(); + } + } + break; case BOOLEAN: - cachedBoolean = (value != 0.0); - return; + builder = new BooleanColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeBoolean(values[i] != 0.0); + } else { + builder.appendNull(); + } + } + break; case TEXT: - cachedBinary = BytesUtils.valueOf(String.valueOf(value)); - return; + builder = new BinaryColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeBinary(BytesUtils.valueOf(String.valueOf(values[i]))); + } else { + builder.appendNull(); + } + } + break; default: throw new UnsupportedOperationException( - String.format("Unsupported target dataType: %s", layerPointReaderDataType)); + String.format("Unsupported target dataType: %s", layerReaderDataType)); } + + Column valueColumn = builder.build(); + Column timeColumn = columns[1]; + return new Column[] {valueColumn, timeColumn}; } - private void cast(boolean value) { + private Column[] castBooleans(Column[] columns) { + if (targetDataType == TSDataType.BOOLEAN) { + return columns; + } + + int count = columns[0].getPositionCount(); + boolean[] values = columns[0].getBooleans(); + boolean[] isNulls = columns[0].isNull(); + ColumnBuilder builder; switch (targetDataType) { case INT32: - cachedInt = value ? 1 : 0; - return; + builder = new IntColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeInt(values[i] ? 1 : 0); + } else { + builder.appendNull(); + } + } + break; case INT64: - cachedLong = value ? 1L : 0; - return; + builder = new LongColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeLong(values[i] ? 1L : 0); + } else { + builder.appendNull(); + } + } + break; case FLOAT: - cachedFloat = value ? 1.0f : 0; - return; + builder = new FloatColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeFloat(values[i] ? 1.0f : 0); + } else { + builder.appendNull(); + } + } + break; case DOUBLE: - cachedDouble = value ? 1.0 : 0; - return; - case BOOLEAN: - cachedBoolean = value; - return; + builder = new DoubleColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeDouble(values[i] ? 1.0 : 0); + } else { + builder.appendNull(); + } + } + break; case TEXT: - cachedBinary = BytesUtils.valueOf(String.valueOf(value)); - return; + builder = new BinaryColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + builder.writeBinary(BytesUtils.valueOf(String.valueOf(values[i]))); + } else { + builder.appendNull(); + } + } + break; default: throw new UnsupportedOperationException( - String.format("Unsupported target dataType: %s", layerPointReaderDataType)); + String.format("Unsupported target dataType: %s", layerReaderDataType)); } + + Column valueColumn = builder.build(); + Column timeColumn = columns[1]; + return new Column[] {valueColumn, timeColumn}; } - private void cast(Binary value) { - String stringValue = value.getStringValue(TSFileConfig.STRING_CHARSET); - // could throw exception when parsing string value + private Column[] castBinaries(Column[] columns) { + if (targetDataType == TSDataType.TEXT) { + return columns; + } + + int count = columns[0].getPositionCount(); + Binary[] values = columns[0].getBinaries(); + boolean[] isNulls = columns[0].isNull(); + ColumnBuilder builder; switch (targetDataType) { case INT32: - cachedInt = Integer.parseInt(stringValue); - return; + builder = new IntColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + String str = values[i].getStringValue(TSFileConfig.STRING_CHARSET); + builder.writeInt(Integer.parseInt(str)); + } else { + builder.appendNull(); + } + } + break; case INT64: - cachedLong = Long.parseLong(stringValue); - return; + builder = new LongColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + String str = values[i].getStringValue(TSFileConfig.STRING_CHARSET); + builder.writeLong(Long.parseLong(str)); + } else { + builder.appendNull(); + } + } + break; case FLOAT: - cachedFloat = CastFunctionHelper.castTextToFloat(stringValue); - return; + builder = new FloatColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + String str = values[i].getStringValue(TSFileConfig.STRING_CHARSET); + builder.writeFloat(CastFunctionHelper.castTextToFloat(str)); + } else { + builder.appendNull(); + } + } + break; case DOUBLE: - cachedDouble = CastFunctionHelper.castTextToDouble(stringValue); - return; + builder = new DoubleColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + String str = values[i].getStringValue(TSFileConfig.STRING_CHARSET); + builder.writeDouble(CastFunctionHelper.castTextToDouble(str)); + } else { + builder.appendNull(); + } + } + break; case BOOLEAN: - cachedBoolean = CastFunctionHelper.castTextToBoolean(stringValue); - return; - case TEXT: - cachedBinary = BytesUtils.valueOf(String.valueOf(value)); - return; + builder = new BooleanColumnBuilder(null, count); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + String str = values[i].getStringValue(TSFileConfig.STRING_CHARSET); + builder.writeBoolean(CastFunctionHelper.castTextToBoolean(str)); + } else { + builder.appendNull(); + } + } + break; default: throw new UnsupportedOperationException( - String.format("Unsupported target dataType: %s", layerPointReaderDataType)); + String.format("Unsupported target dataType: %s", layerReaderDataType)); } + + Column valueColumn = builder.build(); + Column timeColumn = columns[1]; + return new Column[] {valueColumn, timeColumn}; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java index 6a082426b075..d7ab2df5c4a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java @@ -20,14 +20,13 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.scalar; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; -import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.UnaryTransformer; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; -import java.io.IOException; - public class DiffFunctionTransformer extends UnaryTransformer { private final boolean ignoreNull; @@ -37,69 +36,129 @@ public class DiffFunctionTransformer extends UnaryTransformer { // indicate whether lastValue is null private boolean lastValueIsNull = true; - public DiffFunctionTransformer(LayerPointReader layerPointReader, boolean ignoreNull) { - super(layerPointReader); + public DiffFunctionTransformer(LayerReader layerReader, boolean ignoreNull) { + super(layerReader); this.ignoreNull = ignoreNull; } @Override - public TSDataType getDataType() { - return TSDataType.DOUBLE; + public TSDataType[] getDataTypes() { + return new TSDataType[] {TSDataType.DOUBLE}; } @Override - public final YieldableState yieldValue() throws Exception { - final YieldableState yieldableState = layerPointReader.yield(); - if (!YieldableState.YIELDABLE.equals(yieldableState)) { - return yieldableState; + public void transform(Column[] columns, ColumnBuilder builder) throws QueryProcessException { + switch (layerReaderDataType) { + case INT32: + transformInt(columns, builder); + return; + case INT64: + transformLong(columns, builder); + return; + case FLOAT: + transformFloat(columns, builder); + return; + case DOUBLE: + transformDouble(columns, builder); + return; + default: + throw new QueryProcessException("Unsupported data type: " + layerReaderDataType); } + } - if (!isLayerPointReaderConstant) { - cachedTime = layerPointReader.currentTime(); - } + private void transformInt(Column[] columns, ColumnBuilder builder) { + // Position count at first iteration is count + // Then it become count - 1 at latter iteration + int count = columns[0].getPositionCount(); - transformAndCache(); + for (int i = 0; i < count; i++) { + if (columns[0].isNull(i)) { + lastValueIsNull |= !ignoreNull; + builder.appendNull(); + } else { + double currentValue = columns[0].getInt(i); - layerPointReader.readyForNext(); - return YieldableState.YIELDABLE; + if (lastValueIsNull) { + builder.appendNull(); + } else { + builder.writeDouble(currentValue - lastValue); + } + + lastValue = currentValue; + lastValueIsNull = false; + } + } } - @Override - protected void transformAndCache() throws QueryProcessException, IOException { - if (layerPointReader.isCurrentNull()) { - currentNull = true; // currValue is null, append null - - // When currValue is null: - // ignoreNull = true, keep lastValueIsNull as before - // ignoreNull = false, update lastValueIsNull to true - lastValueIsNull |= !ignoreNull; - } else { - double currValue; - switch (layerPointReaderDataType) { - case INT32: - currValue = layerPointReader.currentInt(); - break; - case INT64: - currValue = layerPointReader.currentLong(); - break; - case FLOAT: - currValue = layerPointReader.currentFloat(); - break; - case DOUBLE: - currValue = layerPointReader.currentDouble(); - break; - default: - throw new QueryProcessException( - "Unsupported data type: " + layerPointReader.getDataType().toString()); + private void transformLong(Column[] columns, ColumnBuilder builder) { + // Position count at first iteration is count + // Then it become count - 1 at latter iteration + int count = columns[0].getPositionCount(); + + for (int i = 0; i < count; i++) { + if (columns[0].isNull(i)) { + lastValueIsNull |= !ignoreNull; + builder.appendNull(); + } else { + double currentValue = columns[0].getLong(i); + + if (lastValueIsNull) { + builder.appendNull(); + } else { + builder.writeDouble(currentValue - lastValue); + } + + lastValue = currentValue; + lastValueIsNull = false; } - if (lastValueIsNull) { - currentNull = true; // lastValue is null, append null + } + } + + private void transformFloat(Column[] columns, ColumnBuilder builder) { + // Position count at first iteration is count + // Then it become count - 1 at latter iteration + int count = columns[0].getPositionCount(); + + for (int i = 0; i < count; i++) { + if (columns[0].isNull(i)) { + lastValueIsNull |= !ignoreNull; + builder.appendNull(); } else { - cachedDouble = currValue - lastValue; + double currentValue = columns[0].getFloat(i); + + if (lastValueIsNull) { + builder.appendNull(); + } else { + builder.writeDouble(currentValue - lastValue); + } + + lastValue = currentValue; + lastValueIsNull = false; } + } + } + + private void transformDouble(Column[] columns, ColumnBuilder builder) { + // Position count at first iteration is count + // Then it become count - 1 at latter iteration + int count = columns[0].getPositionCount(); + + for (int i = 0; i < count; i++) { + if (columns[0].isNull(i)) { + lastValueIsNull |= !ignoreNull; + builder.appendNull(); + } else { + double currentValue = columns[0].getDouble(i); + + if (lastValueIsNull) { + builder.appendNull(); + } else { + builder.writeDouble(currentValue - lastValue); + } - lastValue = currValue; // currValue is not null, update lastValue - lastValueIsNull = false; + lastValue = currentValue; + lastValueIsNull = false; + } } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/ReplaceFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/ReplaceFunctionTransformer.java index 9c94e031d2c1..45b699db1c33 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/ReplaceFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/ReplaceFunctionTransformer.java @@ -19,38 +19,50 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.scalar; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.UnaryTransformer; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; - -import java.io.IOException; +import org.apache.tsfile.write.UnSupportedDataTypeException; public class ReplaceFunctionTransformer extends UnaryTransformer { private final String from; private final String to; - public ReplaceFunctionTransformer(LayerPointReader layerPointReader, String from, String to) { - super(layerPointReader); + public ReplaceFunctionTransformer(LayerReader layerReader, String from, String to) { + super(layerReader); this.from = from; this.to = to; + + if (layerReaderDataType != TSDataType.TEXT) { + throw new UnSupportedDataTypeException("Unsupported data type: " + layerReaderDataType); + } } @Override - public TSDataType getDataType() { - return TSDataType.TEXT; + public TSDataType[] getDataTypes() { + return new TSDataType[] {TSDataType.TEXT}; } @Override - protected void transformAndCache() throws QueryProcessException, IOException { - cachedBinary = - BytesUtils.valueOf( - layerPointReader - .currentBinary() - .getStringValue(TSFileConfig.STRING_CHARSET) - .replace(from, to)); + public void transform(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + + Binary[] binaries = columns[0].getBinaries(); + boolean[] isNulls = columns[0].isNull(); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + String res = binaries[i].getStringValue(TSFileConfig.STRING_CHARSET).replace(from, to); + Binary bin = BytesUtils.valueOf(res); + builder.writeBinary(bin); + } else { + builder.appendNull(); + } + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java index 4f8dd17cfb4c..24061fb07649 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java @@ -20,9 +20,11 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.scalar; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.UnaryTransformer; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import java.io.IOException; @@ -32,42 +34,96 @@ public class RoundFunctionTransformer extends UnaryTransformer { protected int places; - public RoundFunctionTransformer( - LayerPointReader layerPointReader, TSDataType targetDataType, int places) { - super(layerPointReader); + public RoundFunctionTransformer(LayerReader layerReader, TSDataType targetDataType, int places) { + super(layerReader); this.targetDataType = targetDataType; this.places = places; } @Override - public TSDataType getDataType() { - return targetDataType; + public TSDataType[] getDataTypes() { + return new TSDataType[] {targetDataType}; } @Override - protected void transformAndCache() throws QueryProcessException, IOException { - switch (layerPointReaderDataType) { + protected void transform(Column[] columns, ColumnBuilder builder) + throws QueryProcessException, IOException { + switch (layerReaderDataType) { case INT32: - cachedDouble = - Math.rint(layerPointReader.currentInt() * Math.pow(10, places)) / Math.pow(10, places); + transformInt(columns, builder); return; case INT64: - cachedDouble = - Math.rint(layerPointReader.currentLong() * Math.pow(10, places)) / Math.pow(10, places); + transformLong(columns, builder); return; case FLOAT: - cachedDouble = - Math.rint(layerPointReader.currentFloat() * Math.pow(10, places)) - / Math.pow(10, places); + transformFloat(columns, builder); return; case DOUBLE: - cachedDouble = - Math.rint(layerPointReader.currentDouble() * Math.pow(10, places)) - / Math.pow(10, places); + transformDouble(columns, builder); return; default: throw new UnsupportedOperationException( - String.format("Unsupported source dataType: %s", layerPointReaderDataType)); + String.format("Unsupported source dataType: %s", layerReaderDataType)); + } + } + + private void transformInt(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + + int[] values = columns[0].getInts(); + boolean[] isNulls = columns[0].isNull(); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + double res = Math.rint(values[i] * Math.pow(10, places)) / Math.pow(10, places); + builder.writeDouble(res); + } else { + builder.appendNull(); + } + } + } + + private void transformLong(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + + long[] values = columns[0].getLongs(); + boolean[] isNulls = columns[0].isNull(); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + double res = Math.rint(values[i] * Math.pow(10, places)) / Math.pow(10, places); + builder.writeDouble(res); + } else { + builder.appendNull(); + } + } + } + + private void transformFloat(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + + float[] values = columns[0].getFloats(); + boolean[] isNulls = columns[0].isNull(); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + double res = Math.rint(values[i] * Math.pow(10, places)) / Math.pow(10, places); + builder.writeDouble(res); + } else { + builder.appendNull(); + } + } + } + + private void transformDouble(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + + double[] values = columns[0].getDoubles(); + boolean[] isNulls = columns[0].isNull(); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + double res = Math.rint(values[i] * Math.pow(10, places)) / Math.pow(10, places); + builder.writeDouble(res); + } else { + builder.appendNull(); + } } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/SubStringFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/SubStringFunctionTransformer.java index 0361019ed34d..3952d442c788 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/SubStringFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/SubStringFunctionTransformer.java @@ -19,15 +19,16 @@ package org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.scalar; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.unary.UnaryTransformer; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; - -import java.io.IOException; +import org.apache.tsfile.write.UnSupportedDataTypeException; import static org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.SubStringFunctionColumnTransformer.EMPTY_STRING; @@ -35,32 +36,48 @@ public class SubStringFunctionTransformer extends UnaryTransformer { private int beginPosition; private int endPosition; - public SubStringFunctionTransformer( - LayerPointReader layerPointReader, int beginPosition, int length) { - super(layerPointReader); + public SubStringFunctionTransformer(LayerReader layerReader, int beginPosition, int length) { + super(layerReader); this.endPosition = (length == Integer.MAX_VALUE ? Integer.MAX_VALUE : beginPosition + length - 1); this.beginPosition = beginPosition > 0 ? beginPosition - 1 : 0; + + if (layerReaderDataType != TSDataType.TEXT && layerReaderDataType != TSDataType.STRING) { + throw new UnSupportedDataTypeException("Unsupported data type: " + layerReaderDataType); + } } @Override - public TSDataType getDataType() { - return TSDataType.TEXT; + public TSDataType[] getDataTypes() { + return new TSDataType[] {TSDataType.TEXT}; } @Override - protected void transformAndCache() throws QueryProcessException, IOException { - String currentValue = - layerPointReader.currentBinary().getStringValue(TSFileConfig.STRING_CHARSET); - if (beginPosition >= currentValue.length() || endPosition < 0) { - currentValue = EMPTY_STRING; + public void transform(Column[] columns, ColumnBuilder builder) { + int count = columns[0].getPositionCount(); + + Binary[] binaries = columns[0].getBinaries(); + boolean[] isNulls = columns[0].isNull(); + for (int i = 0; i < count; i++) { + if (!isNulls[i]) { + String value = binaries[i].getStringValue(TSFileConfig.STRING_CHARSET); + Binary result = BytesUtils.valueOf(substring(value)); + builder.writeBinary(result); + } else { + builder.appendNull(); + } + } + } + + private String substring(String str) { + if (beginPosition >= str.length() || endPosition < 0) { + return EMPTY_STRING; } else { - if (endPosition >= currentValue.length()) { - currentValue = currentValue.substring(beginPosition); + if (endPosition >= str.length()) { + return str.substring(beginPosition); } else { - currentValue = currentValue.substring(beginPosition, endPosition); + return str.substring(beginPosition, endPosition); } } - cachedBinary = BytesUtils.valueOf(currentValue); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/udf/UDTFExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/udf/UDTFExecutor.java index 0c0d99b4a5e7..854ebee073c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/udf/UDTFExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/udf/UDTFExecutor.java @@ -21,6 +21,8 @@ import org.apache.iotdb.commons.udf.service.UDFManagementService; import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.adapter.PointCollectorAdaptor; +import org.apache.iotdb.db.queryengine.transformation.dag.util.InputRowUtils; import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.ElasticSerializableTVList; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; @@ -29,10 +31,13 @@ import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.AccessStrategy; +import org.apache.iotdb.udf.api.utils.RowImpl; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,8 +53,13 @@ public class UDTFExecutor { protected final UDTFConfigurations configurations; protected UDTF udtf; - protected ElasticSerializableTVList collector; - protected Object currentValue; + + protected ElasticSerializableTVList outputStorage; + protected PointCollectorAdaptor collector; + protected Column[] cachedColumns; + + protected boolean isFirstIter = true; + protected boolean useBatch = true; public UDTFExecutor(String functionName, ZoneId zoneId) { this.functionName = functionName; @@ -68,8 +78,8 @@ public void beforeStart( // Mappable UDF does not need PointCollector if (!AccessStrategy.AccessStrategyType.MAPPABLE_ROW_BY_ROW.equals( configurations.getAccessStrategy().getAccessStrategyType())) { - collector = - ElasticSerializableTVList.newElasticSerializableTVList( + outputStorage = + ElasticSerializableTVList.construct( UDFDataTypeTransformer.transformToTsDataType(configurations.getOutputDataType()), queryId, collectorMemoryBudgetInMB, @@ -103,28 +113,132 @@ private void reflectAndValidateUDF( public void execute(Row row, boolean isCurrentRowNull) { try { + // Execute UDTF if (isCurrentRowNull) { // A null row will never trigger any UDF computing collector.putNull(row.getTime()); } else { udtf.transform(row, collector); } + // Store output data + TimeColumn timeColumn = collector.buildTimeColumn(); + Column valueColumn = collector.buildValueColumn(); + + cachedColumns = new Column[] {valueColumn, timeColumn}; + outputStorage.putColumn(timeColumn, valueColumn); } catch (Exception e) { onError("transform(Row, PointCollector)", e); } } - public void execute(Row row) { - try { - currentValue = udtf.transform(row); - } catch (Exception e) { - onError("transform(Row)", e); + public void execute( + Column[] columns, TimeColumnBuilder timeColumnBuilder, ColumnBuilder valueColumnBuilder) + throws Exception { + if (isFirstIter) { + try { + batchExecuteRow(columns, timeColumnBuilder, valueColumnBuilder); + useBatch = true; + } catch (UnsupportedOperationException e) { + singleExecuteRow(columns, timeColumnBuilder, valueColumnBuilder); + useBatch = false; + } catch (Exception e) { + onError("Mappable UDTF execution error", e); + } + + isFirstIter = false; + } else { + try { + if (useBatch) { + batchExecuteRow(columns, timeColumnBuilder, valueColumnBuilder); + } else { + singleExecuteRow(columns, timeColumnBuilder, valueColumnBuilder); + } + } catch (Exception e) { + onError("Mappable UDTF execution error", e); + } } } - public void execute(RowWindow rowWindow) { + private void singleExecuteRow( + Column[] columns, TimeColumnBuilder timeColumnBuilder, ColumnBuilder valueColumnBuilder) + throws Exception { + int colCount = columns.length; + int rowCount = columns[0].getPositionCount(); + + // collect input data types from columns + TSDataType[] dataTypes = new TSDataType[colCount]; + for (int i = 0; i < colCount; i++) { + dataTypes[i] = columns[i].getDataType(); + } + + collector = new PointCollectorAdaptor(timeColumnBuilder, valueColumnBuilder); + // iterate each row + for (int i = 0; i < rowCount; i++) { + // collect values from columns + Object[] values = new Object[colCount]; + for (int j = 0; j < colCount; j++) { + values[j] = columns[j].isNull(i) ? null : columns[j].getObject(i); + } + + // construct input row for executor + RowImpl row = new RowImpl(dataTypes); + row.setRowRecord(values); + + boolean isAllNull = InputRowUtils.isAllNull(values); + if (isAllNull) { + // skip row that all fields are null + collector.putNull(row.getTime()); + } else { + // transform each row by default + udtf.transform(row, collector); + } + } + // Store output data + TimeColumn timeColumn = collector.buildTimeColumn(); + Column valueColumn = collector.buildValueColumn(); + + // Some UDTF only generate data in terminate method + if (timeColumn.getPositionCount() != 0) { + cachedColumns = new Column[] {valueColumn, timeColumn}; + outputStorage.putColumn(timeColumn, valueColumn); + } else { + cachedColumns = null; + } + } + + private void batchExecuteRow( + Column[] columns, TimeColumnBuilder timeColumnBuilder, ColumnBuilder valueColumnBuilder) + throws Exception { + udtf.transform(columns, timeColumnBuilder, valueColumnBuilder); + + Column timeColumn = timeColumnBuilder.build(); + Column valueColumn = valueColumnBuilder.build(); + + // Some UDTF only generate data in terminate method + if (timeColumn.getPositionCount() != 0) { + cachedColumns = new Column[] {valueColumn, timeColumn}; + outputStorage.putColumn((TimeColumn) timeColumn, valueColumn); + } else { + cachedColumns = null; + } + } + + public void execute( + RowWindow rowWindow, TimeColumnBuilder timeColumnBuilder, ColumnBuilder valueColumnBuilder) { try { + collector = new PointCollectorAdaptor(timeColumnBuilder, valueColumnBuilder); + // Execute UDTF udtf.transform(rowWindow, collector); + // Store output data + TimeColumn timeColumn = collector.buildTimeColumn(); + Column valueColumn = collector.buildValueColumn(); + + if (timeColumn.getPositionCount() != 0) { + cachedColumns = new Column[] {valueColumn, timeColumn}; + outputStorage.putColumn(timeColumn, valueColumn); + } else { + cachedColumns = null; + } } catch (Exception e) { onError("transform(RowWindow, PointCollector)", e); } @@ -138,13 +252,22 @@ public void execute(Column[] columns, ColumnBuilder builder) { } } - public Object getCurrentValue() { - return currentValue; + public Column[] getCurrentBlock() { + return cachedColumns; } - public void terminate() { + public void terminate(TimeColumnBuilder timeColumnBuilder, ColumnBuilder valueColumnBuilder) { try { + collector = new PointCollectorAdaptor(timeColumnBuilder, valueColumnBuilder); udtf.terminate(collector); + + // Some terminate method won't produce new data + TimeColumn timeColumn = collector.buildTimeColumn(); + Column valueColumn = collector.buildValueColumn(); + if (valueColumn.getPositionCount() != 0) { + cachedColumns = new Column[] {valueColumn, timeColumn}; + outputStorage.putColumn(timeColumn, valueColumn); + } } catch (Exception e) { onError("terminate(PointCollector)", e); } @@ -171,7 +294,7 @@ public UDTFConfigurations getConfigurations() { return configurations; } - public ElasticSerializableTVList getCollector() { - return collector; + public ElasticSerializableTVList getOutputStorage() { + return outputStorage; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/LayerCacheUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/LayerCacheUtils.java index 86e1001c6720..56b2a81c802a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/LayerCacheUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/LayerCacheUtils.java @@ -19,195 +19,70 @@ package org.apache.iotdb.db.queryengine.transformation.dag.util; -import org.apache.iotdb.commons.udf.utils.UDFBinaryTransformer; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.queryengine.transformation.dag.input.IUDFInputDataSet; -import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowRecordList; +import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowList; import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.ElasticSerializableTVList; -import org.apache.tsfile.enums.TSDataType; - -import java.io.IOException; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.column.TimeColumn; public class LayerCacheUtils { private LayerCacheUtils() {} - public static YieldableState yieldPoints( - TSDataType dataType, - LayerPointReader source, - ElasticSerializableTVList target, - int pointNumber) - throws Exception { - int count = 0; - while (count < pointNumber) { - final YieldableState yieldableState = yieldPoint(dataType, source, target); - if (yieldableState != YieldableState.YIELDABLE) { - return yieldableState; - } - ++count; - } - return YieldableState.YIELDABLE; - } - - public static YieldableState yieldPoint( - TSDataType dataType, LayerPointReader source, ElasticSerializableTVList target) + public static YieldableState yieldPoints(LayerReader source, ElasticSerializableTVList target) throws Exception { final YieldableState yieldableState = source.yield(); if (yieldableState != YieldableState.YIELDABLE) { return yieldableState; } - if (source.isCurrentNull()) { - target.putNull(source.currentTime()); - } else { - switch (dataType) { - case INT32: - case DATE: - target.putInt(source.currentTime(), source.currentInt()); - break; - case INT64: - case TIMESTAMP: - target.putLong(source.currentTime(), source.currentLong()); - break; - case FLOAT: - target.putFloat(source.currentTime(), source.currentFloat()); - break; - case DOUBLE: - target.putDouble(source.currentTime(), source.currentDouble()); - break; - case BOOLEAN: - target.putBoolean(source.currentTime(), source.currentBoolean()); - break; - case TEXT: - case BLOB: - case STRING: - target.putBinary( - source.currentTime(), - UDFBinaryTransformer.transformToUDFBinary(source.currentBinary())); - break; - default: - throw new UnsupportedOperationException(dataType.name()); - } - } - - source.readyForNext(); + // Source would generate two columns: + // First column is the value column; + // Second column is always the time column. + Column[] columns = source.current(); + target.putColumn((TimeColumn) columns[1], columns[0]); + source.consumedAll(); return YieldableState.YIELDABLE; } - /** - * try to generate rows from source. - * - * @return number of actually collected, which may be less than or equals to rowsNumber. - */ - public static YieldableState yieldRows( - IUDFInputDataSet source, ElasticSerializableRowRecordList target, int rowsNumber) - throws Exception { - int count = 0; - while (count < rowsNumber) { - final YieldableState yieldableState = yieldRow(source, target); + public static YieldableState yieldPoints( + LayerReader source, ElasticSerializableTVList target, int count) throws Exception { + while (count > 0) { + final YieldableState yieldableState = source.yield(); if (yieldableState != YieldableState.YIELDABLE) { return yieldableState; } - ++count; - } - return YieldableState.YIELDABLE; - } - public static YieldableState yieldRow( - IUDFInputDataSet source, ElasticSerializableRowRecordList target) throws Exception { - final YieldableState yieldableState = source.canYieldNextRowInObjects(); + Column[] columns = source.current(); + target.putColumn((TimeColumn) columns[1], columns[0]); + source.consumedAll(); - if (yieldableState == YieldableState.YIELDABLE) { - target.put(source.nextRowInObjects()); + int size = columns[0].getPositionCount(); + count -= size; } - return yieldableState; - } - /** - * cachePoints in ElasticSerializableTVList. - * - * @return number of actually collected, which may be less than or equals to pointNumber. - */ - public static int cachePoints( - TSDataType dataType, - LayerPointReader source, - ElasticSerializableTVList target, - int pointNumber) - throws QueryProcessException, IOException { - int count = 0; - while (count < pointNumber && cachePoint(dataType, source, target)) { - ++count; - } - return count; + return YieldableState.YIELDABLE; } - public static boolean cachePoint( - TSDataType dataType, LayerPointReader source, ElasticSerializableTVList target) - throws IOException, QueryProcessException { - if (!source.next()) { - return false; - } - - if (source.isCurrentNull()) { - target.putNull(source.currentTime()); - } else { - switch (dataType) { - case INT32: - target.putInt(source.currentTime(), source.currentInt()); - break; - case INT64: - target.putLong(source.currentTime(), source.currentLong()); - break; - case FLOAT: - target.putFloat(source.currentTime(), source.currentFloat()); - break; - case DOUBLE: - target.putDouble(source.currentTime(), source.currentDouble()); - break; - case BOOLEAN: - target.putBoolean(source.currentTime(), source.currentBoolean()); - break; - case TEXT: - target.putBinary( - source.currentTime(), - UDFBinaryTransformer.transformToUDFBinary(source.currentBinary())); - break; - default: - throw new UnsupportedOperationException(dataType.name()); + public static YieldableState yieldRows( + IUDFInputDataSet source, ElasticSerializableRowList target, int count) throws Exception { + while (count > 0) { + final YieldableState yieldableState = source.yield(); + if (yieldableState != YieldableState.YIELDABLE) { + return yieldableState; } - } - - source.readyForNext(); - return true; - } + Column[] columns = source.currentBlock(); + target.put(columns); - /** - * cache rows in ElasticSerializableRowRecordList. - * - * @return number of actually collected, which may be less than or equals to rowsNumber. - */ - public static int cacheRows( - IUDFInputDataSet source, ElasticSerializableRowRecordList target, int rowsNumber) - throws QueryProcessException, IOException { - int count = 0; - while (count < rowsNumber && cacheRow(source, target)) { - ++count; + int size = columns[0].getPositionCount(); + count -= size; } - return count; - } - public static boolean cacheRow(IUDFInputDataSet source, ElasticSerializableRowRecordList target) - throws IOException, QueryProcessException { - if (source.hasNextRowInObjects()) { - target.put(source.nextRowInObjects()); - return true; - } else { - return false; - } + return YieldableState.YIELDABLE; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java index e085a096d382..187386916f5c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java @@ -21,7 +21,6 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand; -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.ElasticSerializableTVList; import org.apache.iotdb.db.queryengine.transformation.datastructure.util.ValueRecorder; import org.apache.iotdb.db.utils.CommonUtils; @@ -36,7 +35,6 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.UnSupportedDataTypeException; -import java.io.IOException; import java.util.Objects; import java.util.Optional; @@ -88,71 +86,69 @@ public static Column transformConstantOperandToColumn(ConstantOperand constantOp } public static boolean splitWindowForStateWindow( - TSDataType dataType, - ValueRecorder valueRecorder, - double delta, - ElasticSerializableTVList tvList) - throws IOException { + TSDataType dataType, ValueRecorder valueRecorder, double delta, Column values, int index) { boolean res; switch (dataType) { case INT32: if (!valueRecorder.hasRecorded()) { - valueRecorder.recordInt(tvList.getInt(tvList.size() - 2)); + valueRecorder.recordInt(values.getInt(index - 1)); valueRecorder.setRecorded(true); } - res = Math.abs(tvList.getInt(tvList.size() - 1) - valueRecorder.getInt()) > delta; + res = Math.abs(values.getInt(index) - valueRecorder.getInt()) > delta; if (res) { - valueRecorder.recordInt(tvList.getInt(tvList.size() - 1)); + valueRecorder.recordInt(values.getInt(index)); } break; case INT64: if (!valueRecorder.hasRecorded()) { - valueRecorder.recordLong(tvList.getLong(tvList.size() - 2)); + valueRecorder.recordLong(values.getLong(index - 1)); valueRecorder.setRecorded(true); } - res = Math.abs(tvList.getLong(tvList.size() - 1) - valueRecorder.getLong()) > delta; + res = Math.abs(values.getLong(index) - valueRecorder.getLong()) > delta; if (res) { - valueRecorder.recordLong(tvList.getLong(tvList.size() - 1)); + valueRecorder.recordLong(values.getLong(index)); } break; case FLOAT: if (!valueRecorder.hasRecorded()) { - valueRecorder.recordFloat(tvList.getFloat(tvList.size() - 2)); + valueRecorder.recordFloat(values.getFloat(index - 1)); valueRecorder.setRecorded(true); } - res = Math.abs(tvList.getFloat(tvList.size() - 1) - valueRecorder.getFloat()) > delta; + res = Math.abs(values.getFloat(index) - valueRecorder.getFloat()) > delta; if (res) { - valueRecorder.recordFloat(tvList.getFloat(tvList.size() - 1)); + valueRecorder.recordFloat(values.getFloat(index)); } break; case DOUBLE: if (!valueRecorder.hasRecorded()) { - valueRecorder.recordDouble(tvList.getDouble(tvList.size() - 2)); + valueRecorder.recordDouble(values.getDouble(index - 1)); valueRecorder.setRecorded(true); } - res = Math.abs(tvList.getDouble(tvList.size() - 1) - valueRecorder.getDouble()) > delta; + res = Math.abs(values.getDouble(index) - valueRecorder.getDouble()) > delta; if (res) { - valueRecorder.recordDouble(tvList.getDouble(tvList.size() - 1)); + valueRecorder.recordDouble(values.getDouble(index)); } break; case BOOLEAN: if (!valueRecorder.hasRecorded()) { - valueRecorder.recordBoolean(tvList.getBoolean(tvList.size() - 2)); + valueRecorder.recordBoolean(values.getBoolean(index - 1)); valueRecorder.setRecorded(true); } - res = tvList.getBoolean(tvList.size() - 1) != valueRecorder.getBoolean(); + res = values.getBoolean(index) != valueRecorder.getBoolean(); if (res) { - valueRecorder.recordBoolean(tvList.getBoolean(tvList.size() - 1)); + valueRecorder.recordBoolean(values.getBoolean(index)); } break; case TEXT: if (!valueRecorder.hasRecorded()) { - valueRecorder.recordString(tvList.getString(tvList.size() - 2)); + Binary binary = values.getBinary(index - 1); + valueRecorder.recordString(binary.toString()); valueRecorder.setRecorded(true); } - res = !tvList.getString(tvList.size() - 1).equals(valueRecorder.getString()); + String str = values.getBinary(index).toString(); + res = !str.equals(valueRecorder.getString()); if (res) { - valueRecorder.recordString(tvList.getString(tvList.size() - 1)); + valueRecorder.recordString(str); } break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java new file mode 100644 index 000000000000..1bdee2344cad --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.util; + +import org.apache.iotdb.db.exception.query.QueryProcessException; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; +import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; +import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +public class TypeUtils { + public static ColumnBuilder initColumnBuilder(TSDataType type, int count) { + switch (type) { + case INT32: + case DATE: + return new IntColumnBuilder(null, count); + case INT64: + case TIMESTAMP: + return new LongColumnBuilder(null, count); + case FLOAT: + return new FloatColumnBuilder(null, count); + case DOUBLE: + return new DoubleColumnBuilder(null, count); + case BOOLEAN: + return new BooleanColumnBuilder(null, count); + case TEXT: + case BLOB: + case STRING: + return new BinaryColumnBuilder(null, count); + default: + throw new UnSupportedDataTypeException( + "Do not support create ColumnBuilder with data type " + type); + } + } + + public static double castValueToDouble(Column column, TSDataType type, int index) + throws QueryProcessException { + switch (type) { + case INT32: + case DATE: + return column.getInt(index); + case INT64: + case TIMESTAMP: + return column.getLong(index); + case FLOAT: + return column.getFloat(index); + case DOUBLE: + return column.getDouble(index); + case BOOLEAN: + return column.getBoolean(index) ? 1 : 0; + default: + throw new QueryProcessException("Unsupported data type: " + type); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/TVColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/TVColumns.java new file mode 100644 index 000000000000..7d1ef5b7e960 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/TVColumns.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.datastructure; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.column.TimeColumn; + +public class TVColumns { + private boolean isConstant; + private TimeColumn timeColumn; + private Column valueColumn; + + public TVColumns(TimeColumn timeColumn, Column valueColumn) { + this.timeColumn = timeColumn; + this.valueColumn = valueColumn; + isConstant = false; + } + + public TVColumns(Column valueColumn) { + this.valueColumn = valueColumn; + isConstant = true; + } + + public int getPositionCount() { + // In case of constant, use valueColumn to get pos count + return valueColumn.getPositionCount(); + } + + public long getTimeByIndex(int index) { + if (isConstant) { + throw new UnsupportedOperationException(); + } + return timeColumn.getLong(index); + } + + public long getEndTime() { + if (isConstant) { + throw new UnsupportedOperationException(); + } + return timeColumn.getEndTime(); + } + + public Column getValueColumn() { + return valueColumn; + } + + public Column getTimeColumn() { + if (isConstant) { + throw new UnsupportedOperationException(); + } + return timeColumn; + } + + public boolean isConstant() { + return isConstant; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerPointReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/iterator/ListForwardIterator.java similarity index 52% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerPointReader.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/iterator/ListForwardIterator.java index 078645417d1a..1a1bf942be9c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/api/LayerPointReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/iterator/ListForwardIterator.java @@ -17,38 +17,12 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.transformation.api; - -import org.apache.iotdb.db.exception.query.QueryProcessException; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; +package org.apache.iotdb.db.queryengine.transformation.datastructure.iterator; import java.io.IOException; -public interface LayerPointReader extends YieldableReader { - - boolean isConstantPointReader(); - - boolean next() throws QueryProcessException, IOException; - - void readyForNext(); - - TSDataType getDataType(); - - long currentTime() throws IOException; - - int currentInt() throws IOException; - - long currentLong() throws IOException; - - float currentFloat() throws IOException; - - double currentDouble() throws IOException; - - boolean currentBoolean() throws IOException; - - boolean isCurrentNull() throws IOException; +public interface ListForwardIterator { + boolean hasNext() throws IOException; - Binary currentBinary() throws IOException; + void next() throws IOException; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/iterator/RowListForwardIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/iterator/RowListForwardIterator.java new file mode 100644 index 000000000000..ee9cf6aaeeff --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/iterator/RowListForwardIterator.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.datastructure.iterator; + +import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowList; +import org.apache.iotdb.db.queryengine.transformation.datastructure.row.SerializableRowList; + +import org.apache.tsfile.block.column.Column; + +import java.io.IOException; + +// Forward iterator used in ElasticSerializableRowRecordList +// Point to columns(one time column and multiple value columns) +public class RowListForwardIterator implements ListForwardIterator { + private final ElasticSerializableRowList rowList; + + private int externalIndex; // Which SerializableRowRecordList + private int internalIndex; // Which columns in SerializableRowRecordList + + // In case of rowList changing + private int endRowIndex; // Index of last row of the columns(open) + + public RowListForwardIterator(ElasticSerializableRowList rowList) { + this.rowList = rowList; + // Point to dummy block for simplicity + externalIndex = 0; + internalIndex = -1; + endRowIndex = 0; + } + + public RowListForwardIterator( + ElasticSerializableRowList rowList, int externalIndex, int internalIndex) throws IOException { + this.rowList = rowList; + this.externalIndex = externalIndex; + this.internalIndex = internalIndex; + endRowIndex = rowList.getLastRowIndex(externalIndex, internalIndex); + } + + public Column[] currentBlock() throws IOException { + return rowList.getColumns(externalIndex, internalIndex); + } + + @Override + public boolean hasNext() throws IOException { + // First time call, rowList has no data + if (rowList.getSerializableRowListSize() == 0) { + return false; + } + return externalIndex + 1 < rowList.getSerializableRowListSize() + || internalIndex + 1 < rowList.getSerializableRowList(externalIndex).getBlockCount(); + } + + @Override + public void next() throws IOException { + // Move forward iterator + if (internalIndex + 1 == rowList.getBlockCount(externalIndex)) { + internalIndex = 0; + externalIndex++; + } else { + internalIndex++; + } + + // Assume we already consume all data in this block + SerializableRowList internalRowList = rowList.getSerializableRowList(externalIndex); + endRowIndex += internalRowList.getBlockSize(internalIndex); + } + + // When rowList apply new memory control strategy, the origin iterators become invalid. + // We can relocate these old iterators by its startPointIndex + public void adjust() throws IOException { + // Ensure the row list capacity is updated + int capacity = rowList.getInternalRowListCapacity(); + + int externalColumnIndex = endRowIndex / capacity; + int internalRowIndex = endRowIndex % capacity; + // endPointIndex is not closed, i.e. endPointIndex) + int internalColumnIndex = + rowList.getSerializableRowList(externalColumnIndex).getColumnIndex(internalRowIndex - 1); + + this.externalIndex = externalColumnIndex; + this.internalIndex = internalColumnIndex; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/iterator/TVListForwardIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/iterator/TVListForwardIterator.java new file mode 100644 index 000000000000..134961afd458 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/iterator/TVListForwardIterator.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.datastructure.iterator; + +import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.ElasticSerializableTVList; +import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableTVList; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.column.TimeColumn; + +import java.io.IOException; + +// Forward iterator used in ElasticSerializableTVList +// Point to columns(time column and value column) +public class TVListForwardIterator implements ListForwardIterator { + private final ElasticSerializableTVList tvList; + + private int externalIndex; // Which SerializableTVList + private int internalIndex; // Which columns in SerializableTVList + + // In case of tvList changing + private int endPointIndex; // Index of last point of the columns(open) + + public TVListForwardIterator(ElasticSerializableTVList tvList) { + this.tvList = tvList; + // Point to dummy block for simplicity + externalIndex = 0; + internalIndex = -1; + endPointIndex = 0; + } + + public TVListForwardIterator( + ElasticSerializableTVList tvList, int externalIndex, int internalIndex) throws IOException { + this.tvList = tvList; + this.externalIndex = externalIndex; + this.internalIndex = internalIndex; + endPointIndex = tvList.getLastPointIndex(externalIndex, internalIndex); + } + + public TimeColumn currentTimes() throws IOException { + return tvList.getTimeColumn(externalIndex, internalIndex); + } + + public Column currentValues() throws IOException { + return tvList.getValueColumn(externalIndex, internalIndex); + } + + @Override + public boolean hasNext() throws IOException { + // First time call, tvList has no data + if (tvList.getSerializableTVListSize() == 0) { + return false; + } + + return externalIndex + 1 < tvList.getSerializableTVListSize() + || internalIndex + 1 < tvList.getSerializableTVList(externalIndex).getColumnCount(); + } + + @Override + public void next() throws IOException { + // Move forward iterator + if (internalIndex + 1 == tvList.getColumnCount(externalIndex)) { + internalIndex = 0; + externalIndex++; + } else { + internalIndex++; + } + + // Assume we already consume all data in this block + SerializableTVList internalTVList = tvList.getSerializableTVList(externalIndex); + endPointIndex += internalTVList.getColumnSize(internalIndex); + } + + // When tvList apply new memory control strategy, the origin iterators become invalid. + // We can relocate these old iterators by its startPointIndex + public void adjust() throws IOException { + int capacity = tvList.getInternalTVListCapacity(); + + int externalColumnIndex = endPointIndex / capacity; + int internalPointIndex = endPointIndex % capacity; + // endPointIndex is not closed, i.e. endPointIndex) + int internalColumnIndex = + tvList.getSerializableTVList(externalIndex).getColumnIndex(internalPointIndex - 1); + + this.externalIndex = externalColumnIndex; + this.internalIndex = internalColumnIndex; + } + + public int getEndPointIndex() { + return endPointIndex; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/ElasticSerializableRowList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/ElasticSerializableRowList.java new file mode 100644 index 000000000000..d3e2cecb7f5a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/ElasticSerializableRowList.java @@ -0,0 +1,485 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.datastructure.row; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.transformation.datastructure.Cache; +import org.apache.iotdb.db.queryengine.transformation.datastructure.SerializableList; +import org.apache.iotdb.db.queryengine.transformation.datastructure.iterator.RowListForwardIterator; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** An elastic list of records that implements memory control using LRU strategy. */ +public class ElasticSerializableRowList { + protected static final int MEMORY_CHECK_THRESHOLD = 1000; + + protected TSDataType[] dataTypes; + protected String queryId; + protected float memoryLimitInMB; + protected int internalRowListCapacity; + protected int numCacheBlock; + + protected LRUCache cache; + protected List internalRowList; + protected List internalBlockCountList; + + protected int rowCount; + protected int lastRowCount; + protected int evictionUpperBound; + + protected boolean disableMemoryControl; + protected int[] indexListOfTextFields; + protected int byteArrayLengthForMemoryControl; + protected long rowByteArrayLength; + protected long totalByteArrayLengthLimit; + protected long totalByteArrayLength; + + protected List iteratorList; + + /** + * Construct a ElasticSerializableRowRecordList. + * + * @param dataTypes Data types of columns. + * @param queryId Query ID. + * @param memoryLimitInMB Memory limit. + * @param numCacheBlock Number of cache blocks. + * @throws QueryProcessException by SerializableRowRecordList.calculateCapacity + */ + public ElasticSerializableRowList( + TSDataType[] dataTypes, String queryId, float memoryLimitInMB, int numCacheBlock) + throws QueryProcessException { + this.dataTypes = dataTypes; + this.queryId = queryId; + this.memoryLimitInMB = memoryLimitInMB; + int allocatableCapacity = + SerializableRowList.calculateCapacity( + dataTypes, + memoryLimitInMB, + SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL); + internalRowListCapacity = allocatableCapacity / numCacheBlock; + if (internalRowListCapacity == 0) { + numCacheBlock = 1; + internalRowListCapacity = allocatableCapacity; + } + this.numCacheBlock = numCacheBlock; + + cache = new ElasticSerializableRowList.LRUCache(numCacheBlock); + internalRowList = new ArrayList<>(); + internalBlockCountList = new ArrayList<>(); + + rowCount = 0; + evictionUpperBound = 0; + + disableMemoryControl = true; + int textFieldsCount = 0; + for (TSDataType dataType : dataTypes) { + if (dataType.equals(TSDataType.TEXT)) { + ++textFieldsCount; + disableMemoryControl = false; + } + } + indexListOfTextFields = new int[textFieldsCount]; + int fieldIndex = 0; + for (int i = 0; i < dataTypes.length; ++i) { + if (dataTypes[i].equals(TSDataType.TEXT)) { + indexListOfTextFields[fieldIndex++] = i; + } + } + byteArrayLengthForMemoryControl = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL; + totalByteArrayLengthLimit = 0; + totalByteArrayLength = 0; + rowByteArrayLength = (long) byteArrayLengthForMemoryControl * textFieldsCount; + + iteratorList = new ArrayList<>(); + } + + protected ElasticSerializableRowList( + TSDataType[] dataTypes, + String queryId, + float memoryLimitInMB, + int internalRowListCapacity, + int numCacheBlock) { + this.dataTypes = dataTypes; + this.queryId = queryId; + this.memoryLimitInMB = memoryLimitInMB; + this.internalRowListCapacity = internalRowListCapacity; + this.numCacheBlock = numCacheBlock; + + cache = new ElasticSerializableRowList.LRUCache(numCacheBlock); + internalRowList = new ArrayList<>(); + internalBlockCountList = new ArrayList<>(); + + rowCount = 0; + evictionUpperBound = 0; + + disableMemoryControl = true; + + iteratorList = new ArrayList<>(); + } + + public TSDataType[] getDataTypes() { + return dataTypes; + } + + public int size() { + return rowCount; + } + + public int getInternalRowListCapacity() { + return internalRowListCapacity; + } + + public int getSerializableRowListSize() { + return internalRowList.size(); + } + + public SerializableRowList getSerializableRowList(int index) { + // Do not cache this row list + return internalRowList.get(index); + } + + // region single row methods for row window + public long getTime(int index) throws IOException { + return cache.get(index / internalRowListCapacity).getTime(index % internalRowListCapacity); + } + + public Object[] getRowRecord(int index) throws IOException { + return cache.get(index / internalRowListCapacity).getRow(index % internalRowListCapacity); + } + + // endregion + + // region batch rows methods + public Column[] getColumns(int externalIndex, int internalIndex) throws IOException { + return cache.get(externalIndex).getColumns(internalIndex); + } + + public void put(Column[] columns) throws IOException, QueryProcessException { + // Check if we need to add new internal list + checkExpansion(); + + int begin = 0, end = 0; + int total = columns[0].getPositionCount(); + while (total > 0) { + int consumed; + Column[] insertedColumns; + if (total + rowCount % internalRowListCapacity < internalRowListCapacity) { + consumed = total; + if (begin == 0) { + // No need to copy if the columns do not split + insertedColumns = columns; + } else { + insertedColumns = new Column[columns.length]; + for (int i = 0; i < columns.length; i++) { + insertedColumns[i] = columns[i].getRegionCopy(begin, consumed); + } + } + } else { + consumed = internalRowListCapacity - rowCount % internalRowListCapacity; + // Construct sub-regions + insertedColumns = new Column[columns.length]; + for (int i = 0; i < columns.length; i++) { + insertedColumns[i] = columns[i].getRegionCopy(begin, consumed); + } + } + + end += consumed; + begin = end; + total -= consumed; + + // Fill row record list + cache.get(rowCount / internalRowListCapacity).putColumns(insertedColumns); + rowCount += consumed; + if (total > 0) { + doExpansion(); + } + } + + if (!disableMemoryControl) { + int count = columns[0].getPositionCount(); + totalByteArrayLengthLimit += rowByteArrayLength * count; + + for (int i = 0; i < count; i++) { + for (int indexListOfTextField : indexListOfTextFields) { + Column bianryColumn = columns[indexListOfTextField]; + if (!bianryColumn.isNull(i)) { + totalByteArrayLength += bianryColumn.getBinary(i).getLength(); + } + } + } + + checkMemoryUsage(); + } + } + + public void putNulls(int total) throws IOException { + // Only batch insert nulls at the beginning of internal list + assert rowCount % internalRowListCapacity == 0; + + // Check if we need to add new internal list + checkExpansion(); + while (total > 0) { + int consumed = Math.min(total, internalRowListCapacity); + cache.get(rowCount / internalRowListCapacity).putNulls(consumed); + + total -= consumed; + rowCount += consumed; + if (total > 0) { + doExpansion(); + } + } + + if (!disableMemoryControl) { + totalByteArrayLengthLimit += rowByteArrayLength * total; + totalByteArrayLength += rowByteArrayLength * total; + } + } + + // endregion + + public RowListForwardIterator constructIterator() { + RowListForwardIterator iterator = new RowListForwardIterator(this); + iteratorList.add(iterator); + + return iterator; + } + + private void copyLatterColumnsAfterEvictionUpperBound(ElasticSerializableRowList newESRowList) + throws IOException, QueryProcessException { + int externalColumnIndex = evictionUpperBound / internalRowListCapacity; + + int internalRowIndex = evictionUpperBound % internalRowListCapacity; + int internalColumnIndex = + internalRowList.get(externalColumnIndex).getColumnIndex(internalRowIndex); + int rowOffsetInColumns = + internalRowList.get(externalColumnIndex).getRowOffsetInColumns(internalRowIndex); + + // This iterator is for memory control. + // So there is no need to put it into iterator list since it won't be affected by new memory + // control strategy. + RowListForwardIterator iterator = + new RowListForwardIterator(this, externalColumnIndex, internalColumnIndex); + // Get and put split columns after eviction upper bound + Column[] columns = iterator.currentBlock(); + if (rowOffsetInColumns != 0) { + for (int i = 0; i < columns.length; i++) { + columns[i] = columns[i].subColumnCopy(rowOffsetInColumns); + } + } + newESRowList.put(columns); + + // Copy latter columns + while (iterator.hasNext()) { + iterator.next(); + copyColumnByIterator(newESRowList, iterator); + } + } + + private void copyColumnByIterator( + ElasticSerializableRowList target, RowListForwardIterator source) + throws IOException, QueryProcessException { + Column[] columns = source.currentBlock(); + target.put(columns); + } + + private void checkExpansion() { + if (rowCount % internalRowListCapacity == 0) { + doExpansion(); + } + } + + private void doExpansion() { + if (internalRowList.size() > 0) { + int lastIndex = internalRowList.size() - 1; + SerializableRowList lastInternalList = internalRowList.get(lastIndex); + // For put nulls + if (lastInternalList != null) { + internalBlockCountList.add(lastInternalList.getBlockCount()); + } else { + internalBlockCountList.add(0); + } + } + internalRowList.add(SerializableRowList.construct(queryId, dataTypes)); + } + + public int getBlockCount(int index) { + if (index == internalRowList.size() - 1) { + SerializableRowList lastList = internalRowList.get(index); + return lastList.getBlockCount(); + } else { + return internalBlockCountList.get(index); + } + } + + protected void checkMemoryUsage() throws IOException, QueryProcessException { + // Insert more than MEMORY_CHECK_THRESHOLD rows + // and actual memory footprint is greater than expected + // to apply new memory control strategy + if (rowCount - lastRowCount < MEMORY_CHECK_THRESHOLD + || totalByteArrayLength <= totalByteArrayLengthLimit) { + return; + } + + // Exponential growth theoretical byte array length + lastRowCount = rowCount; + int newByteArrayLengthForMemoryControl = byteArrayLengthForMemoryControl; + while ((long) newByteArrayLengthForMemoryControl * rowCount < totalByteArrayLength) { + newByteArrayLengthForMemoryControl *= 2; + } + int newInternalTVListCapacity = + SerializableRowList.calculateCapacity( + dataTypes, memoryLimitInMB, newByteArrayLengthForMemoryControl) + / numCacheBlock; + if (newInternalTVListCapacity > 0) { + applyNewMemoryControlParameters( + newByteArrayLengthForMemoryControl, newInternalTVListCapacity); + return; + } + + // Try to find a more suitable parameters + int delta = + (int) + ((totalByteArrayLength - totalByteArrayLengthLimit) + / rowCount + / indexListOfTextFields.length + / SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL); + newByteArrayLengthForMemoryControl = + byteArrayLengthForMemoryControl + + 2 * (delta + 1) * SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL; + newInternalTVListCapacity = + SerializableRowList.calculateCapacity( + dataTypes, memoryLimitInMB, newByteArrayLengthForMemoryControl) + / numCacheBlock; + if (newInternalTVListCapacity > 0) { + applyNewMemoryControlParameters( + newByteArrayLengthForMemoryControl, newInternalTVListCapacity); + return; + } + + throw new QueryProcessException("Memory is not enough for current query."); + } + + protected void applyNewMemoryControlParameters( + int newByteArrayLengthForMemoryControl, int newInternalRowRecordListCapacity) + throws IOException, QueryProcessException { + ElasticSerializableRowList newESRowList = + new ElasticSerializableRowList( + dataTypes, queryId, memoryLimitInMB, newInternalRowRecordListCapacity, numCacheBlock); + + // Set previous evicted list to null + newESRowList.evictionUpperBound = evictionUpperBound; + int internalListEvictionUpperBound = evictionUpperBound / newInternalRowRecordListCapacity; + for (int i = 0; i < internalListEvictionUpperBound; ++i) { + newESRowList.internalRowList.add(null); + if (i != 0) { + newESRowList.internalBlockCountList.add(0); + } + } + // Put all null columns to middle list + newESRowList.rowCount = internalListEvictionUpperBound * newInternalRowRecordListCapacity; + int emptyColumnSize = evictionUpperBound - newESRowList.rowCount; + if (emptyColumnSize != 0) { + newESRowList.putNulls(emptyColumnSize); + } + // Copy latter columns + copyLatterColumnsAfterEvictionUpperBound(newESRowList); + + // Replace old list with new ones + internalRowListCapacity = newInternalRowRecordListCapacity; + cache = newESRowList.cache; + internalRowList = newESRowList.internalRowList; + internalBlockCountList = newESRowList.internalBlockCountList; + // Update metrics + byteArrayLengthForMemoryControl = newByteArrayLengthForMemoryControl; + rowByteArrayLength = (long) byteArrayLengthForMemoryControl * indexListOfTextFields.length; + totalByteArrayLengthLimit = (long) rowCount * rowByteArrayLength; + + // Notify all iterators to update + notifyAllIterators(); + } + + public void notifyAllIterators() throws IOException { + for (RowListForwardIterator iterator : iteratorList) { + iterator.adjust(); + } + } + + /** true if any field except the timestamp in the current row is null. */ + @TestOnly + public boolean fieldsHasAnyNull(int index) throws IOException { + Object[] row = + internalRowList + .get(index / internalRowListCapacity) + .getRow(index % internalRowListCapacity); + for (Object field : row) { + if (field == null) { + return true; + } + } + + return false; + } + + public int getLastRowIndex(int externalIndex, int internalIndex) { + int index = internalRowListCapacity * externalIndex; + int offset = internalRowList.get(externalIndex).getLastRowIndex(internalIndex); + + return index + offset; + } + + /** + * Set the upper bound. + * + * @param evictionUpperBound the index of the first element that cannot be evicted. in other + * words, elements whose index are less than the evictionUpperBound can be evicted. + */ + public void setEvictionUpperBound(int evictionUpperBound) { + this.evictionUpperBound = evictionUpperBound; + } + + private class LRUCache extends Cache { + + LRUCache(int capacity) { + super(capacity); + } + + SerializableRowList get(int targetIndex) throws IOException { + if (!containsKey(targetIndex)) { + if (cacheCapacity <= super.size()) { + int lastIndex = getLast(); + if (lastIndex < evictionUpperBound / internalRowListCapacity) { + internalRowList.set(lastIndex, null); + } else { + internalRowList.get(lastIndex).serialize(); + } + } + internalRowList.get(targetIndex).deserialize(); + } + putKey(targetIndex); + return internalRowList.get(targetIndex); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/ElasticSerializableRowRecordList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/ElasticSerializableRowRecordList.java deleted file mode 100644 index a5ddd5c1e2b9..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/ElasticSerializableRowRecordList.java +++ /dev/null @@ -1,314 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure.row; - -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.dag.util.InputRowUtils; -import org.apache.iotdb.db.queryengine.transformation.datastructure.Cache; -import org.apache.iotdb.db.queryengine.transformation.datastructure.SerializableList; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BitMap; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** An elastic list of records that implements memory control using LRU strategy. */ -public class ElasticSerializableRowRecordList { - - protected static final int MEMORY_CHECK_THRESHOLD = 1000; - - protected TSDataType[] dataTypes; - protected String queryId; - protected float memoryLimitInMB; - protected int internalRowRecordListCapacity; - protected int numCacheBlock; - - protected LRUCache cache; - protected List rowRecordLists; - - /** Mark bitMaps of correct index when one row has at least one null field. */ - protected List bitMaps; - - protected int size; - protected int evictionUpperBound; - - protected boolean disableMemoryControl; - protected int[] indexListOfTextFields; - protected int byteArrayLengthForMemoryControl; - protected long totalByteArrayLengthLimit; - protected long totalByteArrayLength; - - /** - * Construct a ElasticSerializableRowRecordList. - * - * @param dataTypes Data types of columns. - * @param queryId Query ID. - * @param memoryLimitInMB Memory limit. - * @param numCacheBlock Number of cache blocks. - * @throws QueryProcessException by SerializableRowRecordList.calculateCapacity - */ - public ElasticSerializableRowRecordList( - TSDataType[] dataTypes, String queryId, float memoryLimitInMB, int numCacheBlock) - throws QueryProcessException { - this.dataTypes = dataTypes; - this.queryId = queryId; - this.memoryLimitInMB = memoryLimitInMB; - int allocatableCapacity = - SerializableRowRecordList.calculateCapacity( - dataTypes, - memoryLimitInMB, - SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL); - internalRowRecordListCapacity = allocatableCapacity / numCacheBlock; - if (internalRowRecordListCapacity == 0) { - numCacheBlock = 1; - internalRowRecordListCapacity = allocatableCapacity; - } - this.numCacheBlock = numCacheBlock; - - cache = new ElasticSerializableRowRecordList.LRUCache(numCacheBlock); - rowRecordLists = new ArrayList<>(); - bitMaps = new ArrayList<>(); - size = 0; - evictionUpperBound = 0; - - disableMemoryControl = true; - int textFieldsCount = 0; - for (TSDataType dataType : dataTypes) { - if (dataType.equals(TSDataType.TEXT)) { - ++textFieldsCount; - disableMemoryControl = false; - } - } - indexListOfTextFields = new int[textFieldsCount]; - int fieldIndex = 0; - for (int i = 0; i < dataTypes.length; ++i) { - if (dataTypes[i].equals(TSDataType.TEXT)) { - indexListOfTextFields[fieldIndex++] = i; - } - } - byteArrayLengthForMemoryControl = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL; - totalByteArrayLengthLimit = 0; - totalByteArrayLength = 0; - } - - protected ElasticSerializableRowRecordList( - TSDataType[] dataTypes, - String queryId, - float memoryLimitInMB, - int internalRowRecordListCapacity, - int numCacheBlock) { - this.dataTypes = dataTypes; - this.queryId = queryId; - this.memoryLimitInMB = memoryLimitInMB; - this.internalRowRecordListCapacity = internalRowRecordListCapacity; - this.numCacheBlock = numCacheBlock; - - cache = new ElasticSerializableRowRecordList.LRUCache(numCacheBlock); - rowRecordLists = new ArrayList<>(); - bitMaps = new ArrayList<>(); - size = 0; - evictionUpperBound = 0; - - disableMemoryControl = true; - } - - public int size() { - return size; - } - - public TSDataType[] getDataTypes() { - return dataTypes; - } - - public long getTime(int index) throws IOException { - return cache - .get(index / internalRowRecordListCapacity) - .getTime(index % internalRowRecordListCapacity); - } - - public Object[] getRowRecord(int index) throws IOException { - return cache - .get(index / internalRowRecordListCapacity) - .getRowRecord(index % internalRowRecordListCapacity); - } - - /** true if any field except the timestamp in the current row is null. */ - public boolean fieldsHasAnyNull(int index) { - return bitMaps - .get(index / internalRowRecordListCapacity) - .isMarked(index % internalRowRecordListCapacity); - } - - public void put(Object[] rowRecord) throws IOException, QueryProcessException { - put(rowRecord, InputRowUtils.hasNullField(rowRecord)); - } - - /** - * Put the row in the list with an any-field-null marker, this method is faster than calling put - * directly. - * - * @throws IOException by checkMemoryUsage() - * @throws QueryProcessException by checkMemoryUsage() - */ - private void put(Object[] rowRecord, boolean hasNullField) - throws IOException, QueryProcessException { - checkExpansion(); - cache.get(size / internalRowRecordListCapacity).put(rowRecord); - if (hasNullField) { - bitMaps.get(size / internalRowRecordListCapacity).mark(size % internalRowRecordListCapacity); - } - ++size; - - if (!disableMemoryControl) { - totalByteArrayLengthLimit += - (long) indexListOfTextFields.length * byteArrayLengthForMemoryControl; - - if (rowRecord == null) { - totalByteArrayLength += - (long) indexListOfTextFields.length * byteArrayLengthForMemoryControl; - } else { - for (int indexListOfTextField : indexListOfTextFields) { - Binary binary = (Binary) rowRecord[indexListOfTextField]; - totalByteArrayLength += binary == null ? 0 : binary.getLength(); - } - checkMemoryUsage(); - } - } - } - - private void checkExpansion() { - if (size % internalRowRecordListCapacity == 0) { - rowRecordLists.add( - SerializableRowRecordList.newSerializableRowRecordList( - queryId, dataTypes, internalRowRecordListCapacity)); - bitMaps.add(new BitMap(internalRowRecordListCapacity)); - } - } - - protected void checkMemoryUsage() throws IOException, QueryProcessException { - if (size % MEMORY_CHECK_THRESHOLD != 0 || totalByteArrayLength <= totalByteArrayLengthLimit) { - return; - } - - int newByteArrayLengthForMemoryControl = byteArrayLengthForMemoryControl; - while ((long) newByteArrayLengthForMemoryControl * size < totalByteArrayLength) { - newByteArrayLengthForMemoryControl *= 2; - } - int newInternalTVListCapacity = - SerializableRowRecordList.calculateCapacity( - dataTypes, memoryLimitInMB, newByteArrayLengthForMemoryControl) - / numCacheBlock; - if (0 < newInternalTVListCapacity) { - applyNewMemoryControlParameters( - newByteArrayLengthForMemoryControl, newInternalTVListCapacity); - return; - } - - int delta = - (int) - ((totalByteArrayLength - totalByteArrayLengthLimit) - / size - / indexListOfTextFields.length - / SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL); - newByteArrayLengthForMemoryControl = - byteArrayLengthForMemoryControl - + 2 * (delta + 1) * SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL; - newInternalTVListCapacity = - SerializableRowRecordList.calculateCapacity( - dataTypes, memoryLimitInMB, newByteArrayLengthForMemoryControl) - / numCacheBlock; - if (0 < newInternalTVListCapacity) { - applyNewMemoryControlParameters( - newByteArrayLengthForMemoryControl, newInternalTVListCapacity); - return; - } - - throw new QueryProcessException("Memory is not enough for current query."); - } - - protected void applyNewMemoryControlParameters( - int newByteArrayLengthForMemoryControl, int newInternalRowRecordListCapacity) - throws IOException, QueryProcessException { - ElasticSerializableRowRecordList newElasticSerializableRowRecordList = - new ElasticSerializableRowRecordList( - dataTypes, queryId, memoryLimitInMB, newInternalRowRecordListCapacity, numCacheBlock); - - newElasticSerializableRowRecordList.evictionUpperBound = evictionUpperBound; - int internalListEvictionUpperBound = evictionUpperBound / newInternalRowRecordListCapacity; - for (int i = 0; i < internalListEvictionUpperBound; ++i) { - newElasticSerializableRowRecordList.rowRecordLists.add(null); - newElasticSerializableRowRecordList.bitMaps.add(null); - } - newElasticSerializableRowRecordList.size = - internalListEvictionUpperBound * newInternalRowRecordListCapacity; - for (int i = newElasticSerializableRowRecordList.size; i < evictionUpperBound; ++i) { - newElasticSerializableRowRecordList.put(null); - } - for (int i = evictionUpperBound; i < size; ++i) { - newElasticSerializableRowRecordList.put(getRowRecord(i), fieldsHasAnyNull(i)); - } - - internalRowRecordListCapacity = newInternalRowRecordListCapacity; - cache = newElasticSerializableRowRecordList.cache; - rowRecordLists = newElasticSerializableRowRecordList.rowRecordLists; - bitMaps = newElasticSerializableRowRecordList.bitMaps; - - byteArrayLengthForMemoryControl = newByteArrayLengthForMemoryControl; - totalByteArrayLengthLimit = - (long) size * indexListOfTextFields.length * byteArrayLengthForMemoryControl; - } - - /** - * Set the upper bound. - * - * @param evictionUpperBound the index of the first element that cannot be evicted. in other - * words, elements whose index are less than the evictionUpperBound can be evicted. - */ - public void setEvictionUpperBound(int evictionUpperBound) { - this.evictionUpperBound = evictionUpperBound; - } - - private class LRUCache extends Cache { - - LRUCache(int capacity) { - super(capacity); - } - - SerializableRowRecordList get(int targetIndex) throws IOException { - if (!containsKey(targetIndex)) { - if (cacheCapacity <= super.size()) { - int lastIndex = getLast(); - if (lastIndex < evictionUpperBound / internalRowRecordListCapacity) { - rowRecordLists.set(lastIndex, null); - bitMaps.set(lastIndex, null); - } else { - rowRecordLists.get(lastIndex).serialize(); - } - } - rowRecordLists.get(targetIndex).deserialize(); - } - putKey(targetIndex); - return rowRecordLists.get(targetIndex); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java new file mode 100644 index 000000000000..7772894a86de --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java @@ -0,0 +1,373 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.datastructure.row; + +import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.transformation.datastructure.SerializableList; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.block.column.TsBlockSerde; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; +import static org.apache.iotdb.db.queryengine.transformation.datastructure.util.BinaryUtils.MIN_ARRAY_HEADER_SIZE; +import static org.apache.iotdb.db.queryengine.transformation.datastructure.util.BinaryUtils.MIN_OBJECT_HEADER_SIZE; + +public class SerializableRowList implements SerializableList { + private final SerializationRecorder serializationRecorder; + private final TsBlockSerde serde; + private final TSDataType[] dataTypes; + private final int valueColumnCount; + + private List blocks; + private final List blockSizes; + + private int skipPrefixNullCount; + + private int prefixNullCount; + + public static SerializableRowList construct(String queryId, TSDataType[] dataTypes) { + SerializationRecorder recorder = new SerializationRecorder(queryId); + return new SerializableRowList(recorder, dataTypes); + } + + private SerializableRowList(SerializationRecorder serializationRecorder, TSDataType[] dataTypes) { + this.serializationRecorder = serializationRecorder; + this.dataTypes = dataTypes; + serde = new TsBlockSerde(); + blockSizes = new ArrayList<>(); + + valueColumnCount = dataTypes.length; + prefixNullCount = 0; + skipPrefixNullCount = 0; + + init(); + } + + /** + * Calculate the number of rows that can be cached given the memory limit. + * + * @param dataTypes Data types of columns. + * @param memoryLimitInMB Memory limit. + * @param byteArrayLengthForMemoryControl Max memory usage for a {@link TSDataType#TEXT}. + * @return Number of rows that can be cached. + * @throws QueryProcessException if the result capacity <= 0 + * @throws UnSupportedDataTypeException if the input datatype can not be handled by the given + * branches. + */ + protected static int calculateCapacity( + TSDataType[] dataTypes, float memoryLimitInMB, int byteArrayLengthForMemoryControl) + throws QueryProcessException { + int rowLength = ReadWriteIOUtils.LONG_LEN; // timestamp + for (TSDataType dataType : dataTypes) { // fields + switch (dataType) { + case INT32: + case DATE: + rowLength += ReadWriteIOUtils.INT_LEN; + break; + case INT64: + case TIMESTAMP: + rowLength += ReadWriteIOUtils.LONG_LEN; + break; + case FLOAT: + rowLength += ReadWriteIOUtils.FLOAT_LEN; + break; + case DOUBLE: + rowLength += ReadWriteIOUtils.DOUBLE_LEN; + break; + case BOOLEAN: + rowLength += ReadWriteIOUtils.BOOLEAN_LEN; + break; + case TEXT: + case BLOB: + case STRING: + rowLength += + MIN_OBJECT_HEADER_SIZE + MIN_ARRAY_HEADER_SIZE + byteArrayLengthForMemoryControl; + break; + default: + throw new UnSupportedDataTypeException(dataType.toString()); + } + } + rowLength += ReadWriteIOUtils.BIT_LEN; // null field + + int capacity = (int) (memoryLimitInMB * MB / 2 / rowLength); + if (capacity <= 0) { + throw new QueryProcessException("Memory is not enough for current query."); + } + return capacity; + } + + public int size() { + return prefixNullCount + skipPrefixNullCount; + } + + public int getBlockCount() { + // Threat prefix null values as one column + int additional_null_block = prefixNullCount == 0 ? 0 : 1; + return blockSizes.size() + additional_null_block; + } + + public int getBlockSize(int index) { + if (prefixNullCount != 0) { + return index == 0 ? prefixNullCount : blockSizes.get(index - 1); + } else { + return blockSizes.get(index); + } + } + + // region single data methods + public long getTime(int index) { + // Never access null row's time + assert index >= prefixNullCount; + + return getTimeSkipPrefixNulls(index - prefixNullCount); + } + + private long getTimeSkipPrefixNulls(int index) { + assert index < skipPrefixNullCount; + + int total = 0; + long time = -1; + for (Column[] block : blocks) { + int length = block[0].getPositionCount(); + if (index < total + length) { + int offset = index - total; + + // Last column is always time column + time = block[valueColumnCount].getLong(offset); + break; + } + total += length; + } + + return time; + } + + public Object[] getRow(int index) { + // Fall into prefix null values + if (index < prefixNullCount) { + return null; + } + + return getRowSkipPrefixNulls(index - prefixNullCount); + } + + private Object[] getRowSkipPrefixNulls(int index) { + assert index < skipPrefixNullCount; + + // Value columns + time column + Object[] row = new Object[valueColumnCount + 1]; + + int total = 0; + for (Column[] block : blocks) { + // Find position + int length = block[0].getPositionCount(); + if (index < total + length) { + int offset = index - total; + + // Fill value columns + for (int i = 0; i < block.length - 1; i++) { + switch (dataTypes[i]) { + case INT32: + case DATE: + row[i] = block[i].getInt(offset); + break; + case INT64: + case TIMESTAMP: + row[i] = block[i].getLong(offset); + break; + case FLOAT: + row[i] = block[i].getFloat(offset); + break; + case DOUBLE: + row[i] = block[i].getDouble(offset); + break; + case BOOLEAN: + row[i] = block[i].getBoolean(offset); + break; + case TEXT: + case BLOB: + case STRING: + row[i] = block[i].getBinary(offset); + break; + default: + throw new UnSupportedDataTypeException(dataTypes[i].toString()); + } + } + // Fill time column + row[block.length - 1] = block[block.length - 1].getLong(offset); + + break; + } + total += length; + } + + return row; + } + + // endregion + + // region batch data methods + public void putColumns(Column[] columns) { + blocks.add(columns); + + int size = columns[0].getPositionCount(); + blockSizes.add(size); + skipPrefixNullCount += size; + } + + public void putNulls(int count) { + prefixNullCount += count; + } + + public Column[] getColumns(int index) { + // Skip all null columns at first + if (prefixNullCount != 0) { + index--; + } + // Forbid to fetch first null column + assert index >= 0; + return blocks.get(index); + } + + // endregion + + public int getColumnIndex(int index) { + // Fall into first null column + if (index < prefixNullCount) { + return 0; + } + int addition = prefixNullCount == 0 ? 0 : 1; + return addition + getColumnIndexSkipPrefixNulls(index - prefixNullCount); + } + + private int getColumnIndexSkipPrefixNulls(int index) { + assert index < skipPrefixNullCount; + + int ret = -1; + int total = 0; + for (int i = 0; i < blockSizes.size(); i++) { + int length = blockSizes.get(i); + if (index < total + length) { + ret = i; + break; + } + total += length; + } + + return ret; + } + + public int getRowOffsetInColumns(int index) { + assert index >= prefixNullCount; + + return getRowOffsetInColumnsSkipPrefixNulls(index - prefixNullCount); + } + + private int getRowOffsetInColumnsSkipPrefixNulls(int index) { + assert index < skipPrefixNullCount; + + int ret = -1; + int total = 0; + for (int length : blockSizes) { + if (index < total + length) { + ret = index - total; + break; + } + total += length; + } + + return ret; + } + + public int getLastRowIndex(int blockIndex) { + int total = prefixNullCount; + for (int i = 0; i <= blockIndex; i++) { + total += blockSizes.get(i); + } + + return total; + } + + @Override + public void release() { + blocks = null; + } + + @Override + public void init() { + blocks = new ArrayList<>(); + } + + // Only blocks data are serialized to disk + // Other field are kept in memory + @Override + public void serialize(PublicBAOS outputStream) throws IOException { + int bufferSize = 0; + + // Write TsBlocks count + bufferSize += ReadWriteIOUtils.write(blocks.size(), outputStream); + + for (Column[] block : blocks) { + TimeColumn timeColumn = (TimeColumn) block[block.length - 1]; + Column[] valueColumns = new Column[block.length - 1]; + // Only references are copied + System.arraycopy(block, 0, valueColumns, 0, block.length - 1); + TsBlock tsBlock = new TsBlock(timeColumn, valueColumns); + + ByteBuffer buffer = serde.serialize(tsBlock); + byte[] byteArray = buffer.array(); + // Write TsBlocks data + outputStream.write(byteArray); + bufferSize += byteArray.length; + } + + serializationRecorder.setSerializedByteLength(bufferSize); + } + + // Deserialized blocks from disk to memory + @Override + public void deserialize(ByteBuffer byteBuffer) { + // Read TsBlocks count + int blockCount = ReadWriteIOUtils.readInt(byteBuffer); + blocks = new ArrayList<>(blockCount); + + for (int i = 0; i < blockCount; i++) { + // Read TsBlocks data + TsBlock tsBlock = serde.deserialize(byteBuffer); + blocks.add(tsBlock.getAllColumns()); + } + } + + @Override + public SerializationRecorder getSerializationRecorder() { + return serializationRecorder; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowRecordList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowRecordList.java deleted file mode 100644 index 45b5b8ec4276..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowRecordList.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure.row; - -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.datastructure.SerializableList; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.PublicBAOS; -import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; - -public class SerializableRowRecordList implements SerializableList { - - protected static final int MIN_OBJECT_HEADER_SIZE = 8; - protected static final int MIN_ARRAY_HEADER_SIZE = MIN_OBJECT_HEADER_SIZE + 4; - - public static SerializableRowRecordList newSerializableRowRecordList( - String queryId, TSDataType[] dataTypes, int internalRowRecordListCapacity) { - SerializationRecorder recorder = new SerializationRecorder(queryId); - return new SerializableRowRecordList(recorder, dataTypes, internalRowRecordListCapacity); - } - - /** - * Calculate the number of rows that can be cached given the memory limit. - * - * @param dataTypes Data types of columns. - * @param memoryLimitInMB Memory limit. - * @param byteArrayLengthForMemoryControl Max memory usage for a {@link TSDataType#TEXT}. - * @return Number of rows that can be cached. - * @throws QueryProcessException if the result capacity <= 0 - * @throws UnSupportedDataTypeException if the input datatype can not be handled by the given - * branches. - */ - protected static int calculateCapacity( - TSDataType[] dataTypes, float memoryLimitInMB, int byteArrayLengthForMemoryControl) - throws QueryProcessException { - int rowLength = ReadWriteIOUtils.LONG_LEN; // timestamp - for (TSDataType dataType : dataTypes) { // fields - switch (dataType) { - case INT32: - case DATE: - rowLength += ReadWriteIOUtils.INT_LEN; - break; - case INT64: - case TIMESTAMP: - rowLength += ReadWriteIOUtils.LONG_LEN; - break; - case FLOAT: - rowLength += ReadWriteIOUtils.FLOAT_LEN; - break; - case DOUBLE: - rowLength += ReadWriteIOUtils.DOUBLE_LEN; - break; - case BOOLEAN: - rowLength += ReadWriteIOUtils.BOOLEAN_LEN; - break; - case TEXT: - case STRING: - case BLOB: - rowLength += - MIN_OBJECT_HEADER_SIZE + MIN_ARRAY_HEADER_SIZE + byteArrayLengthForMemoryControl; - break; - default: - throw new UnSupportedDataTypeException(dataType.toString()); - } - } - - // 1 extra bit for null fields mark in bitMap - int size = (int) (memoryLimitInMB * MB / 2 / (rowLength + ReadWriteIOUtils.BIT_LEN)); - if (size <= 0) { - throw new QueryProcessException("Memory is not enough for current query."); - } - return size; - } - - private final SerializationRecorder serializationRecorder; - private final TSDataType[] dataTypes; - private final int internalRowRecordListCapacity; - private final int seriesNumber; - - private List rowRecords; - - private SerializableRowRecordList( - SerializationRecorder serializationRecorder, - TSDataType[] dataTypes, - int internalRowRecordListCapacity) { - this.serializationRecorder = serializationRecorder; - this.dataTypes = dataTypes; - this.internalRowRecordListCapacity = internalRowRecordListCapacity; - seriesNumber = dataTypes.length; - init(); - } - - public int size() { - return rowRecords.size(); - } - - public Object[] getRowRecord(int index) { - return rowRecords.get(index); - } - - public long getTime(int index) { - return (long) rowRecords.get(index)[seriesNumber]; - } - - public void put(Object[] rowRecord) { - rowRecords.add(rowRecord); - } - - @Override - public void release() { - rowRecords = null; - } - - @Override - public void init() { - rowRecords = new ArrayList<>(internalRowRecordListCapacity); - } - - @Override - public void serialize(PublicBAOS outputStream) throws IOException { - int size = rowRecords.size(); - serializationRecorder.setSerializedElementSize(size); - int serializedByteLength = 0; - int nullCount = 0; - for (Object[] record : rowRecords) { - if (record != null) { - break; - } - ++nullCount; - } - serializedByteLength += ReadWriteIOUtils.write(nullCount, outputStream); - for (int i = nullCount; i < size; ++i) { - Object[] rowRecord = rowRecords.get(i); - serializedByteLength += ReadWriteIOUtils.write((long) rowRecord[seriesNumber], outputStream); - serializedByteLength += writeFields(rowRecord, outputStream); - } - serializationRecorder.setSerializedByteLength(serializedByteLength); - } - - @Override - public void deserialize(ByteBuffer byteBuffer) { - int serializedElementSize = serializationRecorder.getSerializedElementSize(); - int nullCount = ReadWriteIOUtils.readInt(byteBuffer); - for (int i = 0; i < nullCount; ++i) { - put(null); - } - for (int i = nullCount; i < serializedElementSize; ++i) { - Object[] rowRecord = new Object[seriesNumber + 1]; - rowRecord[seriesNumber] = ReadWriteIOUtils.readLong(byteBuffer); // timestamp - readFields(byteBuffer, rowRecord); - put(rowRecord); - } - } - - private int writeFields(Object[] rowRecord, PublicBAOS outputStream) throws IOException { - int serializedByteLength = 0; - for (int i = 0; i < seriesNumber; ++i) { - Object field = rowRecord[i]; - boolean isNull = field == null; - serializedByteLength += ReadWriteIOUtils.write(isNull, outputStream); - if (isNull) { - continue; - } - - switch (dataTypes[i]) { - case INT32: - serializedByteLength += ReadWriteIOUtils.write((int) field, outputStream); - break; - case INT64: - serializedByteLength += ReadWriteIOUtils.write((long) field, outputStream); - break; - case FLOAT: - serializedByteLength += ReadWriteIOUtils.write((float) field, outputStream); - break; - case DOUBLE: - serializedByteLength += ReadWriteIOUtils.write((double) field, outputStream); - break; - case BOOLEAN: - serializedByteLength += ReadWriteIOUtils.write((boolean) field, outputStream); - break; - case TEXT: - serializedByteLength += ReadWriteIOUtils.write((Binary) field, outputStream); - break; - default: - throw new UnSupportedDataTypeException(dataTypes[i].toString()); - } - } - return serializedByteLength; - } - - private void readFields(ByteBuffer byteBuffer, Object[] rowRecord) { - for (int i = 0; i < seriesNumber; ++i) { - boolean isNull = ReadWriteIOUtils.readBool(byteBuffer); - if (isNull) { - continue; - } - - switch (dataTypes[i]) { - case INT32: - rowRecord[i] = ReadWriteIOUtils.readInt(byteBuffer); - break; - case INT64: - rowRecord[i] = ReadWriteIOUtils.readLong(byteBuffer); - break; - case FLOAT: - rowRecord[i] = ReadWriteIOUtils.readFloat(byteBuffer); - break; - case DOUBLE: - rowRecord[i] = ReadWriteIOUtils.readDouble(byteBuffer); - break; - case BOOLEAN: - rowRecord[i] = ReadWriteIOUtils.readBool(byteBuffer); - break; - case TEXT: - rowRecord[i] = ReadWriteIOUtils.readBinary(byteBuffer); - break; - default: - throw new UnSupportedDataTypeException(dataTypes[i].toString()); - } - } - } - - @Override - public SerializationRecorder getSerializationRecorder() { - return serializationRecorder; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/ElasticSerializableBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/ElasticSerializableBinaryTVList.java index 50ad8823df52..fd6d111ef45a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/ElasticSerializableBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/ElasticSerializableBinaryTVList.java @@ -19,17 +19,22 @@ package org.apache.iotdb.db.queryengine.transformation.datastructure.tv; -import org.apache.iotdb.commons.udf.utils.UDFBinaryTransformer; import org.apache.iotdb.db.queryengine.transformation.datastructure.SerializableList; -import org.apache.iotdb.udf.api.type.Binary; +import org.apache.iotdb.db.queryengine.transformation.datastructure.iterator.TVListForwardIterator; +import org.apache.iotdb.db.queryengine.transformation.datastructure.util.BinaryUtils; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import java.io.IOException; public class ElasticSerializableBinaryTVList extends ElasticSerializableTVList { - protected static final int MEMORY_CHECK_THRESHOLD = 1000; protected int byteArrayLengthForMemoryControl; @@ -37,6 +42,8 @@ public class ElasticSerializableBinaryTVList extends ElasticSerializableTVList { protected long totalByteArrayLengthLimit; protected long totalByteArrayLength; + protected int lastPointCount; + public ElasticSerializableBinaryTVList(String queryId, float memoryLimitInMB, int cacheSize) { super(TSDataType.TEXT, queryId, memoryLimitInMB, cacheSize); byteArrayLengthForMemoryControl = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL; @@ -45,54 +52,60 @@ public ElasticSerializableBinaryTVList(String queryId, float memoryLimitInMB, in } @Override - public void putBinary(long timestamp, Binary value) throws IOException { - super.putBinary(timestamp, value); - totalByteArrayLengthLimit += byteArrayLengthForMemoryControl; - totalByteArrayLength += value.getLength(); - checkMemoryUsage(); - } + public void putColumn(TimeColumn timeColumn, Column valueColumn) throws IOException { + super.putColumn(timeColumn, valueColumn); + + // Update two memory control statistics + long count = timeColumn.getPositionCount(); + totalByteArrayLengthLimit += byteArrayLengthForMemoryControl * count; + for (int i = 0; i < count; i++) { + if (valueColumn.isNull(i)) { + totalByteArrayLength += byteArrayLengthForMemoryControl; + } else { + totalByteArrayLength += valueColumn.getBinary(i).getLength(); + } + } - @Override - public void putString(long timestamp, String value) throws IOException { - org.apache.tsfile.utils.Binary binary = BytesUtils.valueOf(value); - super.putBinary(timestamp, UDFBinaryTransformer.transformToUDFBinary(binary)); - totalByteArrayLengthLimit += byteArrayLengthForMemoryControl; - totalByteArrayLength += binary.getLength(); checkMemoryUsage(); } - protected void checkMemoryUsage() throws IOException { - if (size % MEMORY_CHECK_THRESHOLD != 0 || totalByteArrayLength <= totalByteArrayLengthLimit) { + private void checkMemoryUsage() throws IOException { + // Insert more than MEMORY_CHECK_THRESHOLD points + // and actual memory footprint is greater than expected + // to apply new memory control strategy + if (pointCount - lastPointCount < MEMORY_CHECK_THRESHOLD + || totalByteArrayLength <= totalByteArrayLengthLimit) { return; } + // Exponential growth theoretical byte array length + lastPointCount = pointCount; int newByteArrayLengthForMemoryControl = byteArrayLengthForMemoryControl; - while ((long) newByteArrayLengthForMemoryControl * size < totalByteArrayLength) { + while ((long) newByteArrayLengthForMemoryControl * pointCount < totalByteArrayLength) { newByteArrayLengthForMemoryControl *= 2; } int newInternalTVListCapacity = - SerializableBinaryTVList.calculateCapacity( - memoryLimitInMB, newByteArrayLengthForMemoryControl) + BinaryUtils.calculateCapacity(memoryLimitInMB, newByteArrayLengthForMemoryControl) / cacheSize; - if (0 < newInternalTVListCapacity) { + if (newInternalTVListCapacity > 0) { applyNewMemoryControlParameters( newByteArrayLengthForMemoryControl, newInternalTVListCapacity); return; } + // Try to find a more suitable parameters int delta = (int) ((totalByteArrayLength - totalByteArrayLengthLimit) - / size + / pointCount / SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL); newByteArrayLengthForMemoryControl = byteArrayLengthForMemoryControl + 2 * (delta + 1) * SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL; newInternalTVListCapacity = - SerializableBinaryTVList.calculateCapacity( - memoryLimitInMB, newByteArrayLengthForMemoryControl) + BinaryUtils.calculateCapacity(memoryLimitInMB, newByteArrayLengthForMemoryControl) / cacheSize; - if (0 < newInternalTVListCapacity) { + if (newInternalTVListCapacity > 0) { applyNewMemoryControlParameters( newByteArrayLengthForMemoryControl, newInternalTVListCapacity); return; @@ -101,38 +114,93 @@ protected void checkMemoryUsage() throws IOException { throw new RuntimeException("Memory is not enough for current query."); } - protected void applyNewMemoryControlParameters( + private void applyNewMemoryControlParameters( int newByteArrayLengthForMemoryControl, int newInternalTVListCapacity) throws IOException { - ElasticSerializableTVList newElasticSerializableTVList = + ElasticSerializableTVList newESTVList = new ElasticSerializableTVList( TSDataType.TEXT, queryId, memoryLimitInMB, newInternalTVListCapacity, cacheSize); - newElasticSerializableTVList.evictionUpperBound = evictionUpperBound; + // Set previous evicted list to null + newESTVList.evictionUpperBound = evictionUpperBound; int internalListEvictionUpperBound = evictionUpperBound / newInternalTVListCapacity; for (int i = 0; i < internalListEvictionUpperBound; ++i) { - newElasticSerializableTVList.tvLists.add(null); - newElasticSerializableTVList.bitMaps.add(null); - } - newElasticSerializableTVList.size = internalListEvictionUpperBound * newInternalTVListCapacity; - org.apache.tsfile.utils.Binary empty = BytesUtils.valueOf(""); - for (int i = newElasticSerializableTVList.size; i < evictionUpperBound; ++i) { - newElasticSerializableTVList.putBinary(i, UDFBinaryTransformer.transformToUDFBinary(empty)); + newESTVList.internalTVList.add(null); + if (i != 0) { + newESTVList.internalColumnCountList.add(0); + } } - for (int i = evictionUpperBound; i < size; ++i) { - if (isNull(i)) { - newElasticSerializableTVList.putNull(getTime(i)); - } else { - newElasticSerializableTVList.putBinary( - getTime(i), UDFBinaryTransformer.transformToUDFBinary(getBinary(i))); + // Put all null columns to middle list + newESTVList.pointCount = internalListEvictionUpperBound * newInternalTVListCapacity; + int emptyColumnSize = evictionUpperBound - newESTVList.pointCount; + if (emptyColumnSize != 0) { + Binary empty = BytesUtils.valueOf(""); + TimeColumnBuilder timeColumnBuilder = new TimeColumnBuilder(null, emptyColumnSize); + ColumnBuilder valueColumnBuilder = new BinaryColumnBuilder(null, emptyColumnSize); + for (int i = 0; i < emptyColumnSize; i++) { + timeColumnBuilder.writeLong(i); + valueColumnBuilder.writeBinary(empty); } + TimeColumn timeColumn = (TimeColumn) timeColumnBuilder.build(); + Column valueColumn = valueColumnBuilder.build(); + newESTVList.putColumn(timeColumn, valueColumn); } + // Copy latter lists + copyLatterColumnsAfterEvictionUpperBound(newESTVList); + // Assign new tvList to the old internalTVListCapacity = newInternalTVListCapacity; - cache = newElasticSerializableTVList.cache; - tvLists = newElasticSerializableTVList.tvLists; - bitMaps = newElasticSerializableTVList.bitMaps; - + cache = newESTVList.cache; + internalTVList = newESTVList.internalTVList; + internalColumnCountList = newESTVList.internalColumnCountList; + // Update metrics byteArrayLengthForMemoryControl = newByteArrayLengthForMemoryControl; - totalByteArrayLengthLimit = (long) size * byteArrayLengthForMemoryControl; + totalByteArrayLengthLimit = (long) pointCount * byteArrayLengthForMemoryControl; + + // Notify all iterators to update + notifyAllIterators(); + } + + public void copyLatterColumnsAfterEvictionUpperBound(ElasticSerializableTVList newESTVList) + throws IOException { + int externalColumnIndex = evictionUpperBound / internalTVListCapacity; + + int internalRowIndex = evictionUpperBound % internalTVListCapacity; + int internalColumnIndex = + internalTVList.get(externalColumnIndex).getColumnIndex(internalRowIndex); + int tvOffsetInColumns = + internalTVList.get(externalColumnIndex).getTVOffsetInColumns(internalRowIndex); + + // This iterator is for memory control. + // So there is no need to put it into iterator list since it won't be affected by new memory + // control strategy. + TVListForwardIterator iterator = + new TVListForwardIterator(this, externalColumnIndex, internalColumnIndex); + // Get and put split columns after eviction upper bound + TimeColumn timeColumn = iterator.currentTimes(); + Column valueColumn = iterator.currentValues(); + if (tvOffsetInColumns != 0) { + timeColumn = (TimeColumn) timeColumn.subColumnCopy(tvOffsetInColumns); + valueColumn = valueColumn.subColumnCopy(tvOffsetInColumns); + } + newESTVList.putColumn(timeColumn, valueColumn); + + // Copy latter columns + while (iterator.hasNext()) { + iterator.next(); + copyColumnByIterator(newESTVList, iterator); + } + } + + public void notifyAllIterators() throws IOException { + for (TVListForwardIterator iterator : iteratorList) { + iterator.adjust(); + } + } + + private void copyColumnByIterator(ElasticSerializableTVList target, TVListForwardIterator source) + throws IOException { + TimeColumn timeColumn = source.currentTimes(); + Column valueColumn = source.currentValues(); + target.putColumn(timeColumn, valueColumn); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/ElasticSerializableTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/ElasticSerializableTVList.java index f1306e01d805..324aa38fee3c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/ElasticSerializableTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/ElasticSerializableTVList.java @@ -19,33 +19,20 @@ package org.apache.iotdb.db.queryengine.transformation.datastructure.tv; -import org.apache.iotdb.commons.udf.utils.UDFBinaryTransformer; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; -import org.apache.iotdb.db.queryengine.transformation.api.YieldableState; import org.apache.iotdb.db.queryengine.transformation.datastructure.Cache; -import org.apache.iotdb.udf.api.collector.PointCollector; -import org.apache.iotdb.udf.api.type.Binary; +import org.apache.iotdb.db.queryengine.transformation.datastructure.iterator.TVListForwardIterator; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.BatchData; -import org.apache.tsfile.utils.BitMap; -import org.apache.tsfile.utils.BytesUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.utils.Binary; import java.io.IOException; import java.util.ArrayList; import java.util.List; -public class ElasticSerializableTVList implements PointCollector { - - public static ElasticSerializableTVList newElasticSerializableTVList( - TSDataType dataType, String queryId, float memoryLimitInMB, int cacheSize) { - return dataType.equals(TSDataType.TEXT) - ? new ElasticSerializableBinaryTVList(queryId, memoryLimitInMB, cacheSize) - : new ElasticSerializableTVList(dataType, queryId, memoryLimitInMB, cacheSize); - } - +public class ElasticSerializableTVList { protected TSDataType dataType; protected String queryId; protected float memoryLimitInMB; @@ -53,17 +40,22 @@ public static ElasticSerializableTVList newElasticSerializableTVList( protected int cacheSize; protected LRUCache cache; - protected List tvLists; + protected List internalTVList; + protected List internalColumnCountList; - /** - * the bitmap used to indicate whether one value is null in the tvLists. The size of bitMap is the - * same as tvLists and the length of whole bits is the same as tvLists' length. - */ - protected List bitMaps; - - protected int size; + protected int pointCount; protected int evictionUpperBound; + // Observer pattern + protected List iteratorList; + + public static ElasticSerializableTVList construct( + TSDataType dataType, String queryId, float memoryLimitInMB, int cacheSize) { + return dataType.equals(TSDataType.TEXT) + ? new ElasticSerializableBinaryTVList(queryId, memoryLimitInMB, cacheSize) + : new ElasticSerializableTVList(dataType, queryId, memoryLimitInMB, cacheSize); + } + protected ElasticSerializableTVList( TSDataType dataType, String queryId, float memoryLimitInMB, int cacheSize) { this.dataType = dataType; @@ -78,10 +70,12 @@ protected ElasticSerializableTVList( this.cacheSize = cacheSize; cache = new LRUCache(cacheSize); - bitMaps = new ArrayList<>(); - tvLists = new ArrayList<>(); - size = 0; + internalTVList = new ArrayList<>(); + internalColumnCountList = new ArrayList<>(); + pointCount = 0; evictionUpperBound = 0; + + iteratorList = new ArrayList<>(); } protected ElasticSerializableTVList( @@ -97,10 +91,12 @@ protected ElasticSerializableTVList( this.cacheSize = cacheSize; cache = new LRUCache(cacheSize); - bitMaps = new ArrayList<>(); - tvLists = new ArrayList<>(); - size = 0; + internalTVList = new ArrayList<>(); + internalColumnCountList = new ArrayList<>(); + pointCount = 0; evictionUpperBound = 0; + + iteratorList = new ArrayList<>(); } public TSDataType getDataType() { @@ -108,246 +104,137 @@ public TSDataType getDataType() { } public int size() { - return size; + return pointCount; + } + + public int getInternalTVListCapacity() { + return internalTVListCapacity; + } + + public SerializableTVList getSerializableTVList(int index) { + // Do not cache this TVList + return internalTVList.get(index); + } + + public int getSerializableTVListSize() { + return internalTVList.size(); } - public boolean isNull(int index) { - return bitMaps.get(index / internalTVListCapacity).isMarked(index % internalTVListCapacity); + // region single data point methods for row window + public boolean isNull(int index) throws IOException { + return cache.get(index / internalTVListCapacity).isNull(index % internalTVListCapacity); } public long getTime(int index) throws IOException { - return cache.get(index / internalTVListCapacity).getTimeByIndex(index % internalTVListCapacity); + return cache.get(index / internalTVListCapacity).getTime(index % internalTVListCapacity); } public int getInt(int index) throws IOException { - return cache.get(index / internalTVListCapacity).getIntByIndex(index % internalTVListCapacity); + return cache.get(index / internalTVListCapacity).getInt(index % internalTVListCapacity); } public long getLong(int index) throws IOException { - return cache.get(index / internalTVListCapacity).getLongByIndex(index % internalTVListCapacity); + return cache.get(index / internalTVListCapacity).getLong(index % internalTVListCapacity); } public float getFloat(int index) throws IOException { - return cache - .get(index / internalTVListCapacity) - .getFloatByIndex(index % internalTVListCapacity); + return cache.get(index / internalTVListCapacity).getFloat(index % internalTVListCapacity); } public double getDouble(int index) throws IOException { - return cache - .get(index / internalTVListCapacity) - .getDoubleByIndex(index % internalTVListCapacity); + return cache.get(index / internalTVListCapacity).getDouble(index % internalTVListCapacity); } public boolean getBoolean(int index) throws IOException { - return cache - .get(index / internalTVListCapacity) - .getBooleanByIndex(index % internalTVListCapacity); + return cache.get(index / internalTVListCapacity).getBoolean(index % internalTVListCapacity); } - public org.apache.tsfile.utils.Binary getBinary(int index) throws IOException { - return cache - .get(index / internalTVListCapacity) - .getBinaryByIndex(index % internalTVListCapacity); + public Binary getBinary(int index) throws IOException { + return cache.get(index / internalTVListCapacity).getBinary(index % internalTVListCapacity); } public String getString(int index) throws IOException { return cache .get(index / internalTVListCapacity) - .getBinaryByIndex(index % internalTVListCapacity) + .getBinary(index % internalTVListCapacity) .getStringValue(TSFileConfig.STRING_CHARSET); } - public void put(long timestamp, Object value) throws IOException { - switch (dataType) { - case INT32: - putInt(timestamp, (Integer) value); - break; - case INT64: - putLong(timestamp, (Long) value); - break; - case FLOAT: - putFloat(timestamp, (Float) value); - break; - case DOUBLE: - putDouble(timestamp, (Double) value); - break; - case BOOLEAN: - putBoolean(timestamp, (Boolean) value); - break; - case TEXT: - putBinary( - timestamp, - UDFBinaryTransformer.transformToUDFBinary((org.apache.tsfile.utils.Binary) value)); - break; - default: - throw new UnSupportedDataTypeException( - String.format("Data type %s is not supported.", dataType)); - } - } - - @Override - public void putInt(long timestamp, int value) throws IOException { - checkExpansion(); - cache.get(size / internalTVListCapacity).putInt(timestamp, value); - ++size; - } - - @Override - public void putLong(long timestamp, long value) throws IOException { - checkExpansion(); - cache.get(size / internalTVListCapacity).putLong(timestamp, value); - ++size; - } - - @Override - public void putFloat(long timestamp, float value) throws IOException { - checkExpansion(); - cache.get(size / internalTVListCapacity).putFloat(timestamp, value); - ++size; - } - - @Override - public void putDouble(long timestamp, double value) throws IOException { - checkExpansion(); - cache.get(size / internalTVListCapacity).putDouble(timestamp, value); - ++size; - } + // endregion - @Override - public void putBoolean(long timestamp, boolean value) throws IOException { - checkExpansion(); - cache.get(size / internalTVListCapacity).putBoolean(timestamp, value); - ++size; + // region batch data points methods + public TimeColumn getTimeColumn(int externalIndex, int internalIndex) throws IOException { + return cache.get(externalIndex).getTimeColumn(internalIndex); } - @Override - public void putBinary(long timestamp, Binary value) throws IOException { - checkExpansion(); - cache - .get(size / internalTVListCapacity) - .putBinary(timestamp, UDFBinaryTransformer.transformToBinary(value)); - ++size; + public Column getValueColumn(int externalIndex, int internalIndex) throws IOException { + return cache.get(externalIndex).getValueColumn(internalIndex); } - @Override - public void putString(long timestamp, String value) throws IOException { + public void putColumn(TimeColumn timeColumn, Column valueColumn) throws IOException { checkExpansion(); - cache.get(size / internalTVListCapacity).putBinary(timestamp, BytesUtils.valueOf(value)); - ++size; - } - - public void putNull(long timestamp) throws IOException { - switch (dataType) { - case INT32: - putInt(timestamp, 0); - break; - case INT64: - putLong(timestamp, 0L); - break; - case FLOAT: - putFloat(timestamp, 0.0F); - break; - case DOUBLE: - putDouble(timestamp, 0.0D); - break; - case BOOLEAN: - putBoolean(timestamp, false); - break; - case TEXT: - putBinary( - timestamp, - UDFBinaryTransformer.transformToUDFBinary(org.apache.tsfile.utils.Binary.EMPTY_VALUE)); - break; - default: - throw new UnSupportedDataTypeException( - String.format("Data type %s is not supported.", dataType)); - } - bitMaps.get((size - 1) / internalTVListCapacity).mark((size - 1) % internalTVListCapacity); - } - - private void checkExpansion() { - if (size % internalTVListCapacity == 0) { - tvLists.add(SerializableTVList.newSerializableTVList(dataType, queryId)); - bitMaps.add(new BitMap(internalTVListCapacity)); - } - } - - public LayerPointReader constructPointReaderUsingTrivialEvictionStrategy() { - return new LayerPointReader() { - - private int currentPointIndex = -1; - - @Override - public boolean isConstantPointReader() { - return false; - } - - @Override - public YieldableState yield() { - throw new UnsupportedOperationException(); - } - - @Override - public boolean next() { - if (size - 1 <= currentPointIndex) { - return false; + int begin = 0, end = 0; + int total = timeColumn.getPositionCount(); + while (total > 0) { + int consumed; + TimeColumn insertedTimeColumn; + Column insertedValueColumn; + if (total + pointCount % internalTVListCapacity < internalTVListCapacity) { + consumed = total; + if (begin == 0) { + // No need to copy if the columns do not split + insertedTimeColumn = timeColumn; + insertedValueColumn = valueColumn; + } else { + insertedTimeColumn = (TimeColumn) timeColumn.getRegionCopy(begin, consumed); + insertedValueColumn = valueColumn.getRegionCopy(begin, consumed); } - ++currentPointIndex; - return true; - } - - @Override - public void readyForNext() { - setEvictionUpperBound(currentPointIndex + 1); + } else { + consumed = internalTVListCapacity - pointCount % internalTVListCapacity; + // Construct sub-regions + insertedTimeColumn = (TimeColumn) timeColumn.getRegionCopy(begin, consumed); + insertedValueColumn = valueColumn.getRegionCopy(begin, consumed); } - @Override - public TSDataType getDataType() { - return dataType; - } + end += consumed; + begin = end; + total -= consumed; - @Override - public long currentTime() throws IOException { - return getTime(currentPointIndex); - } + // Fill row record list + cache + .get(pointCount / internalTVListCapacity) + .putColumns(insertedTimeColumn, insertedValueColumn); + pointCount += consumed; - @Override - public int currentInt() throws IOException { - return getInt(currentPointIndex); + if (total > 0) { + doExpansion(); } + } + } - @Override - public long currentLong() throws IOException { - return getLong(currentPointIndex); - } + // endregion - @Override - public float currentFloat() throws IOException { - return getFloat(currentPointIndex); - } + public TVListForwardIterator constructIterator() { + TVListForwardIterator iterator = new TVListForwardIterator(this); + iteratorList.add(iterator); - @Override - public double currentDouble() throws IOException { - return getDouble(currentPointIndex); - } - - @Override - public boolean currentBoolean() throws IOException { - return getBoolean(currentPointIndex); - } + return iterator; + } - @Override - public org.apache.tsfile.utils.Binary currentBinary() throws IOException { - return getBinary(currentPointIndex); - } + private void checkExpansion() { + if (pointCount % internalTVListCapacity == 0) { + doExpansion(); + } + } - @Override - public boolean isCurrentNull() throws IOException { - return isNull(currentPointIndex); - } - }; + private void doExpansion() { + if (internalTVList.size() > 0) { + int lastIndex = internalTVList.size() - 1; + SerializableTVList lastInternalList = internalTVList.get(lastIndex); + internalColumnCountList.add(lastInternalList.getColumnCount()); + } + internalTVList.add(SerializableTVList.construct(queryId)); } /** @@ -360,27 +247,42 @@ public void setEvictionUpperBound(int evictionUpperBound) { this.evictionUpperBound = evictionUpperBound; } - private class LRUCache extends Cache { + public int getColumnCount(int index) { + if (index == internalTVList.size() - 1) { + SerializableTVList lastList = internalTVList.get(index); + return lastList.getColumnCount(); + } else { + return internalColumnCountList.get(index); + } + } + + public int getLastPointIndex(int externalIndex, int internalIndex) { + int index = internalTVListCapacity * externalIndex; + int offset = internalTVList.get(externalIndex).getLastPointIndex(internalIndex); + + return index + offset; + } + + public class LRUCache extends Cache { LRUCache(int capacity) { super(capacity); } - BatchData get(int targetIndex) throws IOException { + public SerializableTVList get(int targetIndex) throws IOException { if (!containsKey(targetIndex)) { if (cacheCapacity <= super.size()) { int lastIndex = getLast(); if (lastIndex < evictionUpperBound / internalTVListCapacity) { - tvLists.set(lastIndex, null); - bitMaps.set(lastIndex, null); + internalTVList.set(lastIndex, null); } else { - tvLists.get(lastIndex).serialize(); + internalTVList.get(lastIndex).serialize(); } } - tvLists.get(targetIndex).deserialize(); + internalTVList.get(targetIndex).deserialize(); } putKey(targetIndex); - return tvLists.get(targetIndex); + return internalTVList.get(targetIndex); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableBooleanTVList.java deleted file mode 100644 index 85be689b80d3..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableBooleanTVList.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure.tv; - -import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.PublicBAOS; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; - -public class SerializableBooleanTVList extends SerializableTVList { - - protected static int calculateCapacity(float memoryLimitInMB) { - float memoryLimitInB = memoryLimitInMB * MB / 2; - // One time-value pair with 1 extra bit(1/8 Byte) in bitMap - return TSFileConfig.ARRAY_CAPACITY_THRESHOLD - * (int) - (memoryLimitInB - / ((ReadWriteIOUtils.LONG_LEN - + ReadWriteIOUtils.BOOLEAN_LEN - + ReadWriteIOUtils.BIT_LEN) - * TSFileConfig.ARRAY_CAPACITY_THRESHOLD)); - } - - protected SerializableBooleanTVList(SerializationRecorder serializationRecorder) { - super(TSDataType.BOOLEAN, serializationRecorder); - } - - @Override - public void serialize(PublicBAOS outputStream) throws IOException { - int size = length(); - serializationRecorder.setSerializedElementSize(size); - int serializedByteLength = 0; - for (int i = 0; i < size; ++i) { - serializedByteLength += ReadWriteIOUtils.write(getTimeByIndex(i), outputStream); - serializedByteLength += ReadWriteIOUtils.write(getBooleanByIndex(i), outputStream); - } - serializationRecorder.setSerializedByteLength(serializedByteLength); - } - - @Override - public void deserialize(ByteBuffer byteBuffer) { - int serializedElementSize = serializationRecorder.getSerializedElementSize(); - for (int i = 0; i < serializedElementSize; ++i) { - putBoolean(ReadWriteIOUtils.readLong(byteBuffer), ReadWriteIOUtils.readBool(byteBuffer)); - } - } - - @Override - public void release() { - timeRet = null; - booleanRet = null; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableDoubleTVList.java deleted file mode 100644 index 7f5ccc649f44..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableDoubleTVList.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure.tv; - -import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.PublicBAOS; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; - -public class SerializableDoubleTVList extends SerializableTVList { - - protected static int calculateCapacity(float memoryLimitInMB) { - float memoryLimitInB = memoryLimitInMB * MB / 2; - // One time-value pair with 1 extra bit(1/8 Byte) in bitMap - return TSFileConfig.ARRAY_CAPACITY_THRESHOLD - * (int) - (memoryLimitInB - / ((ReadWriteIOUtils.LONG_LEN - + ReadWriteIOUtils.DOUBLE_LEN - + ReadWriteIOUtils.BIT_LEN) - * TSFileConfig.ARRAY_CAPACITY_THRESHOLD)); - } - - protected SerializableDoubleTVList(SerializationRecorder serializationRecorder) { - super(TSDataType.DOUBLE, serializationRecorder); - } - - @Override - public void serialize(PublicBAOS outputStream) throws IOException { - int size = length(); - serializationRecorder.setSerializedElementSize(size); - int serializedByteLength = 0; - for (int i = 0; i < size; ++i) { - serializedByteLength += ReadWriteIOUtils.write(getTimeByIndex(i), outputStream); - serializedByteLength += ReadWriteIOUtils.write(getDoubleByIndex(i), outputStream); - } - serializationRecorder.setSerializedByteLength(serializedByteLength); - } - - @Override - public void deserialize(ByteBuffer byteBuffer) { - int serializedElementSize = serializationRecorder.getSerializedElementSize(); - for (int i = 0; i < serializedElementSize; ++i) { - putDouble(ReadWriteIOUtils.readLong(byteBuffer), ReadWriteIOUtils.readDouble(byteBuffer)); - } - } - - @Override - public void release() { - timeRet = null; - doubleRet = null; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableFloatTVList.java deleted file mode 100644 index 175cc3f0b8f5..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableFloatTVList.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure.tv; - -import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.PublicBAOS; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; - -public class SerializableFloatTVList extends SerializableTVList { - - protected static int calculateCapacity(float memoryLimitInMB) { - float memoryLimitInB = memoryLimitInMB * MB / 2; - // One time-value pair with 1 extra bit(1/8 Byte) in bitMap - return TSFileConfig.ARRAY_CAPACITY_THRESHOLD - * (int) - (memoryLimitInB - / ((ReadWriteIOUtils.LONG_LEN - + ReadWriteIOUtils.FLOAT_LEN - + ReadWriteIOUtils.BIT_LEN) - * TSFileConfig.ARRAY_CAPACITY_THRESHOLD)); - } - - protected SerializableFloatTVList(SerializationRecorder serializationRecorder) { - super(TSDataType.FLOAT, serializationRecorder); - } - - @Override - public void serialize(PublicBAOS outputStream) throws IOException { - int size = length(); - serializationRecorder.setSerializedElementSize(size); - int serializedByteLength = 0; - for (int i = 0; i < size; ++i) { - serializedByteLength += ReadWriteIOUtils.write(getTimeByIndex(i), outputStream); - serializedByteLength += ReadWriteIOUtils.write(getFloatByIndex(i), outputStream); - } - serializationRecorder.setSerializedByteLength(serializedByteLength); - } - - @Override - public void deserialize(ByteBuffer byteBuffer) { - int serializedElementSize = serializationRecorder.getSerializedElementSize(); - for (int i = 0; i < serializedElementSize; ++i) { - putFloat(ReadWriteIOUtils.readLong(byteBuffer), ReadWriteIOUtils.readFloat(byteBuffer)); - } - } - - @Override - public void release() { - timeRet = null; - floatRet = null; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableIntTVList.java deleted file mode 100644 index a28b8e9de5f2..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableIntTVList.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure.tv; - -import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.PublicBAOS; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; - -public class SerializableIntTVList extends SerializableTVList { - - protected static int calculateCapacity(float memoryLimitInMB) { - float memoryLimitInB = memoryLimitInMB * MB / 2; - // One time-value pair with 1 extra bit(1/8 Byte) in bitMap - return TSFileConfig.ARRAY_CAPACITY_THRESHOLD - * (int) - (memoryLimitInB - / ((ReadWriteIOUtils.LONG_LEN + ReadWriteIOUtils.INT_LEN + ReadWriteIOUtils.BIT_LEN) - * TSFileConfig.ARRAY_CAPACITY_THRESHOLD)); - } - - protected SerializableIntTVList(SerializationRecorder serializationRecorder) { - super(TSDataType.INT32, serializationRecorder); - } - - @Override - public void serialize(PublicBAOS outputStream) throws IOException { - int size = length(); - serializationRecorder.setSerializedElementSize(size); - int serializedByteLength = 0; - for (int i = 0; i < size; ++i) { - serializedByteLength += ReadWriteIOUtils.write(getTimeByIndex(i), outputStream); - serializedByteLength += ReadWriteIOUtils.write(getIntByIndex(i), outputStream); - } - serializationRecorder.setSerializedByteLength(serializedByteLength); - } - - @Override - public void deserialize(ByteBuffer byteBuffer) { - int serializedElementSize = serializationRecorder.getSerializedElementSize(); - for (int i = 0; i < serializedElementSize; ++i) { - putInt(ReadWriteIOUtils.readLong(byteBuffer), ReadWriteIOUtils.readInt(byteBuffer)); - } - } - - @Override - public void release() { - timeRet = null; - intRet = null; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableLongTVList.java deleted file mode 100644 index f4bec46826e3..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableLongTVList.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure.tv; - -import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.PublicBAOS; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; - -public class SerializableLongTVList extends SerializableTVList { - - protected static int calculateCapacity(float memoryLimitInMB) { - float memoryLimitInB = memoryLimitInMB * MB / 2; - // One time-value pair with 1 extra bit(1/8 Byte) in bitMap - return TSFileConfig.ARRAY_CAPACITY_THRESHOLD - * (int) - (memoryLimitInB - / ((ReadWriteIOUtils.LONG_LEN - + ReadWriteIOUtils.LONG_LEN - + ReadWriteIOUtils.BIT_LEN) - * TSFileConfig.ARRAY_CAPACITY_THRESHOLD)); - } - - protected SerializableLongTVList(SerializationRecorder serializationRecorder) { - super(TSDataType.INT64, serializationRecorder); - } - - @Override - public void serialize(PublicBAOS outputStream) throws IOException { - int size = length(); - serializationRecorder.setSerializedElementSize(size); - int serializedByteLength = 0; - for (int i = 0; i < size; ++i) { - serializedByteLength += ReadWriteIOUtils.write(getTimeByIndex(i), outputStream); - serializedByteLength += ReadWriteIOUtils.write(getLongByIndex(i), outputStream); - } - serializationRecorder.setSerializedByteLength(serializedByteLength); - } - - @Override - public void deserialize(ByteBuffer byteBuffer) { - int serializedElementSize = serializationRecorder.getSerializedElementSize(); - for (int i = 0; i < serializedElementSize; ++i) { - putLong(ReadWriteIOUtils.readLong(byteBuffer), ReadWriteIOUtils.readLong(byteBuffer)); - } - } - - @Override - public void release() { - timeRet = null; - longRet = null; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java index 3aa58c5108a7..0c8030f01eef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java @@ -19,89 +19,388 @@ package org.apache.iotdb.db.queryengine.transformation.datastructure.tv; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.queryengine.transformation.datastructure.SerializableList; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.block.column.TsBlockSerde; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; -public abstract class SerializableTVList extends BatchData implements SerializableList { +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; - public static SerializableTVList newSerializableTVList(TSDataType dataType, String queryId) { +import static com.google.common.base.Preconditions.checkState; +import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; +import static org.apache.iotdb.db.queryengine.transformation.datastructure.util.BinaryUtils.MIN_ARRAY_HEADER_SIZE; +import static org.apache.iotdb.db.queryengine.transformation.datastructure.util.BinaryUtils.MIN_OBJECT_HEADER_SIZE; + +public class SerializableTVList implements SerializableList { + protected final SerializationRecorder serializationRecorder; + + private final TsBlockSerde serde; + + private List valueColumns; + + private List timeColumns; + + private final List columnSizes; + + private int size; + + public static SerializableTVList construct(String queryId) { SerializationRecorder recorder = new SerializationRecorder(queryId); - switch (dataType) { - case INT32: - case DATE: - return new SerializableIntTVList(recorder); - case INT64: - case TIMESTAMP: - return new SerializableLongTVList(recorder); - case FLOAT: - return new SerializableFloatTVList(recorder); - case DOUBLE: - return new SerializableDoubleTVList(recorder); - case BOOLEAN: - return new SerializableBooleanTVList(recorder); - case TEXT: - case STRING: - case BLOB: - return new SerializableBinaryTVList(recorder); - default: - throw new UnSupportedDataTypeException(dataType.toString()); - } + return new SerializableTVList(recorder); + } + + protected SerializableTVList(SerializationRecorder serializationRecorder) { + this.serializationRecorder = serializationRecorder; + + serde = new TsBlockSerde(); + columnSizes = new ArrayList<>(); + size = 0; + + init(); } protected static int calculateCapacity(TSDataType dataType, float memoryLimitInMB) { - int size; - switch (dataType) { + int rowLength = ReadWriteIOUtils.LONG_LEN; // timestamp + switch (dataType) { // value case INT32: case DATE: - size = SerializableIntTVList.calculateCapacity(memoryLimitInMB); + rowLength += ReadWriteIOUtils.INT_LEN; break; case INT64: case TIMESTAMP: - size = SerializableLongTVList.calculateCapacity(memoryLimitInMB); + rowLength += ReadWriteIOUtils.LONG_LEN; break; case FLOAT: - size = SerializableFloatTVList.calculateCapacity(memoryLimitInMB); + rowLength += ReadWriteIOUtils.FLOAT_LEN; break; case DOUBLE: - size = SerializableDoubleTVList.calculateCapacity(memoryLimitInMB); + rowLength += ReadWriteIOUtils.DOUBLE_LEN; break; case BOOLEAN: - size = SerializableBooleanTVList.calculateCapacity(memoryLimitInMB); + rowLength += ReadWriteIOUtils.BOOLEAN_LEN; break; case TEXT: case STRING: case BLOB: - size = - SerializableBinaryTVList.calculateCapacity( - memoryLimitInMB, SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL); + rowLength += + MIN_OBJECT_HEADER_SIZE + + MIN_ARRAY_HEADER_SIZE + + SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL; break; default: throw new UnSupportedDataTypeException(dataType.toString()); } + rowLength += ReadWriteIOUtils.BIT_LEN; - if (size <= 0) { + int capacity = + TSFileConfig.ARRAY_CAPACITY_THRESHOLD + * (int) + (memoryLimitInMB * MB / 2 / (rowLength * TSFileConfig.ARRAY_CAPACITY_THRESHOLD)); + if (capacity <= 0) { throw new RuntimeException("Memory is not enough for current query."); } - return size; + return capacity; } - protected final SerializationRecorder serializationRecorder; + @Override + public SerializationRecorder getSerializationRecorder() { + return serializationRecorder; + } - protected SerializableTVList(TSDataType type, SerializationRecorder serializationRecorder) { - super(type); - this.serializationRecorder = serializationRecorder; + public int getColumnCount() { + return columnSizes.size(); + } + + // region single data methods + public long getTime(int index) { + checkState(index < size); + + long ret = 0; + int total = 0; + for (TimeColumn column : timeColumns) { + int length = column.getPositionCount(); + if (index < total + length) { + int offset = index - total; + ret = column.getLong(offset); + break; + } + total += length; + } + + return ret; + } + + public int getInt(int index) { + checkState(index < size); + + int ret = 0; + int total = 0; + for (Column column : valueColumns) { + int length = column.getPositionCount(); + if (index < total + length) { + int offset = index - total; + ret = column.getInt(offset); + break; + } + total += length; + } + + return ret; + } + + public long getLong(int index) { + checkState(index < size); + + long ret = 0; + int total = 0; + for (Column column : valueColumns) { + int length = column.getPositionCount(); + if (index < total + length) { + int offset = index - total; + ret = column.getLong(offset); + break; + } + total += length; + } + + return ret; + } + + public float getFloat(int index) { + checkState(index < size); + + float ret = 0; + int total = 0; + for (Column column : valueColumns) { + int length = column.getPositionCount(); + if (index < total + length) { + int offset = index - total; + ret = column.getFloat(offset); + break; + } + total += length; + } + + return ret; + } + + public double getDouble(int index) { + checkState(index < size); + + double ret = 0; + int total = 0; + for (Column column : valueColumns) { + int length = column.getPositionCount(); + if (index < total + length) { + int offset = index - total; + ret = column.getDouble(offset); + break; + } + total += length; + } + + return ret; + } + + public boolean getBoolean(int index) { + checkState(index < size); + + boolean ret = false; + int total = 0; + for (Column column : valueColumns) { + int length = column.getPositionCount(); + if (index < total + length) { + int offset = index - total; + ret = column.getBoolean(offset); + break; + } + total += length; + } + + return ret; + } + + public Binary getBinary(int index) { + checkState(index < size); + + Binary ret = null; + int total = 0; + for (Column column : valueColumns) { + int length = column.getPositionCount(); + if (index < total + length) { + int offset = index - total; + ret = column.getBinary(offset); + break; + } + total += length; + } + + return ret; + } + + public boolean isNull(int index) { + checkState(index < size); + + boolean ret = false; + int total = 0; + for (Column column : valueColumns) { + int length = column.getPositionCount(); + if (index < total + length) { + int offset = index - total; + ret = column.isNull(offset); + break; + } + total += length; + } + + return ret; + } + + // endregion + + // region batch data method + public TimeColumn getTimeColumn(int index) { + checkState(index < timeColumns.size()); + return timeColumns.get(index); + } + + public Column getValueColumn(int index) { + checkState(index < valueColumns.size()); + return valueColumns.get(index); + } + + public void putColumns(TimeColumn timeColumn, Column valueColumn) { + timeColumns.add(timeColumn); + valueColumns.add(valueColumn); + + int columnSize = timeColumn.getPositionCount(); + columnSizes.add(columnSize); + size += timeColumn.getPositionCount(); + } + + // endregion + + public int getColumnSize(int index) { + checkState(index < columnSizes.size()); + return columnSizes.get(index); + } + + public int getColumnIndex(int pointIndex) { + checkState(pointIndex < size); + + int ret = -1; + int total = 0; + for (int i = 0; i < columnSizes.size(); i++) { + int length = columnSizes.get(i); + if (pointIndex < total + length) { + ret = i; + break; + } + total += length; + } + + return ret; + } + + public int getTVOffsetInColumns(int index) { + checkState(index < size); + + int ret = -1; + int total = 0; + for (int length : columnSizes) { + if (index < total + length) { + ret = index - total; + break; + } + total += length; + } + + return ret; + } + + public int getLastPointIndex(int columnIndex) { + int total = 0; + for (int i = 0; i <= columnIndex; i++) { + total += columnSizes.get(i); + } + + return total; } @Override - public SerializationRecorder getSerializationRecorder() { - return serializationRecorder; + public void release() { + timeColumns = null; + valueColumns = null; } @Override public void init() { - init(getDataType()); + timeColumns = new ArrayList<>(); + valueColumns = new ArrayList<>(); + } + + @Override + public void serialize(PublicBAOS outputStream) throws IOException { + int bufferSize = 0; + + // Write TsBlocks count + bufferSize += ReadWriteIOUtils.write(timeColumns.size(), outputStream); + + for (int i = 0; i < timeColumns.size(); i++) { + // Construct TsBlock + TimeColumn timeColumn = timeColumns.get(i); + Column valueColumn = valueColumns.get(i); + TsBlock tsBlock = new TsBlock(timeColumn, valueColumn); + + ByteBuffer buffer = serde.serialize(tsBlock); + byte[] byteArray = buffer.array(); + // Write TsBlock data + outputStream.write(byteArray); + bufferSize += byteArray.length; + } + + serializationRecorder.setSerializedByteLength(bufferSize); + } + + @Override + public void deserialize(ByteBuffer byteBuffer) { + // Read TsBlocks count + int blockCount = ReadWriteIOUtils.readInt(byteBuffer); + timeColumns = new ArrayList<>(blockCount); + valueColumns = new ArrayList<>(blockCount); + + for (int i = 0; i < blockCount; i++) { + // Read TsBlocks data + TsBlock tsBlock = serde.deserialize(byteBuffer); + + // Unpack columns + TimeColumn timeColumn = tsBlock.getTimeColumn(); + Column valueColumn = tsBlock.getColumn(0); + + timeColumns.add(timeColumn); + valueColumns.add(valueColumn); + } + } + + @TestOnly + public List getTimeColumns() { + return timeColumns; + } + + @TestOnly + public List getValueColumns() { + return valueColumns; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/util/BinaryUtils.java similarity index 51% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableBinaryTVList.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/util/BinaryUtils.java index 3a24ba35a040..38092efef8e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/util/BinaryUtils.java @@ -17,24 +17,19 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.transformation.datastructure.tv; +package org.apache.iotdb.db.queryengine.transformation.datastructure.util; import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; -import java.io.IOException; -import java.nio.ByteBuffer; - import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; -public class SerializableBinaryTVList extends SerializableTVList { +public class BinaryUtils { + public static final int MIN_OBJECT_HEADER_SIZE = 8; - protected static final int MIN_OBJECT_HEADER_SIZE = 8; - protected static final int MIN_ARRAY_HEADER_SIZE = MIN_OBJECT_HEADER_SIZE + 4; + public static final int MIN_ARRAY_HEADER_SIZE = MIN_OBJECT_HEADER_SIZE + 4; - protected static int calculateCapacity(float memoryLimitInMB, int byteArrayLength) { + public static int calculateCapacity(float memoryLimitInMB, int byteArrayLength) { float memoryLimitInB = memoryLimitInMB * MB / 2; return TSFileConfig.ARRAY_CAPACITY_THRESHOLD * (int) @@ -43,41 +38,11 @@ protected static int calculateCapacity(float memoryLimitInMB, int byteArrayLengt * calculateSingleBinaryTVPairMemory(byteArrayLength))); } - protected static float calculateSingleBinaryTVPairMemory(int byteArrayLength) { + public static float calculateSingleBinaryTVPairMemory(int byteArrayLength) { return ReadWriteIOUtils.LONG_LEN // time + MIN_OBJECT_HEADER_SIZE // value: header length of Binary + MIN_ARRAY_HEADER_SIZE // value: header length of values in Binary + byteArrayLength // value: length of values array in Binary + ReadWriteIOUtils.BIT_LEN; // extra bit(1/8 Byte): whether the Binary is null } - - protected SerializableBinaryTVList(SerializationRecorder serializationRecorder) { - super(TSDataType.TEXT, serializationRecorder); - } - - @Override - public void serialize(PublicBAOS outputStream) throws IOException { - int size = length(); - serializationRecorder.setSerializedElementSize(size); - int serializedByteLength = 0; - for (int i = 0; i < size; ++i) { - serializedByteLength += ReadWriteIOUtils.write(getTimeByIndex(i), outputStream); - serializedByteLength += ReadWriteIOUtils.write(getBinaryByIndex(i), outputStream); - } - serializationRecorder.setSerializedByteLength(serializedByteLength); - } - - @Override - public void deserialize(ByteBuffer byteBuffer) { - int serializedElementSize = serializationRecorder.getSerializedElementSize(); - for (int i = 0; i < serializedElementSize; ++i) { - putBinary(ReadWriteIOUtils.readLong(byteBuffer), ReadWriteIOUtils.readBinary(byteBuffer)); - } - } - - @Override - public void release() { - timeRet = null; - binaryRet = null; - } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/builder/EvaluationDAGBuilderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/builder/EvaluationDAGBuilderTest.java index 80afc839d479..5f4ee8d9e400 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/builder/EvaluationDAGBuilderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/builder/EvaluationDAGBuilderTest.java @@ -45,7 +45,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; import org.apache.iotdb.db.queryengine.plan.statement.Statement; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.api.LayerReader; import org.apache.iotdb.db.queryengine.transformation.dag.transformer.multi.UDFQueryRowWindowTransformer; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.IDataRegionForQuery; @@ -73,7 +73,7 @@ public void testBuildWithNonMappable() { Assert.assertNotNull(operator); TransformOperator transformOperator = (TransformOperator) ((IdentitySinkOperator) operator).getChildren().get(0); - LayerPointReader[] transformers = transformOperator.getTransformers(); + LayerReader[] transformers = transformOperator.getTransformers(); Assert.assertEquals(6, transformers.length); Assert.assertTrue(transformers[0] instanceof UDFQueryRowWindowTransformer); } catch (Exception e) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableRowListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableRowListTest.java new file mode 100644 index 000000000000..0544f18d4b6f --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableRowListTest.java @@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.datastructure; + +import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.transformation.datastructure.iterator.RowListForwardIterator; +import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowList; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; +import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; +import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Optional; +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class ElasticSerializableRowListTest extends SerializableListTest { + + private static final TSDataType[] DATA_TYPES = { + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.BOOLEAN, + TSDataType.TEXT, + TSDataType.TEXT + }; + + private ElasticSerializableRowList rowList; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Override + @After + public void tearDown() { + super.tearDown(); + } + + @Test + public void testPutAndGet() { + initESRowRecordList(); + + testPuts(); + + testGetByIndex(); + } + + private void initESRowRecordList() { + try { + rowList = + new ElasticSerializableRowList( + DATA_TYPES, QUERY_ID, MEMORY_USAGE_LIMIT_IN_MB, CACHE_SIZE); + } catch (QueryProcessException e) { + fail(e.toString()); + } + assertEquals(0, rowList.size()); + } + + private void testPuts() { + try { + Column[] columns = generateColumns(); + rowList.put(columns); + } catch (IOException | QueryProcessException e) { + fail(e.toString()); + } + assertEquals(ITERATION_TIMES, rowList.size()); + } + + private Column[] generateColumns() { + Column[] columns = new Column[DATA_TYPES.length + 1]; + + boolean[] isNulls = new boolean[ITERATION_TIMES]; + for (int i = 0; i < ITERATION_TIMES; i++) { + isNulls[i] = i % 7 == 0; + } + // Int columns + IntColumnBuilder intColumnBuilder = new IntColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + if (i % 7 == 0) { + intColumnBuilder.appendNull(); + } else { + intColumnBuilder.writeInt(i); + } + } + columns[0] = intColumnBuilder.build(); + + // Long columns + LongColumnBuilder longColumnBuilder = new LongColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + if (i % 7 == 0) { + longColumnBuilder.appendNull(); + } else { + longColumnBuilder.writeLong(i); + } + } + columns[1] = longColumnBuilder.build(); + + // Float columns + FloatColumnBuilder floatColumnBuilder = new FloatColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + if (i % 7 == 0) { + floatColumnBuilder.appendNull(); + } else { + floatColumnBuilder.writeFloat(i); + } + } + columns[2] = floatColumnBuilder.build(); + + // Double columns + DoubleColumnBuilder doubleColumnBuilder = new DoubleColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + if (i % 7 == 0) { + doubleColumnBuilder.appendNull(); + } else { + doubleColumnBuilder.writeDouble(i); + } + } + columns[3] = doubleColumnBuilder.build(); + + // Boolean columns + BooleanColumnBuilder booleanColumnBuilder = new BooleanColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + if (i % 7 == 0) { + booleanColumnBuilder.appendNull(); + } else { + booleanColumnBuilder.writeBoolean(i % 2 == 0); + } + } + columns[4] = booleanColumnBuilder.build(); + + // Binary columns + BinaryColumnBuilder binaryColumnBuilder = new BinaryColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + if (i % 7 == 0) { + binaryColumnBuilder.appendNull(); + } else { + Binary binary = BytesUtils.valueOf(String.valueOf(i)); + binaryColumnBuilder.writeBinary(binary); + } + } + columns[5] = binaryColumnBuilder.build(); + + // Another binary columns + Binary[] binaries = columns[5].getBinaries().clone(); + columns[6] = new BinaryColumn(ITERATION_TIMES, Optional.of(isNulls), binaries.clone()); + + // The last time columns + TimeColumnBuilder timeColumnBuilder = new TimeColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + timeColumnBuilder.writeLong(i); + } + columns[7] = timeColumnBuilder.build(); + + return columns; + } + + private void testRowRecord(Object[] rowRecord, int expected) { + for (int j = 0; j < DATA_TYPES.length; ++j) { + switch (DATA_TYPES[j]) { + case INT32: + assertEquals(expected, (int) rowRecord[j]); + break; + case INT64: + assertEquals(expected, (long) rowRecord[j]); + break; + case FLOAT: + assertEquals(expected, (float) rowRecord[j], 0); + break; + case DOUBLE: + assertEquals(expected, (double) rowRecord[j], 0); + break; + case BOOLEAN: + assertEquals(expected % 2 == 0, rowRecord[j]); + break; + case TEXT: + assertEquals(BytesUtils.valueOf(String.valueOf(expected)), rowRecord[j]); + break; + } + } + assertEquals(DATA_TYPES.length, rowRecord.length - 1); + } + + private void testGetByIndex() { + try { + for (int i = 0; i < ITERATION_TIMES; ++i) { + if (i % 7 == 0) { + assertTrue(rowList.fieldsHasAnyNull(i)); + } else { + assertFalse(rowList.fieldsHasAnyNull(i)); + testRowRecord(rowList.getRowRecord(i), i); + } + } + } catch (IOException e) { + fail(e.toString()); + } + } + + @Test + public void testMemoryControl() { + initESRowRecordList(); + + int byteLengthMin = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 2; + int byteLengthMax = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 8; + + try { + Column[] columns = + generateColumnsWithRandomBinaries(ITERATION_TIMES, byteLengthMin, byteLengthMax); + rowList.put(columns); + rowList.setEvictionUpperBound(rowList.size()); + RowListForwardIterator iterator = rowList.constructIterator(); + testRowList(iterator); + + byteLengthMin = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 16; + byteLengthMax = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 32; + columns = generateColumnsWithRandomBinaries(ITERATION_TIMES, byteLengthMin, byteLengthMax); + rowList.put(columns); + rowList.setEvictionUpperBound(rowList.size()); + testRowList(iterator); + + byteLengthMin = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 256; + byteLengthMax = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 512; + columns = generateColumnsWithRandomBinaries(ITERATION_TIMES, byteLengthMin, byteLengthMax); + rowList.put(columns); + rowList.setEvictionUpperBound(rowList.size()); + testRowList(iterator); + + columns = generateColumnsWithRandomBinaries(ITERATION_TIMES, byteLengthMin, byteLengthMax); + rowList.put(columns); + rowList.setEvictionUpperBound(rowList.size()); + testRowList(iterator); + + columns = generateColumnsWithRandomBinaries(ITERATION_TIMES, byteLengthMin, byteLengthMax); + rowList.put(columns); + rowList.setEvictionUpperBound(rowList.size()); + testRowList(iterator); + + assertEquals(ITERATION_TIMES * 5, rowList.size()); + } catch (QueryProcessException | IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private void testRowList(RowListForwardIterator iterator) throws IOException { + int index = 0; + + while (iterator.hasNext()) { + iterator.next(); + Column[] columns = iterator.currentBlock(); + int count = columns[0].getPositionCount(); + for (int i = 0; i < count; i++, index++) { + if (index % 7 == 0) { + assertTrue(fieldHasAnyNull(columns, i)); + } else { + assertFalse(fieldHasAnyNull(columns, i)); + } + } + } + } + + private boolean fieldHasAnyNull(Column[] columns, int index) { + for (Column column : columns) { + if (column.isNull(index)) { + return true; + } + } + return false; + } + + private Column[] generateColumnsWithRandomBinaries( + int iterTimes, int byteLengthMin, int byteLengthMax) { + Random random = new Random(); + Column[] columns = new Column[DATA_TYPES.length + 1]; + + // Int columns + IntColumnBuilder intColumnBuilder = new IntColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < iterTimes; i++) { + if (i % 7 == 0) { + intColumnBuilder.appendNull(); + } else { + intColumnBuilder.writeInt(i); + } + } + columns[0] = intColumnBuilder.build(); + + // Long columns + LongColumnBuilder longColumnBuilder = new LongColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < iterTimes; i++) { + if (i % 7 == 0) { + longColumnBuilder.appendNull(); + } else { + longColumnBuilder.writeLong(i); + } + } + columns[1] = longColumnBuilder.build(); + + // Float columns + FloatColumnBuilder floatColumnBuilder = new FloatColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < iterTimes; i++) { + if (i % 7 == 0) { + floatColumnBuilder.appendNull(); + } else { + floatColumnBuilder.writeFloat(i); + } + } + columns[2] = floatColumnBuilder.build(); + + // Double columns + DoubleColumnBuilder doubleColumnBuilder = new DoubleColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < iterTimes; i++) { + if (i % 7 == 0) { + doubleColumnBuilder.appendNull(); + } else { + doubleColumnBuilder.writeDouble(i); + } + } + columns[3] = doubleColumnBuilder.build(); + + // Boolean columns + BooleanColumnBuilder booleanColumnBuilder = new BooleanColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < iterTimes; i++) { + if (i % 7 == 0) { + booleanColumnBuilder.appendNull(); + } else { + booleanColumnBuilder.writeBoolean(i % 2 == 0); + } + } + columns[4] = booleanColumnBuilder.build(); + + // Binary columns + BinaryColumnBuilder binaryColumnBuilder = new BinaryColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < iterTimes; i++) { + if (i % 7 == 0) { + binaryColumnBuilder.appendNull(); + } else { + Binary binary = + BytesUtils.valueOf( + generateStringByLength( + byteLengthMin + random.nextInt(byteLengthMax - byteLengthMin))); + binaryColumnBuilder.writeBinary(binary); + } + } + columns[5] = binaryColumnBuilder.build(); + + // Another binary columns + BinaryColumnBuilder anotherbinaryColumnBuilder = new BinaryColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < iterTimes; i++) { + if (i % 7 == 0) { + anotherbinaryColumnBuilder.appendNull(); + } else { + Binary binary = + BytesUtils.valueOf( + generateStringByLength( + byteLengthMin + random.nextInt(byteLengthMax - byteLengthMin))); + anotherbinaryColumnBuilder.writeBinary(binary); + } + } + columns[6] = anotherbinaryColumnBuilder.build(); + + // The last time columns + TimeColumnBuilder timeColumnBuilder = new TimeColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < iterTimes; i++) { + timeColumnBuilder.writeLong(i); + } + columns[7] = timeColumnBuilder.build(); + + return columns; + } + + private String generateStringByLength(int length) { + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 0; i < length; ++i) { + stringBuilder.append('.'); + } + return stringBuilder.toString(); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableRowRecordListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableRowRecordListTest.java deleted file mode 100644 index 5844dc4e2f36..000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableRowRecordListTest.java +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure; - -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.datastructure.row.ElasticSerializableRowRecordList; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.BytesUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.Random; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class ElasticSerializableRowRecordListTest extends SerializableListTest { - - private static final TSDataType[] DATA_TYPES = { - TSDataType.INT32, - TSDataType.INT64, - TSDataType.FLOAT, - TSDataType.DOUBLE, - TSDataType.BOOLEAN, - TSDataType.TEXT, - TSDataType.TEXT - }; - - private ElasticSerializableRowRecordList rowRecordList; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - } - - @Override - @After - public void tearDown() { - super.tearDown(); - } - - @Test - public void testESRowRecordList() { - initESRowRecordList(); - - testPut(); - - testOrderedAccessByIndex(); - } - - private void initESRowRecordList() { - try { - rowRecordList = - new ElasticSerializableRowRecordList( - DATA_TYPES, QUERY_ID, MEMORY_USAGE_LIMIT_IN_MB, CACHE_SIZE); - } catch (QueryProcessException e) { - fail(e.toString()); - } - assertEquals(0, rowRecordList.size()); - } - - private void testPut() { - try { - for (int i = 0; i < ITERATION_TIMES; ++i) { - Object[] rowRecord = new Object[DATA_TYPES.length + 1]; - rowRecord[DATA_TYPES.length] = (long) i; - if (i % 7 != 0) { - for (int j = 0; j < DATA_TYPES.length; ++j) { - switch (DATA_TYPES[j]) { - case INT32: - rowRecord[j] = i; - break; - case INT64: - rowRecord[j] = (long) i; - break; - case FLOAT: - rowRecord[j] = (float) i; - break; - case DOUBLE: - rowRecord[j] = (double) i; - break; - case BOOLEAN: - rowRecord[j] = i % 2 == 0; - break; - case TEXT: - rowRecord[j] = BytesUtils.valueOf(String.valueOf(i)); - break; - } - } - } - - rowRecordList.put(rowRecord); - } - } catch (IOException | QueryProcessException e) { - fail(e.toString()); - } - assertEquals(ITERATION_TIMES, rowRecordList.size()); - } - - private void testRowRecord(Object[] rowRecord, int expected) { - for (int j = 0; j < DATA_TYPES.length; ++j) { - switch (DATA_TYPES[j]) { - case INT32: - assertEquals(expected, (int) rowRecord[j]); - break; - case INT64: - assertEquals(expected, (long) rowRecord[j]); - break; - case FLOAT: - assertEquals(expected, (float) rowRecord[j], 0); - break; - case DOUBLE: - assertEquals(expected, (double) rowRecord[j], 0); - break; - case BOOLEAN: - assertEquals(expected % 2 == 0, rowRecord[j]); - break; - case TEXT: - assertEquals(BytesUtils.valueOf(String.valueOf(expected)), rowRecord[j]); - break; - } - } - assertEquals(DATA_TYPES.length, rowRecord.length - 1); - } - - private void testOrderedAccessByIndex() { - try { - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i % 7 == 0) { - assertTrue(rowRecordList.fieldsHasAnyNull(i)); - } else { - assertFalse(rowRecordList.fieldsHasAnyNull(i)); - testRowRecord(rowRecordList.getRowRecord(i), i); - } - } - } catch (IOException e) { - fail(e.toString()); - } - } - - @Test - public void testMemoryControl() { - initESRowRecordList(); - - int byteLengthMin = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 2; - int byteLengthMax = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 8; - Random random = new Random(); - - try { - for (int i = 0; i < ITERATION_TIMES; i++) { - if (i % 7 == 0) { - rowRecordList.put(generateRowRecordWithAllNullFields(i)); - } else { - rowRecordList.put( - generateRowRecord(i, byteLengthMin + random.nextInt(byteLengthMax - byteLengthMin))); - } - } - rowRecordList.setEvictionUpperBound(rowRecordList.size()); - for (int i = 0; i < ITERATION_TIMES; i++) { - if (i % 7 == 0) { - assertTrue(rowRecordList.fieldsHasAnyNull(i)); - } else { - assertFalse(rowRecordList.fieldsHasAnyNull(i)); - } - } - - byteLengthMin = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 16; - byteLengthMax = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 32; - for (int i = 0; i < ITERATION_TIMES; i++) { - if (i % 7 == 0) { - rowRecordList.put(generateRowRecordWithAllNullFields(i)); - } else { - rowRecordList.put( - generateRowRecord(i, byteLengthMin + random.nextInt(byteLengthMax - byteLengthMin))); - } - } - rowRecordList.setEvictionUpperBound(rowRecordList.size()); - for (int i = 0; i < ITERATION_TIMES; i++) { - if (i % 7 == 0) { - assertTrue(rowRecordList.fieldsHasAnyNull(i + ITERATION_TIMES)); - } else { - assertFalse(rowRecordList.fieldsHasAnyNull(i + ITERATION_TIMES)); - } - } - - byteLengthMin = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 256; - byteLengthMax = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 512; - for (int i = 0; i < ITERATION_TIMES; i++) { - if (i % 7 == 0) { - rowRecordList.put(generateRowRecordWithAllNullFields(i)); - } else { - rowRecordList.put( - generateRowRecord(i, byteLengthMin + random.nextInt(byteLengthMax - byteLengthMin))); - } - } - rowRecordList.setEvictionUpperBound(rowRecordList.size()); - for (int i = 0; i < ITERATION_TIMES; i++) { - if (i % 7 == 0) { - assertTrue(rowRecordList.fieldsHasAnyNull(i + 2 * ITERATION_TIMES)); - } else { - assertFalse(rowRecordList.fieldsHasAnyNull(i + 2 * ITERATION_TIMES)); - } - } - - for (int i = 0; i < 2 * ITERATION_TIMES; i++) { - if (i % 7 == 0) { - rowRecordList.put(generateRowRecordWithAllNullFields(i)); - } else { - rowRecordList.put( - generateRowRecord(i, byteLengthMin + random.nextInt(byteLengthMax - byteLengthMin))); - } - rowRecordList.setEvictionUpperBound(rowRecordList.size()); - } - - for (int i = 0; i < ITERATION_TIMES; i++) { - if (i % 7 == 0) { - assertTrue(rowRecordList.fieldsHasAnyNull(i + 3 * ITERATION_TIMES)); - } else { - assertFalse(rowRecordList.fieldsHasAnyNull(i + 3 * ITERATION_TIMES)); - } - } - - assertEquals(ITERATION_TIMES * 5, rowRecordList.size()); - } catch (QueryProcessException | IOException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - private Object[] generateRowRecord(int time, int byteLength) { - Object[] rowRecord = new Object[DATA_TYPES.length + 1]; - rowRecord[DATA_TYPES.length] = (long) time; - for (int i = 0; i < DATA_TYPES.length; ++i) { - switch (DATA_TYPES[i]) { - case INT32: - rowRecord[i] = time; - break; - case INT64: - rowRecord[i] = (long) time; - break; - case FLOAT: - rowRecord[i] = (float) time; - break; - case DOUBLE: - rowRecord[i] = (double) time; - break; - case BOOLEAN: - rowRecord[i] = time % 2 == 0; - break; - case TEXT: - rowRecord[i] = BytesUtils.valueOf(generateRandomString(byteLength)); - break; - } - } - return rowRecord; - } - - private Object[] generateRowRecordWithAllNullFields(int time) { - Object[] rowRecord = new Object[DATA_TYPES.length + 1]; - rowRecord[DATA_TYPES.length] = (long) time; - return rowRecord; - } - - private String generateRandomString(int length) { - StringBuilder stringBuilder = new StringBuilder(); - for (int i = 0; i < length; ++i) { - stringBuilder.append('.'); - } - return stringBuilder.toString(); - } -} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableTVListTest.java index 1250f937d675..148d34f54d67 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableTVListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/ElasticSerializableTVListTest.java @@ -19,19 +19,26 @@ package org.apache.iotdb.db.queryengine.transformation.datastructure; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.queryengine.transformation.api.LayerPointReader; +import org.apache.iotdb.db.queryengine.transformation.datastructure.iterator.TVListForwardIterator; import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.ElasticSerializableTVList; -import org.apache.iotdb.udf.api.type.Binary; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.IntColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.IOException; +import java.util.Optional; import java.util.Random; +import java.util.stream.IntStream; +import java.util.stream.LongStream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -55,187 +62,41 @@ public void tearDown() { } @Test - public void testESIntTVList() { - testESTVList(TSDataType.INT32); - } - - @Test - public void testESLongTVList() { - testESTVList(TSDataType.INT64); - } - - @Test - public void testESFloatTVList() { - testESTVList(TSDataType.FLOAT); - } - - @Test - public void testESDoubleTVList() { - testESTVList(TSDataType.DOUBLE); - } - - @Test - public void testESTextTVList() { - testESTVList(TSDataType.TEXT); - } - - @Test - public void testESBooleanTVList() { - testESTVList(TSDataType.BOOLEAN); - } - - private void testESTVList(TSDataType dataType) { - initESTVList(dataType); - - testPut(dataType); - - testOrderedAccessByIndex(dataType); - } + public void testESIntTVListWithBatchInsert() { + initESTVList(TSDataType.INT32); + + long[] times = LongStream.range(0, ITERATION_TIMES).toArray(); + int[] values = IntStream.range(0, ITERATION_TIMES).toArray(); + boolean[] isNulls = new boolean[ITERATION_TIMES]; + for (int i = 0; i < ITERATION_TIMES; i++) { + isNulls[i] = i % 7 == 0; + } - private void initESTVList(TSDataType dataType) { - tvList = - ElasticSerializableTVList.newElasticSerializableTVList( - dataType, QUERY_ID, MEMORY_USAGE_LIMIT_IN_MB, CACHE_SIZE); - assertEquals(0, tvList.size()); - } + TimeColumn timeColumn = new TimeColumn(ITERATION_TIMES, times); + IntColumn valueColumn = new IntColumn(ITERATION_TIMES, Optional.of(isNulls), values); - private void testPut(TSDataType dataType) { try { - switch (dataType) { - case INT32: - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i % 7 == 0) { - tvList.putNull(i); - } else { - tvList.putInt(i, i); - } - } - break; - case INT64: - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i % 7 == 0) { - tvList.putNull(i); - } else { - tvList.putLong(i, i); - } - } - break; - case FLOAT: - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i % 7 == 0) { - tvList.putNull(i); - } else { - tvList.putFloat(i, i); - } - } - break; - case DOUBLE: - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i % 7 == 0) { - tvList.putNull(i); - } else { - tvList.putDouble(i, i); - } - } - break; - case BOOLEAN: - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i % 7 == 0) { - tvList.putNull(i); - } else { - tvList.putBoolean(i, i % 2 == 0); - } - } - break; - case TEXT: - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i % 7 == 0) { - tvList.putNull(i); - } else { - tvList.putBinary(i, Binary.valueOf(String.valueOf(i))); - } - } - break; + tvList.putColumn(timeColumn, valueColumn); + + for (int i = 0; i < ITERATION_TIMES; ++i) { + assertEquals(i, tvList.getTime(i)); + if (i % 7 == 0) { + assertTrue(tvList.isNull(i)); + } else { + assertFalse(tvList.isNull(i)); + assertEquals(i, tvList.getInt(i)); + } } } catch (IOException e) { fail(e.toString()); } - assertEquals(ITERATION_TIMES, tvList.size()); } - private void testOrderedAccessByIndex(TSDataType dataType) { - try { - switch (dataType) { - case INT32: - for (int i = 0; i < ITERATION_TIMES; ++i) { - assertEquals(i, tvList.getTime(i)); - if (i % 7 == 0) { - assertTrue(tvList.isNull(i)); - } else { - assertFalse(tvList.isNull(i)); - assertEquals(i, tvList.getInt(i)); - } - } - break; - case INT64: - for (int i = 0; i < ITERATION_TIMES; ++i) { - assertEquals(i, tvList.getTime(i)); - if (i % 7 == 0) { - assertTrue(tvList.isNull(i)); - } else { - assertFalse(tvList.isNull(i)); - assertEquals(i, tvList.getLong(i)); - } - } - break; - case FLOAT: - for (int i = 0; i < ITERATION_TIMES; ++i) { - assertEquals(i, tvList.getTime(i)); - if (i % 7 == 0) { - assertTrue(tvList.isNull(i)); - } else { - assertFalse(tvList.isNull(i)); - assertEquals(i, tvList.getFloat(i), 0); - } - } - break; - case DOUBLE: - for (int i = 0; i < ITERATION_TIMES; ++i) { - assertEquals(i, tvList.getTime(i)); - if (i % 7 == 0) { - assertTrue(tvList.isNull(i)); - } else { - assertFalse(tvList.isNull(i)); - assertEquals(i, tvList.getDouble(i), 0); - } - } - break; - case BOOLEAN: - for (int i = 0; i < ITERATION_TIMES; ++i) { - assertEquals(i, tvList.getTime(i)); - if (i % 7 == 0) { - assertTrue(tvList.isNull(i)); - } else { - assertFalse(tvList.isNull(i)); - assertEquals(i % 2 == 0, tvList.getBoolean(i)); - } - } - break; - case TEXT: - for (int i = 0; i < ITERATION_TIMES; ++i) { - assertEquals(i, tvList.getTime(i)); - if (i % 7 == 0) { - assertTrue(tvList.isNull(i)); - } else { - assertFalse(tvList.isNull(i)); - assertEquals(BytesUtils.valueOf(String.valueOf(i)), tvList.getBinary(i)); - } - } - break; - } - } catch (IOException e) { - fail(e.toString()); - } + private void initESTVList(TSDataType dataType) { + tvList = + ElasticSerializableTVList.construct( + dataType, QUERY_ID, MEMORY_USAGE_LIMIT_IN_MB, CACHE_SIZE); + assertEquals(0, tvList.size()); } @Test @@ -244,113 +105,76 @@ public void testMemoryControl() { int byteLengthMin = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 2; int byteLengthMax = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 8; - Random random = new Random(); try { - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i % 7 == 0) { - tvList.putNull(i); - } else { - tvList.putBinary( - i, - Binary.valueOf( - generateRandomString( - byteLengthMin + random.nextInt(byteLengthMax - byteLengthMin)))); - } - } - LayerPointReader reader = tvList.constructPointReaderUsingTrivialEvictionStrategy(); - int index = 0; - while (reader.next()) { - if (index % 7 == 0) { - assertTrue(reader.isCurrentNull()); - } else { - int length = reader.currentBinary().getLength(); - assertTrue(byteLengthMin <= length && length < byteLengthMax); - } - reader.readyForNext(); - index++; - } + generateColumnsWithRandomSizeBinaries(ITERATION_TIMES, byteLengthMin, byteLengthMax); + TVListForwardIterator iterator = tvList.constructIterator(); + testBinaryLengthMatch(iterator, byteLengthMin, byteLengthMax); byteLengthMin = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 16; byteLengthMax = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 32; - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i % 7 == 0) { - tvList.putNull(i); - } else { - tvList.putBinary( - i, - Binary.valueOf( - generateRandomString( - byteLengthMin + random.nextInt(byteLengthMax - byteLengthMin)))); - } - } - index = 0; - while (reader.next()) { - if (index % 7 == 0) { - assertTrue(reader.isCurrentNull()); - } else { - int length = reader.currentBinary().getLength(); - assertTrue(byteLengthMin <= length && length < byteLengthMax); - } - reader.readyForNext(); - index++; - } + generateColumnsWithRandomSizeBinaries(ITERATION_TIMES, byteLengthMin, byteLengthMax); + testBinaryLengthMatch(iterator, byteLengthMin, byteLengthMax); byteLengthMin = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 256; byteLengthMax = SerializableList.INITIAL_BYTE_ARRAY_LENGTH_FOR_MEMORY_CONTROL * 512; - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i % 7 == 0) { - tvList.putNull(i); - } else { - tvList.putBinary( - i, - Binary.valueOf( - generateRandomString( - byteLengthMin + random.nextInt(byteLengthMax - byteLengthMin)))); - } - } - index = 0; - while (reader.next()) { - if (index % 7 == 0) { - assertTrue(reader.isCurrentNull()); - } else { - int length = reader.currentBinary().getLength(); - assertTrue(byteLengthMin <= length && length < byteLengthMax); - } - reader.readyForNext(); - index++; + generateColumnsWithRandomSizeBinaries(ITERATION_TIMES, byteLengthMin, byteLengthMax); + testBinaryLengthMatch(iterator, byteLengthMin, byteLengthMax); + + generateColumnsWithRandomSizeBinaries(ITERATION_TIMES * 2, byteLengthMin, byteLengthMax); + testBinaryLengthMatch(iterator, byteLengthMin, byteLengthMax); + + assertEquals(ITERATION_TIMES * 5, tvList.size()); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private void generateColumnsWithRandomSizeBinaries( + int iterTimes, int byteLengthMin, int byteLengthMax) throws IOException { + Random random = new Random(); + TimeColumnBuilder timeColumnBuilder = new TimeColumnBuilder(null, iterTimes); + BinaryColumnBuilder binaryColumnBuilder = new BinaryColumnBuilder(null, iterTimes); + + for (int i = 0; i < iterTimes; i++) { + timeColumnBuilder.writeLong(i); + if (i % 7 == 0) { + binaryColumnBuilder.appendNull(); + } else { + String randomString = + generateStringByLength(byteLengthMin + random.nextInt(byteLengthMax - byteLengthMin)); + Binary value = BytesUtils.valueOf(randomString); + binaryColumnBuilder.writeBinary(value); } + } - index = 0; - for (int i = 0; i < 2 * ITERATION_TIMES; ++i) { - if (i % 7 == 0) { - tvList.putNull(i); - } else { - tvList.putBinary( - i, - Binary.valueOf( - generateRandomString( - byteLengthMin + random.nextInt(byteLengthMax - byteLengthMin)))); - } - reader.next(); + TimeColumn timeColumn = (TimeColumn) timeColumnBuilder.build(); + Column valueColumn = binaryColumnBuilder.build(); + + tvList.putColumn(timeColumn, valueColumn); + } + + private void testBinaryLengthMatch( + TVListForwardIterator iterator, int byteLengthMin, int byteLengthMax) throws IOException { + int index = 0; + + while (iterator.hasNext()) { + iterator.next(); + Column column = iterator.currentValues(); + for (int i = 0; i < column.getPositionCount(); i++, index++) { if (index % 7 == 0) { - assertTrue(reader.isCurrentNull()); + assertTrue(column.isNull(i)); } else { - int length = reader.currentBinary().getLength(); + Binary binary = column.getBinary(i); + int length = binary.getLength(); assertTrue(byteLengthMin <= length && length < byteLengthMax); } - reader.readyForNext(); - index++; } - - assertEquals(ITERATION_TIMES * 5, tvList.size()); - } catch (QueryProcessException | IOException e) { - e.printStackTrace(); - fail(e.getMessage()); } } - private String generateRandomString(int length) { + private String generateStringByLength(int length) { StringBuilder stringBuilder = new StringBuilder(); for (int i = 0; i < length; ++i) { stringBuilder.append('.'); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableBinaryTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableBinaryTVListTest.java deleted file mode 100644 index becc377ea96a..000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableBinaryTVListTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure; - -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableBinaryTVList; -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableTVList; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BytesUtils; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class SerializableBinaryTVListTest extends SerializableTVListTest { - - private List originalList; - private SerializableBinaryTVList testList; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - originalList = new ArrayList<>(); - testList = - (SerializableBinaryTVList) - SerializableTVList.newSerializableTVList(TSDataType.TEXT, QUERY_ID); - } - - @Override - @After - public void tearDown() { - super.tearDown(); - } - - @Override - protected void generateData(int index) { - Binary value = BytesUtils.valueOf(String.valueOf(index)); - originalList.add(value); - testList.putBinary(index, value); - } - - @Override - protected void serializeAndDeserializeOnce() { - try { - testList.serialize(); - } catch (IOException e) { - fail(); - } - try { - testList.deserialize(); - } catch (IOException e) { - fail(); - } - int count = 0; - while (testList.hasCurrent()) { - assertEquals(count, testList.currentTime()); - assertEquals(originalList.get(count), testList.getBinary()); - testList.next(); - ++count; - } - assertEquals(ITERATION_TIMES, count); - } -} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableBooleanTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableBooleanTVListTest.java deleted file mode 100644 index e8ccfe9b6bae..000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableBooleanTVListTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure; - -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableBooleanTVList; -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableTVList; - -import org.apache.tsfile.enums.TSDataType; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class SerializableBooleanTVListTest extends SerializableTVListTest { - - private List originalList; - private SerializableBooleanTVList testList; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - originalList = new ArrayList<>(); - testList = - (SerializableBooleanTVList) - SerializableTVList.newSerializableTVList(TSDataType.BOOLEAN, QUERY_ID); - } - - @Override - @After - public void tearDown() { - super.tearDown(); - } - - @Override - protected void generateData(int index) { - boolean value = index % 2 == 0; - originalList.add(value); - testList.putBoolean(index, value); - } - - @Override - protected void serializeAndDeserializeOnce() { - try { - testList.serialize(); - } catch (IOException e) { - fail(); - } - try { - testList.deserialize(); - } catch (IOException e) { - fail(); - } - int count = 0; - while (testList.hasCurrent()) { - assertEquals(count, testList.currentTime()); - assertEquals(originalList.get(count), testList.getBoolean()); - testList.next(); - ++count; - } - assertEquals(ITERATION_TIMES, count); - } -} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableDoubleTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableDoubleTVListTest.java deleted file mode 100644 index d6fe5e73e9aa..000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableDoubleTVListTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure; - -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableDoubleTVList; -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableTVList; - -import org.apache.tsfile.enums.TSDataType; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class SerializableDoubleTVListTest extends SerializableTVListTest { - - private List originalList; - private SerializableDoubleTVList testList; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - originalList = new ArrayList<>(); - testList = - (SerializableDoubleTVList) - SerializableTVList.newSerializableTVList(TSDataType.DOUBLE, QUERY_ID); - } - - @Override - @After - public void tearDown() { - super.tearDown(); - } - - @Override - protected void generateData(int index) { - originalList.add((double) index); - testList.putDouble(index, index); - } - - @Override - protected void serializeAndDeserializeOnce() { - try { - testList.serialize(); - } catch (IOException e) { - fail(); - } - try { - testList.deserialize(); - } catch (IOException e) { - fail(); - } - int count = 0; - while (testList.hasCurrent()) { - assertEquals(count, testList.currentTime()); - assertEquals(originalList.get(count), testList.getDouble(), 0); - testList.next(); - ++count; - } - assertEquals(ITERATION_TIMES, count); - } -} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableFloatTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableFloatTVListTest.java deleted file mode 100644 index b3510c8156ad..000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableFloatTVListTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure; - -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableFloatTVList; -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableTVList; - -import org.apache.tsfile.enums.TSDataType; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class SerializableFloatTVListTest extends SerializableTVListTest { - - private List originalList; - private SerializableFloatTVList testList; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - originalList = new ArrayList<>(); - testList = - (SerializableFloatTVList) - SerializableTVList.newSerializableTVList(TSDataType.FLOAT, QUERY_ID); - } - - @Override - @After - public void tearDown() { - super.tearDown(); - } - - @Override - protected void generateData(int index) { - originalList.add((float) index); - testList.putFloat(index, index); - } - - @Override - protected void serializeAndDeserializeOnce() { - try { - testList.serialize(); - } catch (IOException e) { - fail(); - } - try { - testList.deserialize(); - } catch (IOException e) { - fail(); - } - int count = 0; - while (testList.hasCurrent()) { - assertEquals(count, testList.currentTime()); - assertEquals(originalList.get(count), testList.getFloat(), 0); - testList.next(); - ++count; - } - assertEquals(ITERATION_TIMES, count); - } -} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableIntTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableIntTVListTest.java deleted file mode 100644 index dc1b8635a9ce..000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableIntTVListTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure; - -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableIntTVList; -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableTVList; - -import org.apache.tsfile.enums.TSDataType; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class SerializableIntTVListTest extends SerializableTVListTest { - - private List originalList; - private SerializableIntTVList testList; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - originalList = new ArrayList<>(); - testList = - (SerializableIntTVList) - SerializableTVList.newSerializableTVList(TSDataType.INT32, QUERY_ID); - } - - @Override - @After - public void tearDown() { - super.tearDown(); - } - - @Override - protected void generateData(int index) { - originalList.add(index); - testList.putInt(index, index); - } - - @Override - protected void serializeAndDeserializeOnce() { - try { - testList.serialize(); - } catch (IOException e) { - fail(); - } - try { - testList.deserialize(); - } catch (IOException e) { - fail(); - } - int count = 0; - while (testList.hasCurrent()) { - assertEquals(count, testList.currentTime()); - assertEquals(originalList.get(count), testList.getInt(), 0); - testList.next(); - ++count; - } - assertEquals(ITERATION_TIMES, count); - } -} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableLongTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableLongTVListTest.java deleted file mode 100644 index 4ba1348d39ce..000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableLongTVListTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.datastructure; - -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableLongTVList; -import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableTVList; - -import org.apache.tsfile.enums.TSDataType; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class SerializableLongTVListTest extends SerializableTVListTest { - - private List originalList; - private SerializableLongTVList testList; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - originalList = new ArrayList<>(); - testList = - (SerializableLongTVList) - SerializableTVList.newSerializableTVList(TSDataType.INT64, QUERY_ID); - } - - @Override - @After - public void tearDown() { - super.tearDown(); - } - - @Override - protected void generateData(int index) { - originalList.add((long) index); - testList.putLong(index, index); - } - - @Override - protected void serializeAndDeserializeOnce() { - try { - testList.serialize(); - } catch (IOException e) { - fail(); - } - try { - testList.deserialize(); - } catch (IOException e) { - fail(); - } - int count = 0; - while (testList.hasCurrent()) { - assertEquals(count, testList.currentTime()); - assertEquals(originalList.get(count), testList.getLong(), 0); - testList.next(); - ++count; - } - assertEquals(ITERATION_TIMES, count); - } -} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableRowRecordListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableRowListTest.java similarity index 57% rename from iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableRowRecordListTest.java rename to iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableRowListTest.java index 272a1cf08f20..266623373e3d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableRowRecordListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableRowListTest.java @@ -19,11 +19,20 @@ package org.apache.iotdb.db.queryengine.transformation.datastructure; -import org.apache.iotdb.db.queryengine.transformation.datastructure.row.SerializableRowRecordList; +import org.apache.iotdb.db.queryengine.transformation.datastructure.row.SerializableRowList; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; +import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; +import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.junit.After; import org.junit.Before; @@ -36,7 +45,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -public class SerializableRowRecordListTest extends SerializableListTest { +public class SerializableRowListTest extends SerializableListTest { private static final TSDataType[] DATA_TYPES = { TSDataType.INT32, @@ -48,16 +57,14 @@ public class SerializableRowRecordListTest extends SerializableListTest { }; private List originalList; - private SerializableRowRecordList testList; + private SerializableRowList testList; @Override @Before public void setUp() throws Exception { super.setUp(); originalList = new ArrayList<>(); - testList = - SerializableRowRecordList.newSerializableRowRecordList( - QUERY_ID, DATA_TYPES, INTERNAL_ROW_RECORD_LIST_CAPACITY); + testList = SerializableRowList.construct(QUERY_ID, DATA_TYPES); } @Override @@ -69,21 +76,17 @@ public void tearDown() { @Test public void serializeAndDeserializeTest() { for (int i = 0; i < ITERATION_TIMES; ++i) { - generateData(i); - } - serializeAndDeserializeOnce(); - serializeAndDeserializeOnce(); - originalList.clear(); - testList.release(); - testList.init(); - for (int i = 0; i < ITERATION_TIMES; ++i) { - generateData(i); + RowRecord row = generateRowRecord(i); + originalList.add(row); } + Column[] columns = generateColumns(); + testList.putColumns(columns); + serializeAndDeserializeOnce(); serializeAndDeserializeOnce(); } - protected void generateData(int index) { + protected RowRecord generateRowRecord(int index) { RowRecord rowRecord = new RowRecord(index); for (TSDataType dataType : DATA_TYPES) { switch (dataType) { @@ -107,36 +110,64 @@ protected void generateData(int index) { break; } } - originalList.add(rowRecord); - testList.put(convertRowRecordToRowInObjects(rowRecord)); + + return rowRecord; } - protected Object[] convertRowRecordToRowInObjects(RowRecord rowRecord) { - Object[] rowInObjects = new Object[rowRecord.getFields().size() + 1]; - rowInObjects[rowRecord.getFields().size()] = rowRecord.getTimestamp(); - for (int i = 0; i < rowRecord.getFields().size(); ++i) { - switch (rowRecord.getFields().get(i).getDataType()) { - case INT32: - rowInObjects[i] = rowRecord.getFields().get(i).getIntV(); - break; - case INT64: - rowInObjects[i] = rowRecord.getFields().get(i).getLongV(); - break; - case FLOAT: - rowInObjects[i] = rowRecord.getFields().get(i).getFloatV(); - break; - case DOUBLE: - rowInObjects[i] = rowRecord.getFields().get(i).getDoubleV(); - break; - case BOOLEAN: - rowInObjects[i] = rowRecord.getFields().get(i).getBoolV(); - break; - case TEXT: - rowInObjects[i] = rowRecord.getFields().get(i).getBinaryV(); - break; - } + protected Column[] generateColumns() { + Column[] columns = new Column[DATA_TYPES.length + 1]; + + // Int columns + IntColumnBuilder intColumnBuilder = new IntColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + intColumnBuilder.writeInt(i); + } + columns[0] = intColumnBuilder.build(); + + // Long columns + LongColumnBuilder longColumnBuilder = new LongColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + longColumnBuilder.writeLong(i); + } + columns[1] = longColumnBuilder.build(); + + // Float columns + FloatColumnBuilder floatColumnBuilder = new FloatColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + floatColumnBuilder.writeFloat(i); + } + columns[2] = floatColumnBuilder.build(); + + // Double columns + DoubleColumnBuilder doubleColumnBuilder = new DoubleColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + doubleColumnBuilder.writeDouble(i); + } + columns[3] = doubleColumnBuilder.build(); + + // Boolean columns + BooleanColumnBuilder booleanColumnBuilder = new BooleanColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + booleanColumnBuilder.writeBoolean(i % 2 == 0); + } + columns[4] = booleanColumnBuilder.build(); + + // Binary columns + BinaryColumnBuilder binaryColumnBuilder = new BinaryColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + Binary binary = BytesUtils.valueOf(String.valueOf(i)); + binaryColumnBuilder.writeBinary(binary); + } + columns[5] = binaryColumnBuilder.build(); + + // The last time columns + TimeColumnBuilder timeColumnBuilder = new TimeColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; i++) { + timeColumnBuilder.writeLong(i); } - return rowInObjects; + columns[6] = timeColumnBuilder.build(); + + return columns; } protected void serializeAndDeserializeOnce() { @@ -155,7 +186,7 @@ protected void serializeAndDeserializeOnce() { for (int i = 0; i < testList.size(); ++i) { assertEquals(originalList.get(i).getTimestamp(), testList.getTime(i)); List originalFields = originalList.get(i).getFields(); - Object[] testFields = testList.getRowRecord(i); + Object[] testFields = testList.getRow(i); for (int j = 0; j < DATA_TYPES.length; ++j) { switch (DATA_TYPES[j]) { case INT32: diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableTVListTest.java index 11c028341614..0ee4dfa84a6f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableTVListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/datastructure/SerializableTVListTest.java @@ -19,20 +19,458 @@ package org.apache.iotdb.db.queryengine.transformation.datastructure; +import org.apache.iotdb.db.queryengine.transformation.datastructure.tv.SerializableTVList; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; +import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; +import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.junit.Before; import org.junit.Test; -public abstract class SerializableTVListTest extends SerializableListTest { +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class SerializableTVListTest extends SerializableListTest { + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Test + public void serializableBooleanTVListTest() { + List compared = new ArrayList<>(); + SerializableTVList target = SerializableTVList.construct(QUERY_ID); + + serializeAndDeserializeBooleanTVListTest(compared, target); + } + + @Test + public void serializableIntTVListTest() { + List compared = new ArrayList<>(); + SerializableTVList target = SerializableTVList.construct(QUERY_ID); + + serializeAndDeserializeIntTVListTest(compared, target); + } + + @Test + public void serializableLongTVListTest() { + List compared = new ArrayList<>(); + SerializableTVList target = SerializableTVList.construct(QUERY_ID); + + serializeAndDeserializeLongTVListTest(compared, target); + } + + @Test + public void serializableFloatTVListTest() { + List compared = new ArrayList<>(); + SerializableTVList target = SerializableTVList.construct(QUERY_ID); + + serializeAndDeserializeFloatTVListTest(compared, target); + } + + @Test + public void serializableDoubleTVListTest() { + List compared = new ArrayList<>(); + SerializableTVList target = SerializableTVList.construct(QUERY_ID); + + serializeAndDeserializeDoubleTVListTest(compared, target); + } @Test - public void serializeAndDeserializeTest() { + public void serializableBinaryTVListTest() { + List compared = new ArrayList<>(); + SerializableTVList target = SerializableTVList.construct(QUERY_ID); + + serializeAndDeserializeBinaryTVListTest(compared, target); + } + + private void serializeAndDeserializeBooleanTVListTest( + List compared, SerializableTVList target) { + generateBooleanData(compared, target); + + serializeAndDeserializeBooleanListOnce(compared, target); + serializeAndDeserializeBooleanListOnce(compared, target); + } + + private static void generateBooleanData(List compared, SerializableTVList target) { + ColumnBuilder timeBuilder = new TimeColumnBuilder(null, ITERATION_TIMES); + ColumnBuilder valueBuilder = new BooleanColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; ++i) { + compared.add(i % 2 == 0); + + timeBuilder.writeLong(i); + valueBuilder.writeBoolean(i % 2 == 0); + } + Column times = timeBuilder.build(); + Column values = valueBuilder.build(); + target.putColumns((TimeColumn) times, values); + } + + protected void serializeAndDeserializeBooleanListOnce( + List compared, SerializableTVList target) { + try { + target.serialize(); + } catch (IOException e) { + fail(); + } + try { + target.deserialize(); + } catch (IOException e) { + fail(); + } + checkBooleanListEquality(compared, target); + } + + private static void checkBooleanListEquality(List compared, SerializableTVList target) { + int count = 0; + ForwardIterator iterator = new ForwardIterator(target); + + while (iterator.hasNext()) { + long time = iterator.currentTime(); + boolean value = iterator.currentBoolean(); + + assertEquals(count, time); + assertEquals(compared.get(count), value); + + iterator.next(); + count++; + } + assertEquals(ITERATION_TIMES, count); + } + + private void serializeAndDeserializeIntTVListTest( + List compared, SerializableTVList target) { + generateIntData(compared, target); + + serializeAndDeserializeIntListOnce(compared, target); + serializeAndDeserializeIntListOnce(compared, target); + } + + private static void generateIntData(List compared, SerializableTVList target) { + ColumnBuilder timeBuilder = new TimeColumnBuilder(null, ITERATION_TIMES); + ColumnBuilder valueBuilder = new IntColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; ++i) { + compared.add(i); + + timeBuilder.writeLong(i); + valueBuilder.writeInt(i); + } + Column times = timeBuilder.build(); + Column values = valueBuilder.build(); + target.putColumns((TimeColumn) times, values); + } + + protected void serializeAndDeserializeIntListOnce( + List compared, SerializableTVList target) { + try { + target.serialize(); + } catch (IOException e) { + fail(); + } + try { + target.deserialize(); + } catch (IOException e) { + fail(); + } + checkIntListEquality(compared, target); + } + + private static void checkIntListEquality(List compared, SerializableTVList target) { + int count = 0; + ForwardIterator iterator = new ForwardIterator(target); + + while (iterator.hasNext()) { + long time = iterator.currentTime(); + int value = iterator.currentInt(); + + assertEquals(count, time); + assertEquals((int) compared.get(count), value); + + iterator.next(); + count++; + } + assertEquals(ITERATION_TIMES, count); + } + + private void serializeAndDeserializeLongTVListTest( + List compared, SerializableTVList target) { + generateLongData(compared, target); + + serializeAndDeserializeLongListOnce(compared, target); + serializeAndDeserializeLongListOnce(compared, target); + } + + private static void generateLongData(List compared, SerializableTVList target) { + ColumnBuilder timeBuilder = new TimeColumnBuilder(null, ITERATION_TIMES); + ColumnBuilder valueBuilder = new LongColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; ++i) { + compared.add((long) i); + + timeBuilder.writeLong(i); + valueBuilder.writeLong(i); + } + Column times = timeBuilder.build(); + Column values = valueBuilder.build(); + target.putColumns((TimeColumn) times, values); + } + + protected void serializeAndDeserializeLongListOnce( + List compared, SerializableTVList target) { + try { + target.serialize(); + } catch (IOException e) { + fail(); + } + try { + target.deserialize(); + } catch (IOException e) { + fail(); + } + checkLongListEquality(compared, target); + } + + private static void checkLongListEquality(List compared, SerializableTVList target) { + int count = 0; + ForwardIterator iterator = new ForwardIterator(target); + + while (iterator.hasNext()) { + long time = iterator.currentTime(); + long value = iterator.currentLong(); + + assertEquals(count, time); + assertEquals((long) compared.get(count), value); + + iterator.next(); + count++; + } + assertEquals(ITERATION_TIMES, count); + } + + private void serializeAndDeserializeFloatTVListTest( + List compared, SerializableTVList target) { + generateFloatData(compared, target); + + serializeAndDeserializeFloatListOnce(compared, target); + serializeAndDeserializeFloatListOnce(compared, target); + } + + private static void generateFloatData(List compared, SerializableTVList target) { + ColumnBuilder timeBuilder = new TimeColumnBuilder(null, ITERATION_TIMES); + ColumnBuilder valueBuilder = new FloatColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; ++i) { + compared.add((float) i); + + timeBuilder.writeLong(i); + valueBuilder.writeFloat((float) i); + } + Column times = timeBuilder.build(); + Column values = valueBuilder.build(); + target.putColumns((TimeColumn) times, values); + } + + protected void serializeAndDeserializeFloatListOnce( + List compared, SerializableTVList target) { + try { + target.serialize(); + } catch (IOException e) { + fail(); + } + try { + target.deserialize(); + } catch (IOException e) { + fail(); + } + checkFloatListEquality(compared, target); + } + + private static void checkFloatListEquality(List compared, SerializableTVList target) { + int count = 0; + ForwardIterator iterator = new ForwardIterator(target); + + while (iterator.hasNext()) { + long time = iterator.currentTime(); + float value = iterator.currentFloat(); + + assertEquals(count, time); + assertEquals(compared.get(count), value, 0); + + iterator.next(); + count++; + } + assertEquals(ITERATION_TIMES, count); + } + + private void serializeAndDeserializeDoubleTVListTest( + List compared, SerializableTVList target) { + generateDoubleData(compared, target); + + serializeAndDeserializeDoubleListOnce(compared, target); + serializeAndDeserializeDoubleListOnce(compared, target); + } + + private static void generateDoubleData(List compared, SerializableTVList target) { + ColumnBuilder timeBuilder = new TimeColumnBuilder(null, ITERATION_TIMES); + ColumnBuilder valueBuilder = new DoubleColumnBuilder(null, ITERATION_TIMES); for (int i = 0; i < ITERATION_TIMES; ++i) { - generateData(i); + compared.add((double) i); + + timeBuilder.writeLong(i); + valueBuilder.writeDouble(i); + } + Column times = timeBuilder.build(); + Column values = valueBuilder.build(); + target.putColumns((TimeColumn) times, values); + } + + protected void serializeAndDeserializeDoubleListOnce( + List compared, SerializableTVList target) { + try { + target.serialize(); + } catch (IOException e) { + fail(); + } + try { + target.deserialize(); + } catch (IOException e) { + fail(); + } + checkDoubleListEquality(compared, target); + } + + private static void checkDoubleListEquality(List compared, SerializableTVList target) { + int count = 0; + ForwardIterator iterator = new ForwardIterator(target); + + while (iterator.hasNext()) { + long time = iterator.currentTime(); + double value = iterator.currentDouble(); + + assertEquals(count, time); + assertEquals(compared.get(count), value, 0); + + iterator.next(); + count++; } - serializeAndDeserializeOnce(); - serializeAndDeserializeOnce(); + assertEquals(ITERATION_TIMES, count); } - protected abstract void generateData(int index); + private void serializeAndDeserializeBinaryTVListTest( + List compared, SerializableTVList target) { + generateBinaryData(compared, target); - protected abstract void serializeAndDeserializeOnce(); + serializeAndDeserializeBinaryListOnce(compared, target); + serializeAndDeserializeBinaryListOnce(compared, target); + } + + private static void generateBinaryData(List compared, SerializableTVList target) { + ColumnBuilder timeBuilder = new TimeColumnBuilder(null, ITERATION_TIMES); + ColumnBuilder valueBuilder = new BinaryColumnBuilder(null, ITERATION_TIMES); + for (int i = 0; i < ITERATION_TIMES; ++i) { + Binary value = BytesUtils.valueOf(String.valueOf(i)); + compared.add(value); + + timeBuilder.writeLong(i); + valueBuilder.writeBinary(value); + } + Column times = timeBuilder.build(); + Column values = valueBuilder.build(); + target.putColumns((TimeColumn) times, values); + } + + protected void serializeAndDeserializeBinaryListOnce( + List compared, SerializableTVList target) { + try { + target.serialize(); + } catch (IOException e) { + fail(); + } + try { + target.deserialize(); + } catch (IOException e) { + fail(); + } + checkBinaryListEquality(compared, target); + } + + private static void checkBinaryListEquality(List compared, SerializableTVList target) { + int count = 0; + ForwardIterator iterator = new ForwardIterator(target); + while (iterator.hasNext()) { + long time = iterator.currentTime(); + Binary value = iterator.currentBinary(); + + assertEquals(count, time); + assertEquals(compared.get(count), value); + + iterator.next(); + count++; + } + assertEquals(ITERATION_TIMES, count); + } + + private static class ForwardIterator { + int index; + int offset; + List timeColumns; + List valueColumns; + + ForwardIterator(SerializableTVList tvList) { + timeColumns = tvList.getTimeColumns(); + valueColumns = tvList.getValueColumns(); + } + + public boolean hasNext() { + return index < timeColumns.size(); + } + + public long currentTime() { + return timeColumns.get(index).getLong(offset); + } + + public boolean currentBoolean() { + return valueColumns.get(index).getBoolean(offset); + } + + public int currentInt() { + return valueColumns.get(index).getInt(offset); + } + + public long currentLong() { + return valueColumns.get(index).getLong(offset); + } + + public float currentFloat() { + return valueColumns.get(index).getFloat(offset); + } + + public double currentDouble() { + return valueColumns.get(index).getDouble(offset); + } + + public Binary currentBinary() { + return valueColumns.get(index).getBinary(offset); + } + + public void next() { + if (offset == timeColumns.get(index).getPositionCount() - 1) { + offset = 0; + index++; + } else { + offset++; + } + } + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/MasterRepairUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/MasterRepairUtil.java index 9f99a8671b82..f8367822cad7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/MasterRepairUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/MasterRepairUtil.java @@ -44,7 +44,7 @@ public MasterRepairUtil(int columnCnt, long omega, double eta, int k) { this.k = k; } - public boolean isNullRow(Row row) { + public boolean isNullRow(Row row) throws IOException { boolean flag = true; for (int i = 0; i < row.size(); i++) { if (!row.isNull(i)) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/series/util/ConsecutiveUtil.java b/library-udf/src/main/java/org/apache/iotdb/library/series/util/ConsecutiveUtil.java index 6b3fbeeccbc2..2c9ea997c5fe 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/series/util/ConsecutiveUtil.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/series/util/ConsecutiveUtil.java @@ -81,7 +81,7 @@ public long getLast() { } /** check Null values. */ - public boolean check(Row row) { + public boolean check(Row row) throws IOException { for (int i = 0; i < row.size(); i++) { if (row.isNull(i)) { return true;