From 4ad3b872b5868845ec7a89ab9e56b80c084cb040 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Sun, 17 Nov 2024 16:17:38 +0800 Subject: [PATCH 01/12] Support uncorrelated scalar subquery in where clause without UT/IT and code cleaning --- .../org/apache/iotdb/SessionExampleTest.java | 921 ++++++++++++++++++ iotdb-client/client-py/setup.py | 64 ++ .../SimpleNestedLoopCrossJoinOperator.java | 243 +++++ .../plan/planner/TableOperatorGenerator.java | 19 +- .../plan/planner/plan/node/PlanVisitor.java | 17 + .../plan/relational/analyzer/Analysis.java | 39 + .../analyzer/ExpressionAnalyzer.java | 21 + .../planner/ExpressionExtractor.java | 22 +- .../plan/relational/planner/QueryPlanner.java | 19 +- .../relational/planner/RelationPlanner.java | 2 +- .../relational/planner/SubqueryPlanner.java | 348 ++++++- .../TableDistributedPlanGenerator.java | 56 +- .../planner/ir/ExpressionRewriter.java | 12 + .../planner/ir/ExpressionTreeRewriter.java | 36 + .../rule/PruneCorrelatedJoinColumns.java | 122 +++ .../rule/PruneCorrelatedJoinCorrelation.java | 66 ++ .../rule/PruneEnforceSingleRowColumns.java | 42 + .../RemoveRedundantEnforceSingleRowNode.java | 45 + .../TransformUncorrelatedSubqueryToJoin.java | 133 +++ .../relational/planner/node/ApplyNode.java | 242 +++++ .../planner/node/CorrelatedJoinNode.java | 166 ++++ .../planner/node/EnforceSingleRowNode.java | 80 ++ .../relational/planner/node/JoinNode.java | 4 + .../relational/planner/node/Patterns.java | 142 ++- .../planner/optimizations/Cardinality.java | 56 ++ .../CheckSubqueryNodesAreRewritten.java | 65 ++ .../optimizations/LogicalOptimizeFactory.java | 15 + .../PushAggregationIntoTableScan.java | 4 + .../PushPredicateIntoTableScan.java | 58 +- .../optimizations/QueryCardinalityUtil.java | 208 ++++ .../planner/optimizations/SymbolMapper.java | 30 +- .../UnaliasSymbolReferences.java | 133 +++ .../plan/relational/sql/ast/SingleColumn.java | 4 + 33 files changed, 3248 insertions(+), 186 deletions(-) create mode 100644 example/session/src/main/java/org/apache/iotdb/SessionExampleTest.java create mode 100644 iotdb-client/client-py/setup.py create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinColumns.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinCorrelation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneEnforceSingleRowColumns.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveRedundantEnforceSingleRowNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedSubqueryToJoin.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ApplyNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CorrelatedJoinNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/EnforceSingleRowNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Cardinality.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CheckSubqueryNodesAreRewritten.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExampleTest.java b/example/session/src/main/java/org/apache/iotdb/SessionExampleTest.java new file mode 100644 index 000000000000..4214b2e73008 --- /dev/null +++ b/example/session/src/main/java/org/apache/iotdb/SessionExampleTest.java @@ -0,0 +1,921 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb; + +import org.apache.iotdb.common.rpc.thrift.TAggregationType; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.SessionDataSet.DataIterator; +import org.apache.iotdb.isession.template.Template; +import org.apache.iotdb.isession.util.Version; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.template.MeasurementNode; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +@SuppressWarnings({"squid:S106", "squid:S1144", "squid:S125"}) +public class SessionExampleTest { + + private static Session session; + private static Session sessionEnableRedirect; + private static final String ROOT_SG1_D1_S1 = "root.sg1.d1.s1"; + private static final String ROOT_SG1_D1_S2 = "root.sg1.d1.s2"; + private static final String ROOT_SG1_D1_S3 = "root.sg1.d1.s3"; + private static final String ROOT_SG1_D1_S4 = "root.sg1.d1.s4"; + private static final String ROOT_SG1_D1_S5 = "root.sg1.d1.s5"; + private static final String ROOT_SG1_D1 = "root.sg1.d1"; + private static final String ROOT_SG1 = "root.sg1"; + private static final String LOCAL_HOST = "127.0.0.1"; + public static final String SELECT_D1 = "select * from root.sg1.d1"; + + private static Random random = new Random(); + + public static void main(String[] args) + throws IoTDBConnectionException, StatementExecutionException { + session = + new Session.Builder() + .host(LOCAL_HOST) + .port(6667) + .username("root") + .password("root") + .sqlDialect("table") + .database("test") + .version(Version.V_1_0) + .build(); + session.open(false); + String sql = "select sum(s1) from (select * from table1)"; + // String sql = "select sum(s1) from table1"; + + // set session fetchSize + session.setFetchSize(10000); + long loop = 20; + long start = System.currentTimeMillis(); + for (int i = 0; i < loop; i++) { + try (SessionDataSet result = session.executeQueryStatement(sql)) { + while (result.hasNext()) { + result.next(); + } + } + } + long end = System.currentTimeMillis(); + System.out.println("Avg execution time is : " + (end - start) / loop + "ms"); + + // try { + // session.createDatabase("root.test"); + // } catch (StatementExecutionException e) { + // if (e.getStatusCode() != TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode()) { + // throw e; + // } + // } + // + // List schemaList = new ArrayList<>(); + // schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + // schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); + // schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); + // final List columnTypes = + // Arrays.asList( + // Tablet.ColumnType.MEASUREMENT, + // Tablet.ColumnType.MEASUREMENT, + // Tablet.ColumnType.MEASUREMENT); + // Tablet tablet = new Tablet("table1", schemaList, columnTypes, 100000); + // long timestamp = 0; + // for (long row = 0; row < 100000; row++) { + // int rowIndex = tablet.rowSize++; + // tablet.addTimestamp(rowIndex, timestamp); + // for (int s = 0; s < 3; s++) { + // long value = timestamp; + // tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + // } + // timestamp++; + // } + // try { + // session.insertRelationalTablet(tablet); + // } catch (Exception e) { + // } finally { + // session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='false'"); + // } + + session.close(); + } + + private static void createAndDropContinuousQueries() + throws StatementExecutionException, IoTDBConnectionException { + session.executeNonQueryStatement( + "CREATE CONTINUOUS QUERY cq1 " + + "BEGIN SELECT max_value(s1) INTO temperature_max FROM root.sg1.* " + + "GROUP BY time(10s) END"); + session.executeNonQueryStatement( + "CREATE CONTINUOUS QUERY cq2 " + + "BEGIN SELECT count(s2) INTO temperature_cnt FROM root.sg1.* " + + "GROUP BY time(10s), level=1 END"); + session.executeNonQueryStatement( + "CREATE CONTINUOUS QUERY cq3 " + + "RESAMPLE EVERY 20s FOR 20s " + + "BEGIN SELECT avg(s3) INTO temperature_avg FROM root.sg1.* " + + "GROUP BY time(10s), level=1 END"); + session.executeNonQueryStatement("DROP CONTINUOUS QUERY cq1"); + session.executeNonQueryStatement("DROP CONTINUOUS QUERY cq2"); + session.executeNonQueryStatement("DROP CONTINUOUS QUERY cq3"); + } + + private static void createTimeseries() + throws IoTDBConnectionException, StatementExecutionException { + + if (!session.checkTimeseriesExists(ROOT_SG1_D1_S1)) { + session.createTimeseries( + ROOT_SG1_D1_S1, TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); + } + if (!session.checkTimeseriesExists(ROOT_SG1_D1_S2)) { + session.createTimeseries( + ROOT_SG1_D1_S2, TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); + } + if (!session.checkTimeseriesExists(ROOT_SG1_D1_S3)) { + session.createTimeseries( + ROOT_SG1_D1_S3, TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); + } + + // create timeseries with tags and attributes + if (!session.checkTimeseriesExists(ROOT_SG1_D1_S4)) { + Map tags = new HashMap<>(); + tags.put("tag1", "v1"); + Map attributes = new HashMap<>(); + attributes.put("description", "v1"); + session.createTimeseries( + ROOT_SG1_D1_S4, + TSDataType.INT64, + TSEncoding.RLE, + CompressionType.SNAPPY, + null, + tags, + attributes, + "temperature"); + } + + // create timeseries with SDT property, SDT will take place when flushing + if (!session.checkTimeseriesExists(ROOT_SG1_D1_S5)) { + // COMPDEV is required + // COMPMAXTIME and COMPMINTIME are optional and their unit is ms + Map props = new HashMap<>(); + props.put("LOSS", "sdt"); + props.put("COMPDEV", "0.01"); + props.put("COMPMINTIME", "2"); + props.put("COMPMAXTIME", "10"); + session.createTimeseries( + ROOT_SG1_D1_S5, + TSDataType.INT64, + TSEncoding.RLE, + CompressionType.SNAPPY, + props, + null, + null, + null); + } + } + + private static void createMultiTimeseries() + throws IoTDBConnectionException, StatementExecutionException { + + if (!session.checkTimeseriesExists("root.sg1.d2.s1") + && !session.checkTimeseriesExists("root.sg1.d2.s2")) { + List paths = new ArrayList<>(); + paths.add("root.sg1.d2.s1"); + paths.add("root.sg1.d2.s2"); + List tsDataTypes = new ArrayList<>(); + tsDataTypes.add(TSDataType.INT64); + tsDataTypes.add(TSDataType.INT64); + List tsEncodings = new ArrayList<>(); + tsEncodings.add(TSEncoding.RLE); + tsEncodings.add(TSEncoding.RLE); + List compressionTypes = new ArrayList<>(); + compressionTypes.add(CompressionType.SNAPPY); + compressionTypes.add(CompressionType.SNAPPY); + + List> tagsList = new ArrayList<>(); + Map tags = new HashMap<>(); + tags.put("unit", "kg"); + tagsList.add(tags); + tagsList.add(tags); + + List> attributesList = new ArrayList<>(); + Map attributes = new HashMap<>(); + attributes.put("minValue", "1"); + attributes.put("maxValue", "100"); + attributesList.add(attributes); + attributesList.add(attributes); + + List alias = new ArrayList<>(); + alias.add("weight1"); + alias.add("weight2"); + + session.createMultiTimeseries( + paths, tsDataTypes, tsEncodings, compressionTypes, null, tagsList, attributesList, alias); + } + } + + private static void createTemplate() + throws IoTDBConnectionException, StatementExecutionException, IOException { + + Template template = new Template("template1", false); + MeasurementNode mNodeS1 = + new MeasurementNode("s1", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); + MeasurementNode mNodeS2 = + new MeasurementNode("s2", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); + MeasurementNode mNodeS3 = + new MeasurementNode("s3", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); + + template.addToTemplate(mNodeS1); + template.addToTemplate(mNodeS2); + template.addToTemplate(mNodeS3); + + session.createSchemaTemplate(template); + session.setSchemaTemplate("template1", "root.sg1"); + } + + private static void insertRecord() throws IoTDBConnectionException, StatementExecutionException { + String deviceId = ROOT_SG1_D1; + List measurements = new ArrayList<>(); + List types = new ArrayList<>(); + measurements.add("s1"); + measurements.add("s2"); + measurements.add("s3"); + types.add(TSDataType.INT64); + types.add(TSDataType.INT64); + types.add(TSDataType.INT64); + + for (long time = 0; time < 100; time++) { + List values = new ArrayList<>(); + values.add(1L); + values.add(2L); + values.add(3L); + session.insertRecord(deviceId, time, measurements, types, values); + } + } + + private static void insertRecord4Redirect() + throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < 6; i++) { + for (int j = 0; j < 2; j++) { + String deviceId = "root.redirect" + i + ".d" + j; + List measurements = new ArrayList<>(); + measurements.add("s1"); + measurements.add("s2"); + measurements.add("s3"); + List types = new ArrayList<>(); + types.add(TSDataType.INT64); + types.add(TSDataType.INT64); + types.add(TSDataType.INT64); + + for (long time = 0; time < 5; time++) { + List values = new ArrayList<>(); + values.add(1L + time); + values.add(2L + time); + values.add(3L + time); + session.insertRecord(deviceId, time, measurements, types, values); + } + } + } + } + + private static void insertStrRecord() + throws IoTDBConnectionException, StatementExecutionException { + String deviceId = ROOT_SG1_D1; + List measurements = new ArrayList<>(); + measurements.add("s1"); + measurements.add("s2"); + measurements.add("s3"); + + for (long time = 0; time < 10; time++) { + List values = new ArrayList<>(); + values.add("1"); + values.add("2"); + values.add("3"); + session.insertRecord(deviceId, time, measurements, values); + } + } + + private static void insertRecordInObject() + throws IoTDBConnectionException, StatementExecutionException { + String deviceId = ROOT_SG1_D1; + List measurements = new ArrayList<>(); + List types = new ArrayList<>(); + measurements.add("s1"); + measurements.add("s2"); + measurements.add("s3"); + types.add(TSDataType.INT64); + types.add(TSDataType.INT64); + types.add(TSDataType.INT64); + + for (long time = 0; time < 100; time++) { + session.insertRecord(deviceId, time, measurements, types, 1L, 1L, 1L); + } + } + + private static void insertRecords() throws IoTDBConnectionException, StatementExecutionException { + String deviceId = ROOT_SG1_D1; + List measurements = new ArrayList<>(); + measurements.add("s1"); + measurements.add("s2"); + measurements.add("s3"); + List deviceIds = new ArrayList<>(); + List> measurementsList = new ArrayList<>(); + List> valuesList = new ArrayList<>(); + List timestamps = new ArrayList<>(); + List> typesList = new ArrayList<>(); + + for (long time = 0; time < 500; time++) { + List values = new ArrayList<>(); + List types = new ArrayList<>(); + values.add(1L); + values.add(2L); + values.add(3L); + types.add(TSDataType.INT64); + types.add(TSDataType.INT64); + types.add(TSDataType.INT64); + + deviceIds.add(deviceId); + measurementsList.add(measurements); + valuesList.add(values); + typesList.add(types); + timestamps.add(time); + if (time != 0 && time % 100 == 0) { + session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); + deviceIds.clear(); + measurementsList.clear(); + valuesList.clear(); + typesList.clear(); + timestamps.clear(); + } + } + + session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); + } + + /** + * insert the data of a device. For each timestamp, the number of measurements is the same. + * + *

Users need to control the count of Tablet and write a batch when it reaches the maxBatchSize + */ + private static void insertTablet() throws IoTDBConnectionException, StatementExecutionException { + /* + * A Tablet example: + * device1 + * time s1, s2, s3 + * 1, 1, 1, 1 + * 2, 2, 2, 2 + * 3, 3, 3, 3 + */ + // The schema of measurements of one device + // only measurementId and data type in MeasurementSchema take effects in Tablet + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); + + Tablet tablet = new Tablet(ROOT_SG1_D1, schemaList, 100); + + // Method 1 to add tablet data + long timestamp = System.currentTimeMillis(); + + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + for (int s = 0; s < 3; s++) { + long value = random.nextLong(); + tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + } + if (tablet.rowSize == tablet.getMaxRowNumber()) { + session.insertTablet(tablet, true); + tablet.reset(); + } + timestamp++; + } + + if (tablet.rowSize != 0) { + session.insertTablet(tablet); + tablet.reset(); + } + + // Method 2 to add tablet data + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; + + for (long time = 0; time < 100; time++) { + int row = tablet.rowSize++; + timestamps[row] = time; + for (int i = 0; i < 3; i++) { + long[] sensor = (long[]) values[i]; + sensor[row] = i; + } + if (tablet.rowSize == tablet.getMaxRowNumber()) { + session.insertTablet(tablet, true); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session.insertTablet(tablet); + tablet.reset(); + } + } + + private static void insertTabletWithNullValues() + throws IoTDBConnectionException, StatementExecutionException { + /* + * A Tablet example: + * device1 + * time s1, s2, s3 + * 1, null, 1, 1 + * 2, 2, null, 2 + * 3, 3, 3, null + */ + // The schema of measurements of one device + // only measurementId and data type in MeasurementSchema take effects in Tablet + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); + + Tablet tablet = new Tablet(ROOT_SG1_D1, schemaList, 100); + + // Method 1 to add tablet data + insertTablet1(schemaList, tablet); + + // Method 2 to add tablet data + insertTablet2(schemaList, tablet); + } + + private static void insertTablet1(List schemaList, Tablet tablet) + throws IoTDBConnectionException, StatementExecutionException { + tablet.initBitMaps(); + + long timestamp = System.currentTimeMillis(); + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + for (int s = 0; s < 3; s++) { + long value = random.nextLong(); + // mark null value + if (row % 3 == s) { + tablet.bitMaps[s].mark((int) row); + } + tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + } + if (tablet.rowSize == tablet.getMaxRowNumber()) { + session.insertTablet(tablet, true); + tablet.reset(); + } + timestamp++; + } + + if (tablet.rowSize != 0) { + session.insertTablet(tablet); + tablet.reset(); + } + } + + private static void insertTablet2(List schemaList, Tablet tablet) + throws IoTDBConnectionException, StatementExecutionException { + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; + BitMap[] bitMaps = new BitMap[schemaList.size()]; + for (int s = 0; s < 3; s++) { + bitMaps[s] = new BitMap(tablet.getMaxRowNumber()); + } + tablet.bitMaps = bitMaps; + + for (long time = 0; time < 100; time++) { + int row = tablet.rowSize++; + timestamps[row] = time; + for (int i = 0; i < 3; i++) { + long[] sensor = (long[]) values[i]; + // mark null value + if (row % 3 == i) { + bitMaps[i].mark(row); + } + sensor[row] = i; + } + if (tablet.rowSize == tablet.getMaxRowNumber()) { + session.insertTablet(tablet, true); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session.insertTablet(tablet); + tablet.reset(); + } + } + + private static void insertTablets() throws IoTDBConnectionException, StatementExecutionException { + // The schema of measurements of one device + // only measurementId and data type in MeasurementSchema take effects in Tablet + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); + + Tablet tablet1 = new Tablet(ROOT_SG1_D1, schemaList, 100); + Tablet tablet2 = new Tablet("root.sg1.d2", schemaList, 100); + Tablet tablet3 = new Tablet("root.sg1.d3", schemaList, 100); + + Map tabletMap = new HashMap<>(); + tabletMap.put(ROOT_SG1_D1, tablet1); + tabletMap.put("root.sg1.d2", tablet2); + tabletMap.put("root.sg1.d3", tablet3); + + // Method 1 to add tablet data + long timestamp = System.currentTimeMillis(); + for (long row = 0; row < 100; row++) { + int row1 = tablet1.rowSize++; + int row2 = tablet2.rowSize++; + int row3 = tablet3.rowSize++; + tablet1.addTimestamp(row1, timestamp); + tablet2.addTimestamp(row2, timestamp); + tablet3.addTimestamp(row3, timestamp); + for (int i = 0; i < 3; i++) { + long value = random.nextLong(); + tablet1.addValue(schemaList.get(i).getMeasurementId(), row1, value); + tablet2.addValue(schemaList.get(i).getMeasurementId(), row2, value); + tablet3.addValue(schemaList.get(i).getMeasurementId(), row3, value); + } + if (tablet1.rowSize == tablet1.getMaxRowNumber()) { + session.insertTablets(tabletMap, true); + tablet1.reset(); + tablet2.reset(); + tablet3.reset(); + } + timestamp++; + } + + if (tablet1.rowSize != 0) { + session.insertTablets(tabletMap, true); + tablet1.reset(); + tablet2.reset(); + tablet3.reset(); + } + + // Method 2 to add tablet data + long[] timestamps1 = tablet1.timestamps; + Object[] values1 = tablet1.values; + long[] timestamps2 = tablet2.timestamps; + Object[] values2 = tablet2.values; + long[] timestamps3 = tablet3.timestamps; + Object[] values3 = tablet3.values; + + for (long time = 0; time < 100; time++) { + int row1 = tablet1.rowSize++; + int row2 = tablet2.rowSize++; + int row3 = tablet3.rowSize++; + timestamps1[row1] = time; + timestamps2[row2] = time; + timestamps3[row3] = time; + for (int i = 0; i < 3; i++) { + long[] sensor1 = (long[]) values1[i]; + sensor1[row1] = i; + long[] sensor2 = (long[]) values2[i]; + sensor2[row2] = i; + long[] sensor3 = (long[]) values3[i]; + sensor3[row3] = i; + } + if (tablet1.rowSize == tablet1.getMaxRowNumber()) { + session.insertTablets(tabletMap, true); + + tablet1.reset(); + tablet2.reset(); + tablet3.reset(); + } + } + + if (tablet1.rowSize != 0) { + session.insertTablets(tabletMap, true); + tablet1.reset(); + tablet2.reset(); + tablet3.reset(); + } + } + + /** + * This example shows how to insert data of TSDataType.TEXT. You can use the session interface to + * write data of String type or Binary type. + */ + private static void insertText() throws IoTDBConnectionException, StatementExecutionException { + String device = "root.sg1.text"; + // the first data is String type and the second data is Binary type + List datas = Arrays.asList("String", new Binary("Binary", TSFileConfig.STRING_CHARSET)); + // insertRecord example + for (int i = 0; i < datas.size(); i++) { + // write data of String type or Binary type + session.insertRecord( + device, + i, + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.TEXT), + datas.get(i)); + } + + // insertTablet example + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s2", TSDataType.TEXT)); + Tablet tablet = new Tablet(device, schemaList, 100); + for (int i = 0; i < datas.size(); i++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, i); + // write data of String type or Binary type + tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, datas.get(i)); + } + session.insertTablet(tablet); + try (SessionDataSet dataSet = session.executeQueryStatement("select s1, s2 from " + device)) { + System.out.println(dataSet.getColumnNames()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + } + + private static void selectInto() throws IoTDBConnectionException, StatementExecutionException { + session.executeNonQueryStatement( + "select s1, s2, s3 into into_s1, into_s2, into_s3 from root.sg1.d1"); + + try (SessionDataSet dataSet = + session.executeQueryStatement("select into_s1, into_s2, into_s3 from root.sg1.d1")) { + System.out.println(dataSet.getColumnNames()); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + } + + private static void deleteData() throws IoTDBConnectionException, StatementExecutionException { + String path = ROOT_SG1_D1_S1; + long deleteTime = 99; + session.deleteData(path, deleteTime); + } + + private static void deleteTimeseries() + throws IoTDBConnectionException, StatementExecutionException { + List paths = new ArrayList<>(); + paths.add(ROOT_SG1_D1_S1); + paths.add(ROOT_SG1_D1_S2); + paths.add(ROOT_SG1_D1_S3); + session.deleteTimeseries(paths); + } + + private static void query() throws IoTDBConnectionException, StatementExecutionException { + try (SessionDataSet dataSet = session.executeQueryStatement(SELECT_D1)) { + System.out.println(dataSet.getColumnNames()); + dataSet.setFetchSize(1024); // default is 10000 + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + } + + private static void query4Redirect() + throws IoTDBConnectionException, StatementExecutionException { + String selectPrefix = "select * from root.redirect"; + for (int i = 0; i < 6; i++) { + try (SessionDataSet dataSet = + sessionEnableRedirect.executeQueryStatement(selectPrefix + i + ".d1")) { + + System.out.println(dataSet.getColumnNames()); + dataSet.setFetchSize(1024); // default is 10000 + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + } + + for (int i = 0; i < 6; i++) { + try (SessionDataSet dataSet = + sessionEnableRedirect.executeQueryStatement( + selectPrefix + i + ".d1 where time >= 1 and time < 10")) { + + System.out.println(dataSet.getColumnNames()); + dataSet.setFetchSize(1024); // default is 10000 + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + } + + for (int i = 0; i < 6; i++) { + try (SessionDataSet dataSet = + sessionEnableRedirect.executeQueryStatement( + selectPrefix + i + ".d1 where time >= 1 and time < 10 align by device")) { + + System.out.println(dataSet.getColumnNames()); + dataSet.setFetchSize(1024); // default is 10000 + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + } + + for (int i = 0; i < 6; i++) { + try (SessionDataSet dataSet = + sessionEnableRedirect.executeQueryStatement( + selectPrefix + + i + + ".d1 where time >= 1 and time < 10 and root.redirect" + + i + + ".d1.s1 > 1")) { + System.out.println(dataSet.getColumnNames()); + dataSet.setFetchSize(1024); // default is 10000 + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + } + } + + private static void queryWithTimeout() + throws IoTDBConnectionException, StatementExecutionException { + try (SessionDataSet dataSet = session.executeQueryStatement(SELECT_D1, 2000)) { + System.out.println(dataSet.getColumnNames()); + dataSet.setFetchSize(1024); // default is 10000 + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + } + + private static void rawDataQuery() throws IoTDBConnectionException, StatementExecutionException { + List paths = new ArrayList<>(); + paths.add(ROOT_SG1_D1_S1); + paths.add(ROOT_SG1_D1_S2); + paths.add(ROOT_SG1_D1_S3); + long startTime = 10L; + long endTime = 200L; + long timeOut = 60000; + + try (SessionDataSet dataSet = session.executeRawDataQuery(paths, startTime, endTime, timeOut)) { + + System.out.println(dataSet.getColumnNames()); + dataSet.setFetchSize(1024); + while (dataSet.hasNext()) { + System.out.println(dataSet.next()); + } + } + } + + private static void lastDataQuery() throws IoTDBConnectionException, StatementExecutionException { + List paths = new ArrayList<>(); + paths.add(ROOT_SG1_D1_S1); + paths.add(ROOT_SG1_D1_S2); + paths.add(ROOT_SG1_D1_S3); + try (SessionDataSet sessionDataSet = session.executeLastDataQuery(paths, 3, 60000)) { + System.out.println(sessionDataSet.getColumnNames()); + sessionDataSet.setFetchSize(1024); + while (sessionDataSet.hasNext()) { + System.out.println(sessionDataSet.next()); + } + } + } + + private static void fastLastDataQueryForOneDevice() + throws IoTDBConnectionException, StatementExecutionException { + System.out.println("-------fastLastQuery------"); + List paths = new ArrayList<>(); + paths.add("s1"); + paths.add("s2"); + paths.add("s3"); + try (SessionDataSet sessionDataSet = + sessionEnableRedirect.executeLastDataQueryForOneDevice( + ROOT_SG1, ROOT_SG1_D1, paths, true)) { + System.out.println(sessionDataSet.getColumnNames()); + sessionDataSet.setFetchSize(1024); + while (sessionDataSet.hasNext()) { + System.out.println(sessionDataSet.next()); + } + } + } + + private static void aggregationQuery() + throws IoTDBConnectionException, StatementExecutionException { + List paths = new ArrayList<>(); + paths.add(ROOT_SG1_D1_S1); + paths.add(ROOT_SG1_D1_S2); + paths.add(ROOT_SG1_D1_S3); + + List aggregations = new ArrayList<>(); + aggregations.add(TAggregationType.COUNT); + aggregations.add(TAggregationType.SUM); + aggregations.add(TAggregationType.MAX_VALUE); + try (SessionDataSet sessionDataSet = session.executeAggregationQuery(paths, aggregations)) { + System.out.println(sessionDataSet.getColumnNames()); + sessionDataSet.setFetchSize(1024); + while (sessionDataSet.hasNext()) { + System.out.println(sessionDataSet.next()); + } + } + } + + private static void groupByQuery() throws IoTDBConnectionException, StatementExecutionException { + List paths = new ArrayList<>(); + paths.add(ROOT_SG1_D1_S1); + paths.add(ROOT_SG1_D1_S2); + paths.add(ROOT_SG1_D1_S3); + + List aggregations = new ArrayList<>(); + aggregations.add(TAggregationType.COUNT); + aggregations.add(TAggregationType.SUM); + aggregations.add(TAggregationType.MAX_VALUE); + try (SessionDataSet sessionDataSet = + session.executeAggregationQuery(paths, aggregations, 0, 100, 10, 20)) { + System.out.println(sessionDataSet.getColumnNames()); + sessionDataSet.setFetchSize(1024); + while (sessionDataSet.hasNext()) { + System.out.println(sessionDataSet.next()); + } + } + } + + private static void queryByIterator() + throws IoTDBConnectionException, StatementExecutionException { + try (SessionDataSet dataSet = session.executeQueryStatement(SELECT_D1)) { + + DataIterator iterator = dataSet.iterator(); + System.out.println(dataSet.getColumnNames()); + dataSet.setFetchSize(1024); // default is 10000 + while (iterator.next()) { + StringBuilder builder = new StringBuilder(); + // get time + builder.append(iterator.getLong(1)).append(","); + // get second column + if (!iterator.isNull(2)) { + builder.append(iterator.getLong(2)).append(","); + } else { + builder.append("null").append(","); + } + + // get third column + if (!iterator.isNull(ROOT_SG1_D1_S2)) { + builder.append(iterator.getLong(ROOT_SG1_D1_S2)).append(","); + } else { + builder.append("null").append(","); + } + + // get forth column + if (!iterator.isNull(4)) { + builder.append(iterator.getLong(4)).append(","); + } else { + builder.append("null").append(","); + } + + // get fifth column + if (!iterator.isNull(ROOT_SG1_D1_S4)) { + builder.append(iterator.getObject(ROOT_SG1_D1_S4)); + } else { + builder.append("null"); + } + + System.out.println(builder); + } + } + } + + private static void nonQuery() throws IoTDBConnectionException, StatementExecutionException { + session.executeNonQueryStatement("insert into root.sg1.d1(timestamp,s1) values(200, 1)"); + } + + private static void setTimeout() throws IoTDBConnectionException { + try (Session tempSession = new Session(LOCAL_HOST, 6667, "root", "root", 10000, 20000)) { + tempSession.setQueryTimeout(60000); + } + } +} diff --git a/iotdb-client/client-py/setup.py b/iotdb-client/client-py/setup.py new file mode 100644 index 000000000000..731e9a14a62a --- /dev/null +++ b/iotdb-client/client-py/setup.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import setuptools +import io + + +try: + with io.open("README.md", encoding="utf-8") as f: + long_description = f.read() +except FileNotFoundError: + long_description = "" + + +print(long_description) + +setuptools.setup( + name="apache-iotdb", # Replace with your own username + version="1.3.3.dev0", + author=" Apache Software Foundation", + author_email="dev@iotdb.apache.org", + description="Apache IoTDB client API", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/apache/iotdb", + packages=setuptools.find_packages(), + install_requires=[ + "thrift>=0.14.1", + "pandas>=1.3.5", + "numpy>=1.21.4", + "sqlalchemy<1.5,>=1.4", + "sqlalchemy-utils>=0.37.8", + ], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + python_requires=">=3.7", + license="Apache License, Version 2.0", + website="https://iotdb.apache.org", + entry_points={ + "sqlalchemy.dialects": [ + "iotdb = iotdb.sqlalchemy.IoTDBDialect:IoTDBDialect", + ], + }, +) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java new file mode 100644 index 000000000000..e4847e12fce5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join; + +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.queryengine.execution.operator.AbstractOperator; +import org.apache.iotdb.db.queryengine.execution.operator.Operator; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableInnerJoinOperator.buildResultTsBlock; + +/** + * This Operator is used to implement the simple nested loop join algorithm for Cartesian product. + * It is used to join two tables, one is the probe table and the other is the build table. For now, + * the build table is assumed to be small enough to be cached in memory.(Produced by a scalar + * subquery.) Scalar subquery is always the right child of PlanNode, so we can use right child of + * JoinNode as the build table. + */ +public class SimpleNestedLoopCrossJoinOperator extends AbstractOperator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(SimpleNestedLoopCrossJoinOperator.class); + + private final Operator probeSource; + + // cache the result of buildSource, for now, we assume that the buildChild produces a small number + // of TsBlocks + private final Operator buildSource; + + private final List buildBlocks; + + private final TsBlockBuilder resultBuilder; + + private final MemoryReservationManager memoryReservationManager; + + private final int[] leftOutputSymbolIdx; + + private final int[] rightOutputSymbolIdx; + + private TsBlock cachedProbeBlock; + + private int probeIndex; + + private boolean buildFinished = false; + + public SimpleNestedLoopCrossJoinOperator( + OperatorContext operatorContext, + Operator probeSource, + Operator buildSource, + int[] leftOutputSymbolIdx, + int[] rightOutputSymbolIdx, + List dataTypes) { + this.operatorContext = operatorContext; + this.probeSource = probeSource; + this.buildSource = buildSource; + this.leftOutputSymbolIdx = leftOutputSymbolIdx; + this.rightOutputSymbolIdx = rightOutputSymbolIdx; + this.buildBlocks = new ArrayList<>(); + this.resultBuilder = new TsBlockBuilder(dataTypes); + this.memoryReservationManager = + operatorContext + .getDriverContext() + .getFragmentInstanceContext() + .getMemoryReservationContext(); + } + + @Override + public TsBlock next() throws Exception { + if (retainedTsBlock != null) { + getResultFromRetainedTsBlock(); + } + resultBuilder.reset(); + // start stopwatch + long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS); + long start = System.nanoTime(); + if (!buildFinished) { + TsBlock block = buildSource.next(); + if (block != null && !block.isEmpty()) { + buildBlocks.add(block); + memoryReservationManager.reserveMemoryCumulatively(block.getRetainedSizeInBytes()); + } + if (!buildSource.hasNext()) { + buildFinished = true; + } + // probeSource could still be blocked by now, so we need to check it again + return null; + } + cachedProbeBlock = cachedProbeBlock == null ? probeSource.next() : cachedProbeBlock; + if (cachedProbeBlock == null || cachedProbeBlock.isEmpty()) { + // TsBlock returned by probeSource is null or empty, we need to wait for another round + cachedProbeBlock = null; + return null; + } + while (probeIndex < cachedProbeBlock.getPositionCount() + && System.nanoTime() - start < maxRuntime) { + for (TsBlock buildBlock : buildBlocks) { + for (int i = 0; i < buildBlock.getPositionCount(); i++) { + appendValueToResult(probeIndex, buildBlock); + } + } + probeIndex++; + } + if (probeIndex == cachedProbeBlock.getPositionCount()) { + probeIndex = 0; + cachedProbeBlock = null; + } + if (resultBuilder.isEmpty()) { + return null; + } + + resultTsBlock = buildResultTsBlock(resultBuilder); + return checkTsBlockSizeAndGetResult(); + } + + private void appendValueToResult(int probeIndex, TsBlock buildBlock) { + for (int i = 0; i < buildBlock.getPositionCount(); i++) { + for (int j = 0; j < leftOutputSymbolIdx.length; j++) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(j); + if (cachedProbeBlock.getColumn(leftOutputSymbolIdx[j]).isNull(probeIndex)) { + columnBuilder.appendNull(); + } else { + columnBuilder.write(cachedProbeBlock.getColumn(leftOutputSymbolIdx[j]), probeIndex); + } + } + for (int j = 0; j < rightOutputSymbolIdx.length; j++) { + ColumnBuilder columnBuilder = + resultBuilder.getColumnBuilder(leftOutputSymbolIdx.length + j); + if (buildBlock.getColumn(rightOutputSymbolIdx[j]).isNull(i)) { + columnBuilder.appendNull(); + } else { + columnBuilder.write(buildBlock.getColumn(rightOutputSymbolIdx[j]), i); + } + } + } + resultBuilder.declarePosition(); + } + + @Override + public boolean hasNext() throws Exception { + if (retainedTsBlock != null) { + return true; + } + if (!buildFinished) { + return buildSource.hasNext(); + } + return probeSource.hasNext(); + } + + @Override + public ListenableFuture isBlocked() { + if (buildFinished) { + return probeSource.isBlocked(); + } + return buildSource.isBlocked(); + } + + @Override + public void close() throws Exception { + if (probeSource != null) { + probeSource.close(); + } + if (buildSource != null) { + buildSource.close(); + } + for (TsBlock block : buildBlocks) { + memoryReservationManager.releaseMemoryCumulatively(block.getRetainedSizeInBytes()); + } + buildBlocks.clear(); + cachedProbeBlock = null; + resultTsBlock = null; + retainedTsBlock = null; + } + + @Override + public boolean isFinished() throws Exception { + if (retainedTsBlock != null) { + return false; + } + + return (cachedProbeBlock != null && !cachedProbeBlock.isEmpty()) + && probeSource.isFinished() + && buildFinished; + } + + @Override + public long calculateMaxPeekMemory() { + return Math.max( + Math.max( + probeSource.calculateMaxPeekMemoryWithCounter(), + buildSource.calculateMaxPeekMemoryWithCounter()), + calculateRetainedSizeAfterCallingNext() + calculateMaxReturnSize()); + } + + @Override + public long calculateMaxReturnSize() { + return maxReturnSize; + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + return probeSource.calculateRetainedSizeAfterCallingNext() + + buildSource.calculateRetainedSizeAfterCallingNext() + // cachedProbeBlock + one build block assumed + + maxReturnSize * 2; + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(probeSource) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(buildSource) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) + + resultBuilder.getRetainedSizeInBytes(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 8d67baf49a5b..85d0133d3654 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -60,6 +60,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.gapfill.GapFillWGroupWoMoOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.gapfill.GapFillWoGroupWMoOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.gapfill.GapFillWoGroupWoMoOperator; +import org.apache.iotdb.db.queryengine.execution.operator.process.join.SimpleNestedLoopCrossJoinOperator; import org.apache.iotdb.db.queryengine.execution.operator.schema.CountMergeOperator; import org.apache.iotdb.db.queryengine.execution.operator.schema.SchemaCountOperator; import org.apache.iotdb.db.queryengine.execution.operator.schema.SchemaQueryScanOperator; @@ -1176,21 +1177,31 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { Operator leftChild = node.getLeftChild().accept(this, context); Operator rightChild = node.getRightChild().accept(this, context); - int leftTimeColumnPosition = - node.getLeftChild().getOutputSymbols().indexOf(node.getCriteria().get(0).getLeft()); int[] leftOutputSymbolIdx = new int[node.getLeftOutputSymbols().size()]; for (int i = 0; i < leftOutputSymbolIdx.length; i++) { leftOutputSymbolIdx[i] = node.getLeftChild().getOutputSymbols().indexOf(node.getLeftOutputSymbols().get(i)); } - int rightTimeColumnPosition = - node.getRightChild().getOutputSymbols().indexOf(node.getCriteria().get(0).getRight()); int[] rightOutputSymbolIdx = new int[node.getRightOutputSymbols().size()]; for (int i = 0; i < rightOutputSymbolIdx.length; i++) { rightOutputSymbolIdx[i] = node.getRightChild().getOutputSymbols().indexOf(node.getRightOutputSymbols().get(i)); } + // cross join does not need time column + if (node.isCrossJoin()) { + return new SimpleNestedLoopCrossJoinOperator( + operatorContext, + leftChild, + rightChild, + leftOutputSymbolIdx, + rightOutputSymbolIdx, + dataTypes); + } + int leftTimeColumnPosition = + node.getLeftChild().getOutputSymbols().indexOf(node.getCriteria().get(0).getLeft()); + int rightTimeColumnPosition = + node.getRightChild().getOutputSymbols().indexOf(node.getCriteria().get(0).getRight()); if (requireNonNull(node.getJoinType()) == JoinNode.JoinType.INNER) { return new TableInnerJoinOperator( operatorContext, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 23f7749bfd10..97064ac1b94d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -614,6 +614,23 @@ public R visitFilter( return visitSingleChildProcess(node, context); } + public R visitApply( + org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode node, C context) { + return visitTwoChildProcess(node, context); + } + + public R visitEnforceSingleRow( + org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode node, + C context) { + return visitSingleChildProcess(node, context); + } + + public R visitCorrelatedJoin( + org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode node, + C context) { + return visitTwoChildProcess(node, context); + } + public R visitTableScan(TableScanNode node, C context) { return visitPlan(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 76b80d842eb3..379977fc1c8d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -140,6 +140,8 @@ public class Analysis implements IAnalysis { private final Map, Expression> joins = new LinkedHashMap<>(); private final Map, JoinUsingAnalysis> joinUsing = new LinkedHashMap<>(); private final Map, SubqueryAnalysis> subQueries = new LinkedHashMap<>(); + private final Map, PredicateCoercions> predicateCoercions = + new LinkedHashMap<>(); private final Map, TableEntry> tables = new LinkedHashMap<>(); @@ -672,6 +674,14 @@ public Map getTableColumnSchema(QualifiedObjectName qualif return tableColumnSchemas.get(qualifiedObjectName); } + public void addPredicateCoercions(Map, PredicateCoercions> coercions) { + predicateCoercions.putAll(coercions); + } + + public PredicateCoercions getPredicateCoercions(Expression expression) { + return predicateCoercions.get(NodeRef.of(expression)); + } + public boolean hasValueFilter() { return hasValueFilter; } @@ -1096,6 +1106,35 @@ public List getQuantifiedComparisonSubqueries() } } + /** + * Analysis for predicates such as x IN (subquery) or x = SOME (subquery) + * + */ + public static class PredicateCoercions { + private final Type valueType; + private final Optional valueCoercion; + private final Optional subqueryCoercion; + + public PredicateCoercions( + Type valueType, Optional valueCoercion, Optional subqueryCoercion) { + this.valueType = requireNonNull(valueType, "valueType is null"); + this.valueCoercion = requireNonNull(valueCoercion, "valueCoercion is null"); + this.subqueryCoercion = requireNonNull(subqueryCoercion, "subqueryCoercion is null"); + } + + public Type getValueType() { + return valueType; + } + + public Optional getValueCoercion() { + return valueCoercion; + } + + public Optional getSubqueryCoercion() { + return subqueryCoercion; + } + } + public static class FillAnalysis { protected final FillPolicy fillMethod; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index ecfb0faaa574..404c88b2e1e8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -137,6 +137,8 @@ public class ExpressionAnalyzer { private final Set> existsSubqueries = new LinkedHashSet<>(); private final Set> subqueryInPredicates = new LinkedHashSet<>(); + private final Map, Analysis.PredicateCoercions> predicateCoercions = + new LinkedHashMap<>(); private final Map, ResolvedField> columnReferences = new LinkedHashMap<>(); private final Map, Type> expressionTypes = new LinkedHashMap<>(); @@ -236,6 +238,10 @@ public Set> getSubqueryInPredicates() { return unmodifiableSet(subqueryInPredicates); } + public Map, Analysis.PredicateCoercions> getPredicateCoercions() { + return unmodifiableMap(predicateCoercions); + } + public Map, ResolvedField> getColumnReferences() { return unmodifiableMap(columnReferences); } @@ -1005,6 +1011,20 @@ private Type analyzePredicateWithSubquery( valueRowType, subqueryType)); } + Optional valueCoercion = Optional.empty(); + // if (!valueRowType.equals(commonType.get())) { + // valueCoercion = commonType; + // } + + Optional subQueryCoercion = Optional.empty(); + // if (!subqueryType.equals(commonType.get())) { + // subQueryCoercion = commonType; + // } + + predicateCoercions.put( + NodeRef.of(node), + new Analysis.PredicateCoercions(valueRowType, valueCoercion, subQueryCoercion)); + return subqueryType; } @@ -1466,6 +1486,7 @@ private static void updateAnalysis( analysis.addColumnReferences(analyzer.getColumnReferences()); analysis.addTableColumnReferences( accessControl, session.getIdentity(), analyzer.getTableColumnReferences()); + analysis.addPredicateCoercions(analyzer.getPredicateCoercions()); } public static ExpressionAnalyzer createConstantAnalyzer( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java index 928c4f5c17a2..59d77280a3c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java @@ -15,8 +15,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.SimplePlanVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@ -73,13 +75,12 @@ public Void visitPlan(PlanNode node, Void context) { return null; } - /*@Override - public Void visitGroupReference(GroupReference node, Void context) - { - return lookup.resolve(node).accept(this, context); + @Override + public Void visitGroupReference(GroupReference node, Void context) { + return lookup.resolve(node).accept(this, context); } - @Override + /*@Override public Void visitAggregation(AggregationNode node, Void context) { for (Aggregation aggregation : node.getAggregations().values()) { @@ -100,14 +101,13 @@ public Void visitProject(ProjectNode node, Void context) { return super.visitProject(node, context); } - /*@Override - public Void visitJoin(JoinNode node, Void context) - { - node.getFilter().ifPresent(consumer); - return super.visitJoin(node, context); + @Override + public Void visitJoin(JoinNode node, Void context) { + node.getFilter().ifPresent(consumer); + return super.visitJoin(node, context); } - @Override + /*@Override public Void visitValues(ValuesNode node, Void context) { node.getRows().ifPresent(list -> list.forEach(consumer)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index cfc19a3e7da0..a21af187bb37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -159,7 +159,7 @@ public RelationPlan plan(Query query) { public RelationPlan plan(QuerySpecification node) { PlanBuilder builder = planFrom(node); - builder = filter(builder, analysis.getWhere(node)); + builder = filter(builder, analysis.getWhere(node), node); Expression wherePredicate = null; if (builder.getRoot() instanceof FilterNode) { wherePredicate = ((FilterNode) builder.getRoot()).getPredicate(); @@ -171,7 +171,7 @@ public RelationPlan plan(QuerySpecification node) { timeColumnForGapFill = builder.translate((Expression) gapFillColumn.getChildren().get(2)); } builder = aggregate(builder, node); - builder = filter(builder, analysis.getHaving(node)); + builder = filter(builder, analysis.getHaving(node), node); if (gapFillColumn != null) { if (wherePredicate == null) { @@ -187,13 +187,13 @@ public RelationPlan plan(QuerySpecification node) { } List selectExpressions = analysis.getSelectExpressions(node); + List expressions = + selectExpressions.stream() + .map(Analysis.SelectExpression::getExpression) + .collect(toImmutableList()); + builder = subqueryPlanner.handleSubqueries(builder, expressions, analysis.getSubqueries(node)); if (hasExpressionsToUnfold(selectExpressions)) { - List expressions = - selectExpressions.stream() - .map(Analysis.SelectExpression::getExpression) - .collect(toImmutableList()); - // pre-project the folded expressions to preserve any non-deterministic semantics of functions // that might be referenced builder = builder.appendProjections(expressions, symbolAllocator, queryContext); @@ -323,13 +323,12 @@ private PlanBuilder planFrom(QuerySpecification node) { } } - private PlanBuilder filter(PlanBuilder subPlan, Expression predicate) { + private PlanBuilder filter(PlanBuilder subPlan, Expression predicate, Node node) { if (predicate == null) { return subPlan; } - // planBuilder = subqueryPlanner.handleSubqueries(subPlan, predicate, - // analysis.getSubqueries(node)); + subPlan = subqueryPlanner.handleSubqueries(subPlan, predicate, analysis.getSubqueries(node)); return subPlan.withNewRoot( new FilterNode( queryIdAllocator.genPlanNodeId(), subPlan.getRoot(), subPlan.rewrite(predicate))); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index f1fec1c7ff7f..96dd7e2f05c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -621,7 +621,7 @@ protected RelationPlan visitValues(Values node, Void context) { @Override protected RelationPlan visitSubqueryExpression(SubqueryExpression node, Void context) { - throw new IllegalStateException("SubqueryExpression is not supported in current version."); + return process(node.getQuery(), context); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java index 573f0cb335d8..82f90ac36da9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java @@ -16,20 +16,31 @@ import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Field; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.RelationType; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Scope; import org.apache.iotdb.db.queryengine.plan.relational.planner.QueryPlanner.PlanAndMappings; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Cast; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -47,10 +58,15 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.collect.Streams.stream; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static org.apache.iotdb.db.queryengine.plan.relational.planner.PlanBuilder.newPlanBuilder; import static org.apache.iotdb.db.queryengine.plan.relational.planner.ScopeAware.scopeAwareKey; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression.Quantifier.ALL; +import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toSqlType; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; class SubqueryPlanner { @@ -95,18 +111,31 @@ public PlanBuilder handleSubqueries( public PlanBuilder handleSubqueries( PlanBuilder builder, Expression expression, Analysis.SubqueryAnalysis subqueries) { - /*for (Cluster cluster : cluster(builder.getScope(), selectSubqueries(builder, expression, subqueries.getInPredicatesSubqueries()))) { - builder = planInPredicate(builder, cluster, subqueries); + for (Cluster cluster : + cluster( + builder.getScope(), + selectSubqueries(builder, expression, subqueries.getInPredicatesSubqueries()))) { + builder = planInPredicate(builder, cluster, subqueries); } - for (Cluster cluster : cluster(builder.getScope(), selectSubqueries(builder, expression, subqueries.getSubqueries()))) { - builder = planScalarSubquery(builder, cluster); + for (Cluster cluster : + cluster( + builder.getScope(), + selectSubqueries(builder, expression, subqueries.getSubqueries()))) { + builder = planScalarSubquery(builder, cluster); } - for (Cluster cluster : cluster(builder.getScope(), selectSubqueries(builder, expression, subqueries.getExistsSubqueries()))) { - builder = planExists(builder, cluster); + for (Cluster cluster : + cluster( + builder.getScope(), + selectSubqueries(builder, expression, subqueries.getExistsSubqueries()))) { + builder = planExists(builder, cluster); + } + for (Cluster cluster : + cluster( + builder.getScope(), + selectSubqueries( + builder, expression, subqueries.getQuantifiedComparisonSubqueries()))) { + builder = planQuantifiedComparison(builder, cluster, subqueries); } - for (Cluster cluster : cluster(builder.getScope(), selectSubqueries(builder, expression, subqueries.getQuantifiedComparisonSubqueries()))) { - builder = planQuantifiedComparison(builder, cluster, subqueries); - }*/ return builder; } @@ -156,6 +185,142 @@ private Collection> cluster(Scope scope, List< .collect(toImmutableList()); } + private PlanBuilder planInPredicate( + PlanBuilder subPlan, Cluster cluster, Analysis.SubqueryAnalysis subqueries) { + // Plan one of the predicates from the cluster + InPredicate predicate = cluster.getRepresentative(); + + Expression value = predicate.getValue(); + SubqueryExpression subquery = (SubqueryExpression) predicate.getValueList(); + Symbol output = symbolAllocator.newSymbol(predicate, BOOLEAN); + + subPlan = handleSubqueries(subPlan, value, subqueries); + subPlan = + planInPredicate( + subPlan, value, subquery, output, predicate, analysis.getPredicateCoercions(predicate)); + + return new PlanBuilder( + subPlan + .getTranslations() + .withAdditionalMappings(mapAll(cluster, subPlan.getScope(), output)), + subPlan.getRoot()); + } + + /** + * Plans a correlated subquery for value IN (subQuery) + * + * @param originalExpression the original expression from which the IN predicate was derived. Used + * for subsequent translations. + */ + private PlanBuilder planInPredicate( + PlanBuilder subPlan, + Expression value, + SubqueryExpression subquery, + Symbol output, + Expression originalExpression, + Analysis.PredicateCoercions predicateCoercions) { + PlanAndMappings subqueryPlan = + planSubquery(subquery, predicateCoercions.getSubqueryCoercion(), subPlan.getTranslations()); + PlanAndMappings valuePlan = + planValue( + subPlan, + value, + predicateCoercions.getValueType(), + predicateCoercions.getValueCoercion()); + + return new PlanBuilder( + valuePlan.getSubPlan().getTranslations(), + new ApplyNode( + idAllocator.genPlanNodeId(), + valuePlan.getSubPlan().getRoot(), + subqueryPlan.getSubPlan().getRoot(), + ImmutableMap.of( + output, new ApplyNode.In(valuePlan.get(value), subqueryPlan.get(subquery))), + valuePlan.getSubPlan().getRoot().getOutputSymbols(), + originalExpression)); + } + + private PlanBuilder planScalarSubquery(PlanBuilder subPlan, Cluster cluster) { + // Plan one of the predicates from the cluster + SubqueryExpression scalarSubquery = cluster.getRepresentative(); + + RelationPlan relationPlan = planSubquery(scalarSubquery, subPlan.getTranslations()); + PlanBuilder subqueryPlan = newPlanBuilder(relationPlan, analysis); + + PlanNode root = new EnforceSingleRowNode(idAllocator.genPlanNodeId(), subqueryPlan.getRoot()); + + Type type = analysis.getType(scalarSubquery); + RelationType descriptor = relationPlan.getDescriptor(); + List fieldMappings = relationPlan.getFieldMappings(); + Symbol column; + if (descriptor.getVisibleFieldCount() > 1) { + column = symbolAllocator.newSymbol("row", type); + + ImmutableList.Builder fields = ImmutableList.builder(); + for (int i = 0; i < descriptor.getAllFieldCount(); i++) { + Field field = descriptor.getFieldByIndex(i); + if (!field.isHidden()) { + fields.add(fieldMappings.get(i).toSymbolReference()); + } + } + + Expression expression = new Cast(new Row(fields.build()), toSqlType(type)); + + root = new ProjectNode(idAllocator.genPlanNodeId(), root, Assignments.of(column, expression)); + } else { + column = getOnlyElement(fieldMappings); + } + + return appendCorrelatedJoin( + subPlan, + root, + scalarSubquery.getQuery(), + // Scalar subquery always contains EnforceSingleRowNode. Therefore, it's guaranteed + // that subquery will return single row. Hence, correlated join can be of INNER type. + JoinNode.JoinType.INNER, + TRUE_LITERAL, + mapAll(cluster, subPlan.getScope(), column)); + } + + public PlanBuilder appendCorrelatedJoin( + PlanBuilder subPlan, + PlanNode subquery, + Query query, + JoinNode.JoinType type, + Expression filterCondition, + Map, Symbol> mappings) { + return new PlanBuilder( + subPlan.getTranslations().withAdditionalMappings(mappings), + new CorrelatedJoinNode( + idAllocator.genPlanNodeId(), + subPlan.getRoot(), + subquery, + subPlan.getRoot().getOutputSymbols(), + type, + filterCondition, + query)); + } + + private PlanBuilder planExists(PlanBuilder subPlan, Cluster cluster) { + // Plan one of the predicates from the cluster + ExistsPredicate existsPredicate = cluster.getRepresentative(); + + Expression subquery = existsPredicate.getSubquery(); + Symbol exists = symbolAllocator.newSymbol("exists", BOOLEAN); + + return new PlanBuilder( + subPlan + .getTranslations() + .withAdditionalMappings(mapAll(cluster, subPlan.getScope(), exists)), + new ApplyNode( + idAllocator.genPlanNodeId(), + subPlan.getRoot(), + planSubquery(subquery, subPlan.getTranslations()).getRoot(), + ImmutableMap.of(exists, new ApplyNode.Exists()), + subPlan.getRoot().getOutputSymbols(), + subquery)); + } + private RelationPlan planSubquery(Expression subquery, TranslationMap outerContext) { return new RelationPlanner( analysis, @@ -167,6 +332,103 @@ private RelationPlan planSubquery(Expression subquery, TranslationMap outerConte .process(subquery, null); } + private PlanBuilder planQuantifiedComparison( + PlanBuilder subPlan, + Cluster cluster, + Analysis.SubqueryAnalysis subqueries) { + // Plan one of the predicates from the cluster + QuantifiedComparisonExpression quantifiedComparison = cluster.getRepresentative(); + + ComparisonExpression.Operator operator = quantifiedComparison.getOperator(); + QuantifiedComparisonExpression.Quantifier quantifier = quantifiedComparison.getQuantifier(); + Expression value = quantifiedComparison.getValue(); + SubqueryExpression subquery = (SubqueryExpression) quantifiedComparison.getSubquery(); + + subPlan = handleSubqueries(subPlan, value, subqueries); + + Symbol output = symbolAllocator.newSymbol(quantifiedComparison, BOOLEAN); + + Analysis.PredicateCoercions predicateCoercions = + analysis.getPredicateCoercions(quantifiedComparison); + + switch (operator) { + case EQUAL: + switch (quantifier) { + case ALL: + subPlan = + planQuantifiedComparison( + subPlan, operator, quantifier, value, subquery, output, predicateCoercions); + return new PlanBuilder( + subPlan + .getTranslations() + .withAdditionalMappings( + ImmutableMap.of( + scopeAwareKey(quantifiedComparison, analysis, subPlan.getScope()), + output)), + subPlan.getRoot()); + case ANY: + case SOME: + // A = ANY B <=> A IN B + subPlan = + planInPredicate( + subPlan, value, subquery, output, quantifiedComparison, predicateCoercions); + return new PlanBuilder( + subPlan + .getTranslations() + .withAdditionalMappings(mapAll(cluster, subPlan.getScope(), output)), + subPlan.getRoot()); + default: + throw new IllegalArgumentException(); + } + case NOT_EQUAL: + switch (quantifier) { + case ALL: + // A <> ALL B <=> !(A IN B) + return addNegation( + planInPredicate( + subPlan, value, subquery, output, quantifiedComparison, predicateCoercions), + cluster, + output); + case ANY: + case SOME: + // A <> ANY B <=> min B <> max B || A <> min B <=> !(min B = max B && A = min B) <=> !(A + // = ALL B) + // "A <> ANY B" is equivalent to "NOT (A = ALL B)" so add a rewrite for the initial + // quantifiedComparison to notAll + return addNegation( + planQuantifiedComparison( + subPlan, + ComparisonExpression.Operator.EQUAL, + ALL, + value, + subquery, + output, + predicateCoercions), + cluster, + output); + default: + throw new IllegalArgumentException(); + } + case LESS_THAN: + case LESS_THAN_OR_EQUAL: + case GREATER_THAN: + case GREATER_THAN_OR_EQUAL: + subPlan = + planQuantifiedComparison( + subPlan, operator, quantifier, value, subquery, output, predicateCoercions); + return new PlanBuilder( + subPlan + .getTranslations() + .withAdditionalMappings(mapAll(cluster, subPlan.getScope(), output)), + subPlan.getRoot()); + // Cannot be used with quantified comparison + case IS_DISTINCT_FROM: + default: + throw new IllegalArgumentException( + format("Unexpected quantified comparison: '%s %s'", operator.getValue(), quantifier)); + } + } + /** * Adds a negation of the given input and remaps the provided expression to the negated expression */ @@ -187,6 +449,74 @@ private PlanBuilder addNegation( .build())); } + private PlanBuilder planQuantifiedComparison( + PlanBuilder subPlan, + ComparisonExpression.Operator operator, + QuantifiedComparisonExpression.Quantifier quantifier, + Expression value, + Expression subquery, + Symbol assignment, + Analysis.PredicateCoercions predicateCoercions) { + PlanAndMappings subqueryPlan = + planSubquery(subquery, predicateCoercions.getSubqueryCoercion(), subPlan.getTranslations()); + PlanAndMappings valuePlan = + planValue( + subPlan, + value, + predicateCoercions.getValueType(), + predicateCoercions.getValueCoercion()); + + return new PlanBuilder( + valuePlan.getSubPlan().getTranslations(), + new ApplyNode( + idAllocator.genPlanNodeId(), + valuePlan.getSubPlan().getRoot(), + subqueryPlan.getSubPlan().getRoot(), + ImmutableMap.of( + assignment, + new ApplyNode.QuantifiedComparison( + mapOperator(operator), + mapQuantifier(quantifier), + valuePlan.get(value), + subqueryPlan.get(subquery))), + valuePlan.getSubPlan().getRoot().getOutputSymbols(), + subquery)); + } + + private static ApplyNode.Quantifier mapQuantifier( + QuantifiedComparisonExpression.Quantifier quantifier) { + switch (quantifier) { + case ALL: + return ApplyNode.Quantifier.ALL; + case ANY: + return ApplyNode.Quantifier.ANY; + case SOME: + return ApplyNode.Quantifier.SOME; + default: + throw new IllegalArgumentException(); + } + } + + private static ApplyNode.Operator mapOperator(ComparisonExpression.Operator operator) { + switch (operator) { + case EQUAL: + return ApplyNode.Operator.EQUAL; + case NOT_EQUAL: + return ApplyNode.Operator.NOT_EQUAL; + case LESS_THAN: + return ApplyNode.Operator.LESS_THAN; + case LESS_THAN_OR_EQUAL: + return ApplyNode.Operator.LESS_THAN_OR_EQUAL; + case GREATER_THAN: + return ApplyNode.Operator.GREATER_THAN; + case GREATER_THAN_OR_EQUAL: + return ApplyNode.Operator.GREATER_THAN_OR_EQUAL; + case IS_DISTINCT_FROM: + default: + throw new IllegalArgumentException(); + } + } + private PlanAndMappings planValue( PlanBuilder subPlan, Expression value, Type actualType, Optional coercion) { subPlan = subPlan.appendProjections(ImmutableList.of(value), symbolAllocator, plannerContext); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 41714bdeb696..7482c28b14de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -147,11 +147,6 @@ public List visitOutput(OutputNode node, PlanContext context) { nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); } - if (childrenNodes.size() == 1) { - node.setChild(childrenNodes.get(0)); - return Collections.singletonList(node); - } - node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); return Collections.singletonList(node); } @@ -167,11 +162,6 @@ public List visitFill(FillNode node, PlanContext context) { nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); } - if (childrenNodes.size() == 1) { - node.setChild(childrenNodes.get(0)); - return Collections.singletonList(node); - } - node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); return Collections.singletonList(node); } @@ -185,11 +175,6 @@ public List visitGapFill(GapFillNode node, PlanContext context) { nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); } - if (childrenNodes.size() == 1) { - node.setChild(childrenNodes.get(0)); - return Collections.singletonList(node); - } - node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); return Collections.singletonList(node); } @@ -202,11 +187,6 @@ public List visitLimit(LimitNode node, PlanContext context) { nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); } - if (childrenNodes.size() == 1) { - node.setChild(childrenNodes.get(0)); - return Collections.singletonList(node); - } - // push down LimitNode in distributed plan optimize rule node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); return Collections.singletonList(node); @@ -220,11 +200,6 @@ public List visitOffset(OffsetNode node, PlanContext context) { nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); } - if (childrenNodes.size() == 1) { - node.setChild(childrenNodes.get(0)); - return Collections.singletonList(node); - } - node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); return Collections.singletonList(node); } @@ -434,18 +409,23 @@ public List visitFilter(FilterNode node, PlanContext context) { @Override public List visitJoin(JoinNode node, PlanContext context) { - // child of JoinNode must be SortNode, so after rewritten, the child must be MergeSortNode or - // SortNode - List leftChildrenNodes = node.getLeftChild().accept(this, context); - checkArgument( - leftChildrenNodes.size() == 1, "The size of left children node of JoinNode should be 1"); - node.setLeftChild(leftChildrenNodes.get(0)); + List leftChildrenNodes = node.getLeftChild().accept(this, context); List rightChildrenNodes = node.getRightChild().accept(this, context); - checkArgument( - rightChildrenNodes.size() == 1, "The size of right children node of JoinNode should be 1"); - node.setRightChild(rightChildrenNodes.get(0)); - + if (!node.isCrossJoin()) { + // child of JoinNode(excluding CrossJoin) must be SortNode, so after rewritten, the child must + // be MergeSortNode or + // SortNode + checkArgument( + leftChildrenNodes.size() == 1, "The size of left children node of JoinNode should be 1"); + checkArgument( + rightChildrenNodes.size() == 1, + "The size of right children node of JoinNode should be 1"); + } + // For CrossJoinNode, we need to merge children nodes(It's safe for other JoinNodes here since + // the size of their children is always 1.) + node.setLeftChild(mergeChildrenViaCollectOrMergeSort(null, leftChildrenNodes)); + node.setRightChild(mergeChildrenViaCollectOrMergeSort(null, rightChildrenNodes)); return Collections.singletonList(node); } @@ -693,6 +673,12 @@ private static OrderingScheme constructOrderingSchema(List symbols) { private PlanNode mergeChildrenViaCollectOrMergeSort( OrderingScheme childOrdering, List childrenNodes) { + checkArgument(!childrenNodes.isEmpty(), "childrenNodes should not be empty"); + + if (childrenNodes.size() == 1) { + return childrenNodes.get(0); + } + PlanNode firstChild = childrenNodes.get(0); // children has sort property, use MergeSort to merge children diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java index 5bf6f8ef230f..e11dedc15bdd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CoalesceExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; @@ -46,6 +47,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause; @@ -146,6 +148,16 @@ public Expression rewriteInListExpression( return rewriteExpression(node, context, treeRewriter); } + public Expression rewriteExists( + ExistsPredicate node, C context, ExpressionTreeRewriter treeRewriter) { + return rewriteExpression(node, context, treeRewriter); + } + + public Expression rewriteSubqueryExpression( + SubqueryExpression node, C context, ExpressionTreeRewriter treeRewriter) { + return rewriteExpression(node, context, treeRewriter); + } + public Expression rewriteFunctionCall( FunctionCall node, C context, ExpressionTreeRewriter treeRewriter) { return rewriteExpression(node, context, treeRewriter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java index 4b41a11b9324..0f0d9e8f02aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataTypeParameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; @@ -49,6 +50,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TypeParameter; @@ -526,6 +528,40 @@ protected Expression visitInListExpression(InListExpression node, Context con return node; } + @Override + protected Expression visitExists(ExistsPredicate node, Context context) { + if (!context.isDefaultRewrite()) { + Expression result = + rewriter.rewriteExists(node, context.get(), ExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + Expression subquery = node.getSubquery(); + subquery = rewrite(subquery, context.get()); + + if (subquery != node.getSubquery()) { + return new ExistsPredicate(subquery); + } + + return node; + } + + @Override + public Expression visitSubqueryExpression(SubqueryExpression node, Context context) { + if (!context.isDefaultRewrite()) { + Expression result = + rewriter.rewriteSubqueryExpression(node, context.get(), ExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + // No default rewrite for SubqueryExpression since we do not want to traverse subqueries + return node; + } + @Override protected Expression visitLiteral(Literal node, Context context) { if (!context.isDefaultRewrite()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinColumns.java new file mode 100644 index 000000000000..70d4f3021ddd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinColumns.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; + +import com.google.common.collect.ImmutableSet; + +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.Sets.intersection; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor.extractUnique; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.Util.restrictOutputs; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.correlatedJoin; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isAtMostScalar; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isScalar; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; + +public class PruneCorrelatedJoinColumns extends ProjectOffPushDownRule { + public PruneCorrelatedJoinColumns() { + super(correlatedJoin()); + } + + @Override + protected Optional pushDownProjectOff( + Context context, CorrelatedJoinNode correlatedJoinNode, Set referencedOutputs) { + PlanNode input = correlatedJoinNode.getInput(); + PlanNode subquery = correlatedJoinNode.getSubquery(); + + // remove unused correlated join node, retain input + if (intersection(ImmutableSet.copyOf(subquery.getOutputSymbols()), referencedOutputs) + .isEmpty()) { + // remove unused subquery of inner join + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.INNER + && isScalar(subquery, context.getLookup()) + && correlatedJoinNode.getFilter().equals(TRUE_LITERAL)) { + return Optional.of(input); + } + // remove unused subquery of left join + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.LEFT + && isAtMostScalar(subquery, context.getLookup())) { + return Optional.of(input); + } + } + + Set referencedAndCorrelationSymbols = + ImmutableSet.builder() + .addAll(referencedOutputs) + .addAll(correlatedJoinNode.getCorrelation()) + .build(); + + // remove unused input node, retain subquery + if (intersection(ImmutableSet.copyOf(input.getOutputSymbols()), referencedAndCorrelationSymbols) + .isEmpty()) { + // remove unused input of inner join + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.INNER + && isScalar(input, context.getLookup()) + && correlatedJoinNode.getFilter().equals(TRUE_LITERAL)) { + return Optional.of(subquery); + } + // remove unused input of right join + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.RIGHT + && isAtMostScalar(input, context.getLookup())) { + return Optional.of(subquery); + } + } + + Set filterSymbols = extractUnique(correlatedJoinNode.getFilter()); + + Set referencedAndFilterSymbols = + ImmutableSet.builder().addAll(referencedOutputs).addAll(filterSymbols).build(); + + Optional newSubquery = + restrictOutputs(context.getIdAllocator(), subquery, referencedAndFilterSymbols); + + Set referencedAndFilterAndCorrelationSymbols = + ImmutableSet.builder() + .addAll(referencedAndFilterSymbols) + .addAll(correlatedJoinNode.getCorrelation()) + .build(); + + Optional newInput = + restrictOutputs(context.getIdAllocator(), input, referencedAndFilterAndCorrelationSymbols); + + boolean pruned = newSubquery.isPresent() || newInput.isPresent(); + + if (pruned) { + return Optional.of( + new CorrelatedJoinNode( + correlatedJoinNode.getPlanNodeId(), + newInput.orElse(input), + newSubquery.orElse(subquery), + correlatedJoinNode.getCorrelation(), + correlatedJoinNode.getJoinType(), + correlatedJoinNode.getFilter(), + correlatedJoinNode.getOriginSubquery())); + } + + return Optional.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinCorrelation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinCorrelation.java new file mode 100644 index 000000000000..c8ec3adeea04 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinCorrelation.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import java.util.List; +import java.util.Set; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor.extractUnique; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.correlatedJoin; + +public class PruneCorrelatedJoinCorrelation implements Rule { + private static final Pattern PATTERN = correlatedJoin(); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(CorrelatedJoinNode correlatedJoinNode, Captures captures, Context context) { + Set subquerySymbols = + extractUnique(correlatedJoinNode.getSubquery(), context.getLookup()); + List newCorrelation = + correlatedJoinNode.getCorrelation().stream() + .filter(subquerySymbols::contains) + .collect(toImmutableList()); + + if (newCorrelation.size() < correlatedJoinNode.getCorrelation().size()) { + return Result.ofPlanNode( + new CorrelatedJoinNode( + correlatedJoinNode.getPlanNodeId(), + correlatedJoinNode.getInput(), + correlatedJoinNode.getSubquery(), + newCorrelation, + correlatedJoinNode.getJoinType(), + correlatedJoinNode.getFilter(), + correlatedJoinNode.getOriginSubquery())); + } + + return Result.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneEnforceSingleRowColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneEnforceSingleRowColumns.java new file mode 100644 index 000000000000..17125ef9a802 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneEnforceSingleRowColumns.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; + +import java.util.Optional; +import java.util.Set; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.Util.restrictChildOutputs; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.enforceSingleRow; + +public class PruneEnforceSingleRowColumns extends ProjectOffPushDownRule { + public PruneEnforceSingleRowColumns() { + super(enforceSingleRow()); + } + + @Override + protected Optional pushDownProjectOff( + Context context, EnforceSingleRowNode enforceSingleRowNode, Set referencedOutputs) { + return restrictChildOutputs(context.getIdAllocator(), enforceSingleRowNode, referencedOutputs); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveRedundantEnforceSingleRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveRedundantEnforceSingleRowNode.java new file mode 100644 index 000000000000..4546dd4efc2c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveRedundantEnforceSingleRowNode.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.enforceSingleRow; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isScalar; + +public class RemoveRedundantEnforceSingleRowNode implements Rule { + private static final Pattern PATTERN = enforceSingleRow(); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(EnforceSingleRowNode node, Captures captures, Context context) { + if (isScalar(node.getSource(), context.getLookup())) { + return Result.ofPlanNode(node.getSource()); + } + return Result.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedSubqueryToJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedSubqueryToJoin.java new file mode 100644 index 000000000000..15369b37778a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedSubqueryToJoin.java @@ -0,0 +1,133 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Assignments; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Cast; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.apache.tsfile.read.common.type.Type; + +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.CorrelatedJoin.correlation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.correlatedJoin; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.extractCardinality; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toSqlType; +import static org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern.empty; + +public class TransformUncorrelatedSubqueryToJoin implements Rule { + private static final Pattern PATTERN = + correlatedJoin().with(empty(correlation())); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(CorrelatedJoinNode correlatedJoinNode, Captures captures, Context context) { + // handle INNER and LEFT correlated join + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.INNER + || correlatedJoinNode.getJoinType() == JoinNode.JoinType.LEFT) { + return Result.ofPlanNode( + rewriteToJoin( + correlatedJoinNode, + correlatedJoinNode.getJoinType(), + correlatedJoinNode.getFilter(), + context.getLookup())); + } + + checkState( + correlatedJoinNode.getJoinType() == JoinNode.JoinType.RIGHT + || correlatedJoinNode.getJoinType() == JoinNode.JoinType.FULL, + "unexpected CorrelatedJoin type: " + correlatedJoinNode.getType()); + + // handle RIGHT and FULL correlated join ON TRUE + JoinNode.JoinType type; + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.RIGHT) { + type = JoinNode.JoinType.INNER; + } else { + type = JoinNode.JoinType.LEFT; + } + JoinNode joinNode = rewriteToJoin(correlatedJoinNode, type, TRUE_LITERAL, context.getLookup()); + + if (correlatedJoinNode.getFilter().equals(TRUE_LITERAL)) { + return Result.ofPlanNode(joinNode); + } + + // handle RIGHT correlated join on condition other than TRUE + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.RIGHT) { + Assignments.Builder assignments = Assignments.builder(); + assignments.putIdentities( + Sets.intersection( + ImmutableSet.copyOf(correlatedJoinNode.getSubquery().getOutputSymbols()), + ImmutableSet.copyOf(correlatedJoinNode.getOutputSymbols()))); + for (Symbol inputSymbol : + Sets.intersection( + ImmutableSet.copyOf(correlatedJoinNode.getInput().getOutputSymbols()), + ImmutableSet.copyOf(correlatedJoinNode.getOutputSymbols()))) { + Type inputType = context.getSymbolAllocator().getTypes().getTableModelType(inputSymbol); + assignments.put( + inputSymbol, + new IfExpression( + correlatedJoinNode.getFilter(), + inputSymbol.toSymbolReference(), + new Cast(new NullLiteral(), toSqlType(inputType)))); + } + ProjectNode projectNode = + new ProjectNode(context.getIdAllocator().genPlanNodeId(), joinNode, assignments.build()); + + return Result.ofPlanNode(projectNode); + } + + // no support for FULL correlated join on condition other than TRUE + return Result.empty(); + } + + private JoinNode rewriteToJoin( + CorrelatedJoinNode parent, JoinNode.JoinType type, Expression filter, Lookup lookup) { + if (type == JoinNode.JoinType.LEFT + && extractCardinality(parent.getSubquery(), lookup).isAtLeastScalar() + && filter.equals(TRUE_LITERAL)) { + // input rows will always be matched against subquery rows + type = JoinNode.JoinType.INNER; + } + return new JoinNode( + parent.getPlanNodeId(), + type, + parent.getInput(), + parent.getSubquery(), + ImmutableList.of(), + parent.getInput().getOutputSymbols(), + parent.getSubquery().getOutputSymbols(), + filter.equals(TRUE_LITERAL) ? Optional.empty() : Optional.of(filter), + Optional.empty()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ApplyNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ApplyNode.java new file mode 100644 index 000000000000..4574bf9d254b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ApplyNode.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.TwoChildProcessNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; + +import com.google.common.collect.ImmutableList; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class ApplyNode extends TwoChildProcessNode { + + /** Correlation symbols, returned from input (outer plan) used in subquery (inner plan) */ + private final List correlation; + + /** + * Expressions that use subquery symbols. + * + *

Subquery expressions are different than other expressions in a sense that they might use an + * entire subquery result as an input (e.g: "x IN (subquery)", "x < ALL (subquery)"). Such + * expressions are invalid in linear operator context (e.g: ProjectNode) in logical plan, but are + * correct in ApplyNode context. + * + *

Example 1: - expression: input_symbol_X IN (subquery_symbol_Y) - meaning: if set consisting + * of all values for subquery_symbol_Y contains value represented by input_symbol_X + * + *

Example 2: - expression: input_symbol_X < ALL (subquery_symbol_Y) - meaning: if + * input_symbol_X is smaller than all subquery values represented by subquery_symbol_Y + * + *

+ */ + private final Map subqueryAssignments; + + /** HACK! Used for error reporting in case this ApplyNode is not supported */ + private final Node originSubquery; + + public ApplyNode( + PlanNodeId id, + // will be set as leftChild + PlanNode input, + // will be set as rightChild + PlanNode subquery, + Map subqueryAssignments, + List correlation, + Node originSubquery) { + super(id, input, subquery); + requireNonNull(subqueryAssignments, "subqueryAssignments is null"); + requireNonNull(correlation, "correlation is null"); + requireNonNull(originSubquery, "originSubquery is null"); + + if (input != null) { + checkArgument( + input.getOutputSymbols().containsAll(correlation), + "Input does not contain symbols from correlation"); + } + + this.subqueryAssignments = subqueryAssignments; + this.correlation = ImmutableList.copyOf(correlation); + this.originSubquery = originSubquery; + } + + public PlanNode getInput() { + return leftChild; + } + + public PlanNode getSubquery() { + return rightChild; + } + + public Map getSubqueryAssignments() { + return subqueryAssignments; + } + + public List getCorrelation() { + return correlation; + } + + public Node getOriginSubquery() { + return originSubquery; + } + + @Override + public List getOutputSymbols() { + return ImmutableList.builder() + .addAll(leftChild.getOutputSymbols()) + .addAll(subqueryAssignments.keySet()) + .build(); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitApply(this, context); + } + + @Override + public PlanNode replaceChildren(List newChildren) { + checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); + return new ApplyNode( + getPlanNodeId(), + newChildren.get(0), + newChildren.get(1), + subqueryAssignments, + correlation, + originSubquery); + } + + @Override + public PlanNode clone() { + // clone without children + return new ApplyNode( + getPlanNodeId(), null, null, subqueryAssignments, correlation, originSubquery); + } + + @Override + public List getOutputColumnNames() { + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + throw new UnsupportedOperationException(); + } + + public interface SetExpression { + List inputs(); + } + + public static class In implements SetExpression { + private final Symbol value; + private final Symbol reference; + + public In(Symbol value, Symbol reference) { + this.value = value; + this.reference = reference; + } + + public Symbol getValue() { + return value; + } + + public Symbol getReference() { + return reference; + } + + @Override + public List inputs() { + return ImmutableList.of(value, reference); + } + } + + public static class Exists implements SetExpression { + @Override + public List inputs() { + return ImmutableList.of(); + } + } + + public static class QuantifiedComparison implements SetExpression { + private final Operator operator; + private final Quantifier quantifier; + private final Symbol value; + private final Symbol reference; + + public QuantifiedComparison( + Operator operator, Quantifier quantifier, Symbol value, Symbol reference) { + this.operator = operator; + this.quantifier = quantifier; + this.value = value; + this.reference = reference; + } + + public Operator getOperator() { + return operator; + } + + public Quantifier getQuantifier() { + return quantifier; + } + + public Symbol getValue() { + return value; + } + + public Symbol getReference() { + return reference; + } + + @Override + public List inputs() { + return ImmutableList.of(value, reference); + } + } + + public enum Operator { + EQUAL, + NOT_EQUAL, + LESS_THAN, + LESS_THAN_OR_EQUAL, + GREATER_THAN, + GREATER_THAN_OR_EQUAL, + } + + public enum Quantifier { + ALL, + ANY, + SOME, + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CorrelatedJoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CorrelatedJoinNode.java new file mode 100644 index 000000000000..64e14b2d15a1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CorrelatedJoinNode.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.TwoChildProcessNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; + +import com.google.common.collect.ImmutableList; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +/** + * For every row from {@link #leftChild}(input) a {@link #rightChild}(subquery) relation is + * calculated. Then input row is cross joined with subquery relation and returned as a result. + * + *

INNER - does not return any row for input row when subquery relation is empty LEFT - does + * return input completed with NULL values when subquery relation is empty + */ +public class CorrelatedJoinNode extends TwoChildProcessNode { + + /** Correlation symbols, returned from input (outer plan) used in subquery (inner plan) */ + private final List correlation; + + private final JoinNode.JoinType type; + private final Expression filter; + + /** HACK! Used for error reporting in case this ApplyNode is not supported */ + private final Node originSubquery; + + public CorrelatedJoinNode( + PlanNodeId id, + PlanNode input, + PlanNode subquery, + List correlation, + JoinNode.JoinType type, + Expression filter, + Node originSubquery) { + super(id, input, subquery); + requireNonNull(correlation, "correlation is null"); + requireNonNull(filter, "filter is null"); + // The condition doesn't guarantee that filter is of type boolean, but was found to be a + // practical way to identify + // places where CorrelatedJoinNode could be created without appropriate coercions. + checkArgument( + !(filter instanceof NullLiteral), + "Filter must be an expression of boolean type: %s", + filter); + requireNonNull(originSubquery, "originSubquery is null"); + + if (input != null) { + checkArgument( + input.getOutputSymbols().containsAll(correlation), + "Input does not contain symbols from correlation"); + } + + this.correlation = ImmutableList.copyOf(correlation); + this.type = type; + this.filter = filter; + this.originSubquery = originSubquery; + } + + public PlanNode getInput() { + return leftChild; + } + + public PlanNode getSubquery() { + return rightChild; + } + + public List getCorrelation() { + return correlation; + } + + public JoinNode.JoinType getJoinType() { + return type; + } + + public Expression getFilter() { + return filter; + } + + public Node getOriginSubquery() { + return originSubquery; + } + + public List getSources() { + return ImmutableList.of(leftChild, rightChild); + } + + @Override + public List getOutputSymbols() { + return ImmutableList.builder() + .addAll(leftChild.getOutputSymbols()) + .addAll(rightChild.getOutputSymbols()) + .build(); + } + + @Override + public PlanNode replaceChildren(List newChildren) { + checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); + return new CorrelatedJoinNode( + getPlanNodeId(), + newChildren.get(0), + newChildren.get(1), + correlation, + type, + filter, + originSubquery); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCorrelatedJoin(this, context); + } + + @Override + public PlanNode clone() { + // clone without children + return new CorrelatedJoinNode( + getPlanNodeId(), null, null, correlation, type, filter, originSubquery); + } + + @Override + public List getOutputColumnNames() { + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/EnforceSingleRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/EnforceSingleRowNode.java new file mode 100644 index 000000000000..d67a47a1eba3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/EnforceSingleRowNode.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; + +import com.google.common.collect.Iterables; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +public class EnforceSingleRowNode extends SingleChildProcessNode { + + public EnforceSingleRowNode(PlanNodeId id, PlanNode source) { + super(id, source); + } + + @Override + public List getOutputSymbols() { + return child.getOutputSymbols(); + } + + public PlanNode getSource() { + return child; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitEnforceSingleRow(this, context); + } + + @Override + public PlanNode replaceChildren(List newChildren) { + return new EnforceSingleRowNode(getPlanNodeId(), Iterables.getOnlyElement(newChildren)); + } + + @Override + public PlanNode clone() { + // clone without children + return new EnforceSingleRowNode(getPlanNodeId(), null); + } + + @Override + public List getOutputColumnNames() { + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index 0775075a6852..4a8721222cfd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -292,6 +292,10 @@ public Optional isSpillable() { return spillable; } + public boolean isCrossJoin() { + return criteria.isEmpty() && !filter.isPresent() && joinType == JoinType.INNER; + } + @Override public String toString() { return "JoinNode-" + this.getPlanNodeId(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java index 9cea03e14043..aea9fb529f87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java @@ -14,7 +14,9 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Property; @@ -34,22 +36,21 @@ public static Pattern aggregation() { return typeOf(AggregationNode.class); } - /*public static Pattern assignUniqueId() - { - return typeOf(AssignUniqueId.class); - } - - public static Pattern groupId() - { - return typeOf(GroupIdNode.class); - } + // public static Pattern assignUniqueId() + // { + // return typeOf(AssignUniqueId.class); + // } + // + // public static Pattern groupId() + // { + // return typeOf(GroupIdNode.class); + // } - public static Pattern applyNode() - { - return typeOf(ApplyNode.class); + public static Pattern applyNode() { + return typeOf(ApplyNode.class); } - public static Pattern tableExecute() + /* public static Pattern tableExecute() { return typeOf(TableExecuteNode.class); } @@ -72,13 +73,12 @@ public static Pattern exchange() public static Pattern explainAnalyze() { return typeOf(ExplainAnalyzeNode.class); - } - - public static Pattern enforceSingleRow() - { - return typeOf(EnforceSingleRowNode.class); }*/ + public static Pattern enforceSingleRow() { + return typeOf(EnforceSingleRowNode.class); + } + public static Pattern filter() { return typeOf(FilterNode.class); } @@ -105,13 +105,12 @@ public static Pattern dynamicFilterSource() public static Pattern spatialJoin() { return typeOf(SpatialJoinNode.class); - } - - public static Pattern correlatedJoin() - { - return typeOf(CorrelatedJoinNode.class); }*/ + public static Pattern correlatedJoin() { + return typeOf(CorrelatedJoinNode.class); + } + public static Pattern offset() { return typeOf(OffsetNode.class); } @@ -136,13 +135,13 @@ public static Pattern project() { /*public static Pattern sample() { return typeOf(SampleNode.class); - } - - public static Pattern semiJoin() - { - return typeOf(SemiJoinNode.class); }*/ + // public static Pattern semiJoin() + // { + // return typeOf(SemiJoinNode.class); + // } + public static Pattern gapFill() { return typeOf(GapFillNode.class); } @@ -279,72 +278,63 @@ public static Property step() { return property("step", AggregationNode::getStep); } - } + }*/ - public static final class Apply - { - public static Property> correlation() - { - return property("correlation", ApplyNode::getCorrelation); - } + public static final class Apply { + public static Property> correlation() { + return property("correlation", ApplyNode::getCorrelation); + } } - public static final class DistinctLimit + /*public static final class DistinctLimit { public static Property isPartial() { return property("isPartial", DistinctLimitNode::isPartial); } - } + }*/ - public static final class Exchange + /*public static final class Exchange { public static Property scope() { return property("scope", ExchangeNode::getScope); } - } + }*/ - public static final class Join - { - public static Property type() - { - return property("type", JoinNode::getType); - } + public static final class Join { + public static Property type() { + return property("type", JoinNode::getJoinType); + } - public static Property left() - { - return property("left", (JoinNode joinNode, Lookup lookup) -> lookup.resolve(joinNode.getLeft())); - } + public static Property left() { + return property( + "left", (JoinNode joinNode, Lookup lookup) -> lookup.resolve(joinNode.getLeftChild())); + } - public static Property right() - { - return property("right", (JoinNode joinNode, Lookup lookup) -> lookup.resolve(joinNode.getRight())); - } + public static Property right() { + return property( + "right", (JoinNode joinNode, Lookup lookup) -> lookup.resolve(joinNode.getRightChild())); + } } - public static final class CorrelatedJoin - { - public static Property> correlation() - { - return property("correlation", CorrelatedJoinNode::getCorrelation); - } + public static final class CorrelatedJoin { + public static Property> correlation() { + return property("correlation", CorrelatedJoinNode::getCorrelation); + } - public static Property subquery() - { - return property("subquery", (node, context) -> context.resolve(node.getSubquery())); - } + public static Property subquery() { + return property("subquery", (node, context) -> context.resolve(node.getSubquery())); + } - public static Property filter() - { - return property("filter", CorrelatedJoinNode::getFilter); - } + public static Property filter() { + return property("filter", CorrelatedJoinNode::getFilter); + } - public static Property type() - { - return property("type", CorrelatedJoinNode::getType); - } - }*/ + public static Property type() { + return property("type", CorrelatedJoinNode::getJoinType); + } + } public static final class Limit { public static Property count() { @@ -393,9 +383,9 @@ public static Property rowCount() { return property("rowCount", ValuesNode::getRowCount); } - } + }*/ - public static final class SemiJoin + /*public static final class SemiJoin { public static Property getSource() { @@ -410,9 +400,9 @@ public static Property getFilteringSource() "filteringSource", (SemiJoinNode semiJoin, Lookup lookup) -> lookup.resolve(semiJoin.getFilteringSource())); } - } + }*/ - public static final class Intersect + /*public static final class Intersect { public static Property distinct() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Cardinality.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Cardinality.java new file mode 100644 index 000000000000..2db8c0849d60 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Cardinality.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; + +import com.google.common.collect.Range; + +import static java.util.Objects.requireNonNull; + +public class Cardinality { + private final Range cardinalityRange; + + public Cardinality(Range cardinalityRange) { + this.cardinalityRange = requireNonNull(cardinalityRange, "cardinalityRange is null"); + } + + public boolean isEmpty() { + return isAtMost(0); + } + + public boolean isScalar() { + return Range.singleton(1L).encloses(cardinalityRange); + } + + public boolean isAtLeastScalar() { + return isAtLeast(1L); + } + + public boolean isAtMostScalar() { + return isAtMost(1L); + } + + public boolean isAtLeast(long minCardinality) { + return Range.atLeast(minCardinality).encloses(cardinalityRange); + } + + public boolean isAtMost(long maxCardinality) { + return Range.closed(0L, maxCardinality).encloses(cardinalityRange); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CheckSubqueryNodesAreRewritten.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CheckSubqueryNodesAreRewritten.java new file mode 100644 index 000000000000..eab54f6251ca --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CheckSubqueryNodesAreRewritten.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; + +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; + +import java.util.List; + +import static com.google.common.base.Preconditions.checkState; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.PlanNodeSearcher.searchFrom; +import static org.apache.iotdb.rpc.TSStatusCode.SEMANTIC_ERROR; + +public class CheckSubqueryNodesAreRewritten implements PlanOptimizer { + @Override + public PlanNode optimize(PlanNode plan, Context context) { + searchFrom(plan) + .where(ApplyNode.class::isInstance) + .findFirst() + .ifPresent( + node -> { + ApplyNode applyNode = (ApplyNode) node; + throw error(applyNode.getCorrelation()); + }); + + searchFrom(plan) + .where(CorrelatedJoinNode.class::isInstance) + .findFirst() + .ifPresent( + node -> { + CorrelatedJoinNode correlatedJoinNode = (CorrelatedJoinNode) node; + throw error(correlatedJoinNode.getCorrelation()); + }); + + return plan; + } + + private SemanticException error(List correlation) { + checkState( + !correlation.isEmpty(), + "All the non correlated subqueries should be rewritten at this point"); + throw new SemanticException( + "Given correlated subquery is not supported", SEMANTIC_ERROR.getStatusCode()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 845ed0d855b9..b451afdcf54e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -33,6 +33,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.MergeLimits; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneAggregationColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneAggregationSourceColumns; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneCorrelatedJoinColumns; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneCorrelatedJoinCorrelation; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneEnforceSingleRowColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneFillColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneFilterColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneGapFillColumns; @@ -48,9 +51,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PushLimitThroughOffset; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PushLimitThroughProject; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveDuplicateConditions; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveRedundantEnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveRedundantIdentityProjections; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveTrivialFilters; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.SimplifyExpressions; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedSubqueryToJoin; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -73,6 +78,9 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { // TODO After ValuesNode introduced // new RemoveEmptyGlobalAggregation(), new PruneAggregationSourceColumns(), + new PruneCorrelatedJoinColumns(), + new PruneCorrelatedJoinCorrelation(), + new PruneEnforceSingleRowColumns(), new PruneFilterColumns(), new PruneGapFillColumns(), new PruneFillColumns(), @@ -191,6 +199,13 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new UnaliasSymbolReferences(plannerContext.getMetadata()), columnPruningOptimizer, inlineProjectionLimitFiltersOptimizer, + new IterativeOptimizer( + plannerContext, + ruleStats, + ImmutableSet.of( + new RemoveRedundantEnforceSingleRowNode(), + new TransformUncorrelatedSubqueryToJoin())), + new CheckSubqueryNodesAreRewritten(), new PushPredicateIntoTableScan(), // redo columnPrune and inlineProjections after pushPredicateIntoTableScan columnPruningOptimizer, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java index e5dcffd9ea06..f0489bae6ac0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java @@ -155,6 +155,10 @@ private PushDownLevel calculatePushDownLevel( // calculate DataSet part boolean singleDeviceEntry = tableScanNode.getDeviceEntries().size() < 2; if (groupingKeys.isEmpty()) { + // fixme: we don't expect ProjectNode here. Temporary walkaround. + if (projectNode != null) { + return PushDownLevel.NOOP; + } // GlobalAggregation if (singleDeviceEntry) { return PushDownLevel.COMPLETE; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 8e9296b7f488..a2166e1a1ca2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -642,12 +642,11 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { } if (node.getJoinType() == INNER && newJoinFilter.isPresent() && equiJoinClauses.isEmpty()) { - throw new IllegalStateException("INNER JOIN only support equiJoinClauses"); // if we do not have any equi conjunct we do not pushdown non-equality condition into // inner join, so we plan execution as nested-loops-join followed by filter instead // hash join. - // postJoinPredicate = combineConjuncts(postJoinPredicate, newJoinFilter.get()); - // newJoinFilter = Optional.empty(); + postJoinPredicate = combineConjuncts(postJoinPredicate, newJoinFilter.get()); + newJoinFilter = Optional.empty(); } boolean filtersEquivalent = @@ -680,31 +679,34 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { node.isSpillable()); } - JoinNode.EquiJoinClause joinCriteria = ((JoinNode) output).getCriteria().get(0); - OrderingScheme leftOrderingScheme = - new OrderingScheme( - Collections.singletonList(joinCriteria.getLeft()), - Collections.singletonMap(joinCriteria.getLeft(), ASC_NULLS_LAST)); - OrderingScheme rightOrderingScheme = - new OrderingScheme( - Collections.singletonList(joinCriteria.getRight()), - Collections.singletonMap(joinCriteria.getRight(), ASC_NULLS_LAST)); - SortNode leftSortNode = - new SortNode( - queryId.genPlanNodeId(), - ((JoinNode) output).getLeftChild(), - leftOrderingScheme, - false, - false); - SortNode rightSortNode = - new SortNode( - queryId.genPlanNodeId(), - ((JoinNode) output).getRightChild(), - rightOrderingScheme, - false, - false); - ((JoinNode) output).setLeftChild(leftSortNode); - ((JoinNode) output).setRightChild(rightSortNode); + // sort the left and right child of join node if it is not a cross join + if (!((JoinNode) output).isCrossJoin()) { + JoinNode.EquiJoinClause joinCriteria = ((JoinNode) output).getCriteria().get(0); + OrderingScheme leftOrderingScheme = + new OrderingScheme( + Collections.singletonList(joinCriteria.getLeft()), + Collections.singletonMap(joinCriteria.getLeft(), ASC_NULLS_LAST)); + OrderingScheme rightOrderingScheme = + new OrderingScheme( + Collections.singletonList(joinCriteria.getRight()), + Collections.singletonMap(joinCriteria.getRight(), ASC_NULLS_LAST)); + SortNode leftSortNode = + new SortNode( + queryId.genPlanNodeId(), + ((JoinNode) output).getLeftChild(), + leftOrderingScheme, + false, + false); + SortNode rightSortNode = + new SortNode( + queryId.genPlanNodeId(), + ((JoinNode) output).getRightChild(), + rightOrderingScheme, + false, + false); + ((JoinNode) output).setLeftChild(leftSortNode); + ((JoinNode) output).setRightChild(rightSortNode); + } if (!postJoinPredicate.equals(TRUE_LITERAL)) { output = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java new file mode 100644 index 000000000000..2d7d7a923496 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ExchangeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; + +import com.google.common.collect.Range; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup.noLookup; + +public final class QueryCardinalityUtil { + private QueryCardinalityUtil() {} + + public static boolean isScalar(PlanNode node) { + return isScalar(node, noLookup()); + } + + public static boolean isScalar(PlanNode node, Lookup lookup) { + return extractCardinality(node, lookup).isScalar(); + } + + public static boolean isAtMostScalar(PlanNode node) { + return isAtMostScalar(node, noLookup()); + } + + public static boolean isAtMostScalar(PlanNode node, Lookup lookup) { + return isAtMost(node, lookup, 1L); + } + + public static boolean isAtMost(PlanNode node, Lookup lookup, long maxCardinality) { + return extractCardinality(node, lookup).isAtMost(maxCardinality); + } + + public static boolean isAtLeastScalar(PlanNode node, Lookup lookup) { + return isAtLeast(node, lookup, 1L); + } + + public static boolean isAtLeast(PlanNode node, Lookup lookup, long minCardinality) { + return extractCardinality(node, lookup).isAtLeast(minCardinality); + } + + public static boolean isEmpty(PlanNode node, Lookup lookup) { + return isAtMost(node, lookup, 0); + } + + public static Cardinality extractCardinality(PlanNode node) { + return extractCardinality(node, noLookup()); + } + + public static Cardinality extractCardinality(PlanNode node, Lookup lookup) { + return new Cardinality(node.accept(new CardinalityExtractorPlanVisitor(lookup), null)); + } + + private static final class CardinalityExtractorPlanVisitor + extends PlanVisitor, Void> { + private final Lookup lookup; + + public CardinalityExtractorPlanVisitor(Lookup lookup) { + this.lookup = requireNonNull(lookup, "lookup is null"); + } + + @Override + public Range visitPlan(PlanNode node, Void context) { + return Range.atLeast(0L); + } + + @Override + public Range visitGroupReference(GroupReference node, Void context) { + return lookup.resolve(node).accept(this, context); + } + + @Override + public Range visitEnforceSingleRow(EnforceSingleRowNode node, Void context) { + return Range.singleton(1L); + } + + @Override + public Range visitAggregation(AggregationNode node, Void context) { + if (node.hasSingleGlobalAggregation()) { + // only single default aggregation which will produce exactly single row + return Range.singleton(1L); + } + + Range sourceCardinalityRange = node.getChild().accept(this, null); + + long lower; + if (node.hasDefaultOutput() || sourceCardinalityRange.lowerEndpoint() > 0) { + lower = 1; + } else { + lower = 0; + } + + if (sourceCardinalityRange.hasUpperBound()) { + long upper = Math.max(lower, sourceCardinalityRange.upperEndpoint()); + return Range.closed(lower, upper); + } + + return Range.atLeast(lower); + } + + @Override + public Range visitExchange(ExchangeNode node, Void context) { + if (node.getChildren().size() == 1) { + return getOnlyElement(node.getChildren()).accept(this, null); + } + return Range.atLeast(0L); + } + + @Override + public Range visitProject(ProjectNode node, Void context) { + return node.getChild().accept(this, null); + } + + @Override + public Range visitFilter(FilterNode node, Void context) { + Range sourceCardinalityRange = node.getChild().accept(this, null); + if (sourceCardinalityRange.hasUpperBound()) { + return Range.closed(0L, sourceCardinalityRange.upperEndpoint()); + } + return Range.atLeast(0L); + } + + // @Override + // public Range visitValues(ValuesNode node, Void context) + // { + // return Range.singleton((long) node.getRowCount()); + // } + + @Override + public Range visitOffset(OffsetNode node, Void context) { + Range sourceCardinalityRange = node.getChild().accept(this, null); + + long lower = max(sourceCardinalityRange.lowerEndpoint() - node.getCount(), 0L); + + if (sourceCardinalityRange.hasUpperBound()) { + return Range.closed( + lower, max(sourceCardinalityRange.upperEndpoint() - node.getCount(), 0L)); + } + return Range.atLeast(lower); + } + + @Override + public Range visitLimit(LimitNode node, Void context) { + if (node.isWithTies()) { + Range sourceCardinalityRange = node.getChild().accept(this, null); + long lower = min(node.getCount(), sourceCardinalityRange.lowerEndpoint()); + if (sourceCardinalityRange.hasUpperBound()) { + return Range.closed(lower, sourceCardinalityRange.upperEndpoint()); + } + return Range.atLeast(lower); + } + + return applyLimit(node.getChild(), node.getCount()); + } + + // @Override + // public Range visitTopN(TopNNode node, Void context) + // { + // return applyLimit(node.getSource(), node.getCount()); + // } + + // @Override + // public Range visitWindow(WindowNode node, Void context) + // { + // return node.getSource().accept(this, null); + // } + + private Range applyLimit(PlanNode source, long limit) { + Range sourceCardinalityRange = source.accept(this, null); + if (sourceCardinalityRange.hasUpperBound()) { + limit = min(sourceCardinalityRange.upperEndpoint(), limit); + } + long lower = min(limit, sourceCardinalityRange.lowerEndpoint()); + return Range.closed(lower, limit); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java index 1ec6499ff22b..623a9873d908 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.ir.ExpressionRewriter; import org.apache.iotdb.db.queryengine.plan.relational.planner.ir.ExpressionTreeRewriter; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@ -84,18 +85,23 @@ public Symbol map(Symbol symbol) { return mappingFunction.apply(symbol); } - /*public ApplyNode.SetExpression map(ApplyNode.SetExpression expression) - { - return switch (expression) { - case ApplyNode.Exists exists -> exists; - case ApplyNode.In in -> new ApplyNode.In(map(in.value()), map(in.reference())); - case ApplyNode.QuantifiedComparison comparison -> new ApplyNode.QuantifiedComparison( - comparison.operator(), - comparison.quantifier(), - map(comparison.value()), - map(comparison.reference())); - }; - }*/ + public ApplyNode.SetExpression map(ApplyNode.SetExpression expression) { + if (expression instanceof ApplyNode.Exists) { + return expression; + } else if (expression instanceof ApplyNode.In) { + ApplyNode.In in = (ApplyNode.In) expression; + return new ApplyNode.In(map(in.getValue()), map(in.getReference())); + } else if (expression instanceof ApplyNode.QuantifiedComparison) { + ApplyNode.QuantifiedComparison comparison = (ApplyNode.QuantifiedComparison) expression; + return new ApplyNode.QuantifiedComparison( + comparison.getOperator(), + comparison.getQuantifier(), + map(comparison.getValue()), + map(comparison.getReference())); + } else { + throw new IllegalArgumentException("Unexpected value: " + expression); + } + } public List map(List symbols) { return symbols.stream().map(this::map).collect(toImmutableList()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index cbf6bbf90e75..15f63b61ee74 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -24,6 +24,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolAllocator; import org.apache.iotdb.db.queryengine.plan.relational.planner.ir.DeterminismEvaluator; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; @@ -453,6 +456,136 @@ public PlanAndMappings visitOutput(OutputNode node, UnaliasContext context) { mapping); } + @Override + public PlanAndMappings visitEnforceSingleRow( + EnforceSingleRowNode node, UnaliasContext context) { + PlanAndMappings rewrittenSource = node.getSource().accept(this, context); + + return new PlanAndMappings( + node.replaceChildren(ImmutableList.of(rewrittenSource.getRoot())), + rewrittenSource.getMappings()); + } + + @Override + public PlanAndMappings visitApply(ApplyNode node, UnaliasContext context) { + // it is assumed that apart from correlation (and possibly outer correlation), symbols are + // distinct between Input and Subquery + // rewrite Input + PlanAndMappings rewrittenInput = node.getInput().accept(this, context); + Map inputMapping = new HashMap<>(rewrittenInput.getMappings()); + SymbolMapper mapper = symbolMapper(inputMapping); + + // rewrite correlation with mapping from Input + List rewrittenCorrelation = mapper.mapAndDistinct(node.getCorrelation()); + + // extract new mappings for correlation symbols to apply in Subquery + Set correlationSymbols = ImmutableSet.copyOf(node.getCorrelation()); + Map correlationMapping = new HashMap<>(); + for (Map.Entry entry : inputMapping.entrySet()) { + if (correlationSymbols.contains(entry.getKey())) { + correlationMapping.put(entry.getKey(), mapper.map(entry.getKey())); + } + } + + Map mappingForSubquery = new HashMap<>(); + mappingForSubquery.putAll(context.getCorrelationMapping()); + mappingForSubquery.putAll(correlationMapping); + + // rewrite Subquery + PlanAndMappings rewrittenSubquery = + node.getSubquery().accept(this, new UnaliasContext(mappingForSubquery)); + + // unify mappings from Input and Subquery to rewrite Subquery assignments + Map resultMapping = new HashMap<>(); + resultMapping.putAll(rewrittenInput.getMappings()); + resultMapping.putAll(rewrittenSubquery.getMappings()); + mapper = symbolMapper(resultMapping); + + ImmutableList.Builder> rewrittenAssignments = + ImmutableList.builder(); + for (Map.Entry assignment : + node.getSubqueryAssignments().entrySet()) { + rewrittenAssignments.add( + new SimpleEntry<>(mapper.map(assignment.getKey()), mapper.map(assignment.getValue()))); + } + + // deduplicate assignments + Map deduplicateAssignments = + rewrittenAssignments.build().stream() + .distinct() + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + + mapper = symbolMapper(resultMapping); + + // build new Assignments with canonical outputs + // duplicate entries will be removed by the Builder + ImmutableMap.Builder newAssignments = ImmutableMap.builder(); + for (Map.Entry assignment : + deduplicateAssignments.entrySet()) { + newAssignments.put(mapper.map(assignment.getKey()), assignment.getValue()); + } + + return new PlanAndMappings( + new ApplyNode( + node.getPlanNodeId(), + rewrittenInput.getRoot(), + rewrittenSubquery.getRoot(), + newAssignments.buildOrThrow(), + rewrittenCorrelation, + node.getOriginSubquery()), + resultMapping); + } + + @Override + public PlanAndMappings visitCorrelatedJoin(CorrelatedJoinNode node, UnaliasContext context) { + // it is assumed that apart from correlation (and possibly outer correlation), symbols are + // distinct between left and right CorrelatedJoin source + // rewrite Input + PlanAndMappings rewrittenInput = node.getInput().accept(this, context); + Map inputMapping = new HashMap<>(rewrittenInput.getMappings()); + SymbolMapper mapper = symbolMapper(inputMapping); + + // rewrite correlation with mapping from Input + List rewrittenCorrelation = mapper.mapAndDistinct(node.getCorrelation()); + + // extract new mappings for correlation symbols to apply in Subquery + Set correlationSymbols = ImmutableSet.copyOf(node.getCorrelation()); + Map correlationMapping = new HashMap<>(); + for (Map.Entry entry : inputMapping.entrySet()) { + if (correlationSymbols.contains(entry.getKey())) { + correlationMapping.put(entry.getKey(), mapper.map(entry.getKey())); + } + } + + Map mappingForSubquery = new HashMap<>(); + mappingForSubquery.putAll(context.getCorrelationMapping()); + mappingForSubquery.putAll(correlationMapping); + + // rewrite Subquery + PlanAndMappings rewrittenSubquery = + node.getSubquery().accept(this, new UnaliasContext(mappingForSubquery)); + + // unify mappings from Input and Subquery + Map resultMapping = new HashMap<>(); + resultMapping.putAll(rewrittenInput.getMappings()); + resultMapping.putAll(rewrittenSubquery.getMappings()); + + // rewrite filter with unified mapping + mapper = symbolMapper(resultMapping); + Expression newFilter = mapper.map(node.getFilter()); + + return new PlanAndMappings( + new CorrelatedJoinNode( + node.getPlanNodeId(), + rewrittenInput.getRoot(), + rewrittenSubquery.getRoot(), + rewrittenCorrelation, + node.getJoinType(), + newFilter, + node.getOriginSubquery()), + resultMapping); + } + @Override public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { // it is assumed that symbols are distinct between left and right join source. Only symbols diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SingleColumn.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SingleColumn.java index 9f4e3663e6f7..e4e3cf9f7455 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SingleColumn.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SingleColumn.java @@ -109,6 +109,10 @@ public boolean shallowEquals(Node other) { return false; } + if (alias == null) { + return ((SingleColumn) other).alias == null; + } + return alias.equals(((SingleColumn) other).alias); } } From 3a548e84e5d90c90f91574d0d8db7faab26e33c7 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Sun, 17 Nov 2024 16:20:23 +0800 Subject: [PATCH 02/12] remove accidentally added files --- .../org/apache/iotdb/SessionExampleTest.java | 921 ------------------ iotdb-client/client-py/setup.py | 64 -- 2 files changed, 985 deletions(-) delete mode 100644 example/session/src/main/java/org/apache/iotdb/SessionExampleTest.java delete mode 100644 iotdb-client/client-py/setup.py diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExampleTest.java b/example/session/src/main/java/org/apache/iotdb/SessionExampleTest.java deleted file mode 100644 index 4214b2e73008..000000000000 --- a/example/session/src/main/java/org/apache/iotdb/SessionExampleTest.java +++ /dev/null @@ -1,921 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb; - -import org.apache.iotdb.common.rpc.thrift.TAggregationType; -import org.apache.iotdb.isession.SessionDataSet; -import org.apache.iotdb.isession.SessionDataSet.DataIterator; -import org.apache.iotdb.isession.template.Template; -import org.apache.iotdb.isession.util.Version; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; -import org.apache.iotdb.session.template.MeasurementNode; - -import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.enums.CompressionType; -import org.apache.tsfile.file.metadata.enums.TSEncoding; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BitMap; -import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.IMeasurementSchema; -import org.apache.tsfile.write.schema.MeasurementSchema; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; - -@SuppressWarnings({"squid:S106", "squid:S1144", "squid:S125"}) -public class SessionExampleTest { - - private static Session session; - private static Session sessionEnableRedirect; - private static final String ROOT_SG1_D1_S1 = "root.sg1.d1.s1"; - private static final String ROOT_SG1_D1_S2 = "root.sg1.d1.s2"; - private static final String ROOT_SG1_D1_S3 = "root.sg1.d1.s3"; - private static final String ROOT_SG1_D1_S4 = "root.sg1.d1.s4"; - private static final String ROOT_SG1_D1_S5 = "root.sg1.d1.s5"; - private static final String ROOT_SG1_D1 = "root.sg1.d1"; - private static final String ROOT_SG1 = "root.sg1"; - private static final String LOCAL_HOST = "127.0.0.1"; - public static final String SELECT_D1 = "select * from root.sg1.d1"; - - private static Random random = new Random(); - - public static void main(String[] args) - throws IoTDBConnectionException, StatementExecutionException { - session = - new Session.Builder() - .host(LOCAL_HOST) - .port(6667) - .username("root") - .password("root") - .sqlDialect("table") - .database("test") - .version(Version.V_1_0) - .build(); - session.open(false); - String sql = "select sum(s1) from (select * from table1)"; - // String sql = "select sum(s1) from table1"; - - // set session fetchSize - session.setFetchSize(10000); - long loop = 20; - long start = System.currentTimeMillis(); - for (int i = 0; i < loop; i++) { - try (SessionDataSet result = session.executeQueryStatement(sql)) { - while (result.hasNext()) { - result.next(); - } - } - } - long end = System.currentTimeMillis(); - System.out.println("Avg execution time is : " + (end - start) / loop + "ms"); - - // try { - // session.createDatabase("root.test"); - // } catch (StatementExecutionException e) { - // if (e.getStatusCode() != TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode()) { - // throw e; - // } - // } - // - // List schemaList = new ArrayList<>(); - // schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); - // schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); - // schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); - // final List columnTypes = - // Arrays.asList( - // Tablet.ColumnType.MEASUREMENT, - // Tablet.ColumnType.MEASUREMENT, - // Tablet.ColumnType.MEASUREMENT); - // Tablet tablet = new Tablet("table1", schemaList, columnTypes, 100000); - // long timestamp = 0; - // for (long row = 0; row < 100000; row++) { - // int rowIndex = tablet.rowSize++; - // tablet.addTimestamp(rowIndex, timestamp); - // for (int s = 0; s < 3; s++) { - // long value = timestamp; - // tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); - // } - // timestamp++; - // } - // try { - // session.insertRelationalTablet(tablet); - // } catch (Exception e) { - // } finally { - // session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='false'"); - // } - - session.close(); - } - - private static void createAndDropContinuousQueries() - throws StatementExecutionException, IoTDBConnectionException { - session.executeNonQueryStatement( - "CREATE CONTINUOUS QUERY cq1 " - + "BEGIN SELECT max_value(s1) INTO temperature_max FROM root.sg1.* " - + "GROUP BY time(10s) END"); - session.executeNonQueryStatement( - "CREATE CONTINUOUS QUERY cq2 " - + "BEGIN SELECT count(s2) INTO temperature_cnt FROM root.sg1.* " - + "GROUP BY time(10s), level=1 END"); - session.executeNonQueryStatement( - "CREATE CONTINUOUS QUERY cq3 " - + "RESAMPLE EVERY 20s FOR 20s " - + "BEGIN SELECT avg(s3) INTO temperature_avg FROM root.sg1.* " - + "GROUP BY time(10s), level=1 END"); - session.executeNonQueryStatement("DROP CONTINUOUS QUERY cq1"); - session.executeNonQueryStatement("DROP CONTINUOUS QUERY cq2"); - session.executeNonQueryStatement("DROP CONTINUOUS QUERY cq3"); - } - - private static void createTimeseries() - throws IoTDBConnectionException, StatementExecutionException { - - if (!session.checkTimeseriesExists(ROOT_SG1_D1_S1)) { - session.createTimeseries( - ROOT_SG1_D1_S1, TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); - } - if (!session.checkTimeseriesExists(ROOT_SG1_D1_S2)) { - session.createTimeseries( - ROOT_SG1_D1_S2, TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); - } - if (!session.checkTimeseriesExists(ROOT_SG1_D1_S3)) { - session.createTimeseries( - ROOT_SG1_D1_S3, TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); - } - - // create timeseries with tags and attributes - if (!session.checkTimeseriesExists(ROOT_SG1_D1_S4)) { - Map tags = new HashMap<>(); - tags.put("tag1", "v1"); - Map attributes = new HashMap<>(); - attributes.put("description", "v1"); - session.createTimeseries( - ROOT_SG1_D1_S4, - TSDataType.INT64, - TSEncoding.RLE, - CompressionType.SNAPPY, - null, - tags, - attributes, - "temperature"); - } - - // create timeseries with SDT property, SDT will take place when flushing - if (!session.checkTimeseriesExists(ROOT_SG1_D1_S5)) { - // COMPDEV is required - // COMPMAXTIME and COMPMINTIME are optional and their unit is ms - Map props = new HashMap<>(); - props.put("LOSS", "sdt"); - props.put("COMPDEV", "0.01"); - props.put("COMPMINTIME", "2"); - props.put("COMPMAXTIME", "10"); - session.createTimeseries( - ROOT_SG1_D1_S5, - TSDataType.INT64, - TSEncoding.RLE, - CompressionType.SNAPPY, - props, - null, - null, - null); - } - } - - private static void createMultiTimeseries() - throws IoTDBConnectionException, StatementExecutionException { - - if (!session.checkTimeseriesExists("root.sg1.d2.s1") - && !session.checkTimeseriesExists("root.sg1.d2.s2")) { - List paths = new ArrayList<>(); - paths.add("root.sg1.d2.s1"); - paths.add("root.sg1.d2.s2"); - List tsDataTypes = new ArrayList<>(); - tsDataTypes.add(TSDataType.INT64); - tsDataTypes.add(TSDataType.INT64); - List tsEncodings = new ArrayList<>(); - tsEncodings.add(TSEncoding.RLE); - tsEncodings.add(TSEncoding.RLE); - List compressionTypes = new ArrayList<>(); - compressionTypes.add(CompressionType.SNAPPY); - compressionTypes.add(CompressionType.SNAPPY); - - List> tagsList = new ArrayList<>(); - Map tags = new HashMap<>(); - tags.put("unit", "kg"); - tagsList.add(tags); - tagsList.add(tags); - - List> attributesList = new ArrayList<>(); - Map attributes = new HashMap<>(); - attributes.put("minValue", "1"); - attributes.put("maxValue", "100"); - attributesList.add(attributes); - attributesList.add(attributes); - - List alias = new ArrayList<>(); - alias.add("weight1"); - alias.add("weight2"); - - session.createMultiTimeseries( - paths, tsDataTypes, tsEncodings, compressionTypes, null, tagsList, attributesList, alias); - } - } - - private static void createTemplate() - throws IoTDBConnectionException, StatementExecutionException, IOException { - - Template template = new Template("template1", false); - MeasurementNode mNodeS1 = - new MeasurementNode("s1", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); - MeasurementNode mNodeS2 = - new MeasurementNode("s2", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); - MeasurementNode mNodeS3 = - new MeasurementNode("s3", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); - - template.addToTemplate(mNodeS1); - template.addToTemplate(mNodeS2); - template.addToTemplate(mNodeS3); - - session.createSchemaTemplate(template); - session.setSchemaTemplate("template1", "root.sg1"); - } - - private static void insertRecord() throws IoTDBConnectionException, StatementExecutionException { - String deviceId = ROOT_SG1_D1; - List measurements = new ArrayList<>(); - List types = new ArrayList<>(); - measurements.add("s1"); - measurements.add("s2"); - measurements.add("s3"); - types.add(TSDataType.INT64); - types.add(TSDataType.INT64); - types.add(TSDataType.INT64); - - for (long time = 0; time < 100; time++) { - List values = new ArrayList<>(); - values.add(1L); - values.add(2L); - values.add(3L); - session.insertRecord(deviceId, time, measurements, types, values); - } - } - - private static void insertRecord4Redirect() - throws IoTDBConnectionException, StatementExecutionException { - for (int i = 0; i < 6; i++) { - for (int j = 0; j < 2; j++) { - String deviceId = "root.redirect" + i + ".d" + j; - List measurements = new ArrayList<>(); - measurements.add("s1"); - measurements.add("s2"); - measurements.add("s3"); - List types = new ArrayList<>(); - types.add(TSDataType.INT64); - types.add(TSDataType.INT64); - types.add(TSDataType.INT64); - - for (long time = 0; time < 5; time++) { - List values = new ArrayList<>(); - values.add(1L + time); - values.add(2L + time); - values.add(3L + time); - session.insertRecord(deviceId, time, measurements, types, values); - } - } - } - } - - private static void insertStrRecord() - throws IoTDBConnectionException, StatementExecutionException { - String deviceId = ROOT_SG1_D1; - List measurements = new ArrayList<>(); - measurements.add("s1"); - measurements.add("s2"); - measurements.add("s3"); - - for (long time = 0; time < 10; time++) { - List values = new ArrayList<>(); - values.add("1"); - values.add("2"); - values.add("3"); - session.insertRecord(deviceId, time, measurements, values); - } - } - - private static void insertRecordInObject() - throws IoTDBConnectionException, StatementExecutionException { - String deviceId = ROOT_SG1_D1; - List measurements = new ArrayList<>(); - List types = new ArrayList<>(); - measurements.add("s1"); - measurements.add("s2"); - measurements.add("s3"); - types.add(TSDataType.INT64); - types.add(TSDataType.INT64); - types.add(TSDataType.INT64); - - for (long time = 0; time < 100; time++) { - session.insertRecord(deviceId, time, measurements, types, 1L, 1L, 1L); - } - } - - private static void insertRecords() throws IoTDBConnectionException, StatementExecutionException { - String deviceId = ROOT_SG1_D1; - List measurements = new ArrayList<>(); - measurements.add("s1"); - measurements.add("s2"); - measurements.add("s3"); - List deviceIds = new ArrayList<>(); - List> measurementsList = new ArrayList<>(); - List> valuesList = new ArrayList<>(); - List timestamps = new ArrayList<>(); - List> typesList = new ArrayList<>(); - - for (long time = 0; time < 500; time++) { - List values = new ArrayList<>(); - List types = new ArrayList<>(); - values.add(1L); - values.add(2L); - values.add(3L); - types.add(TSDataType.INT64); - types.add(TSDataType.INT64); - types.add(TSDataType.INT64); - - deviceIds.add(deviceId); - measurementsList.add(measurements); - valuesList.add(values); - typesList.add(types); - timestamps.add(time); - if (time != 0 && time % 100 == 0) { - session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); - deviceIds.clear(); - measurementsList.clear(); - valuesList.clear(); - typesList.clear(); - timestamps.clear(); - } - } - - session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); - } - - /** - * insert the data of a device. For each timestamp, the number of measurements is the same. - * - *

Users need to control the count of Tablet and write a batch when it reaches the maxBatchSize - */ - private static void insertTablet() throws IoTDBConnectionException, StatementExecutionException { - /* - * A Tablet example: - * device1 - * time s1, s2, s3 - * 1, 1, 1, 1 - * 2, 2, 2, 2 - * 3, 3, 3, 3 - */ - // The schema of measurements of one device - // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); - schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); - schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); - schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); - - Tablet tablet = new Tablet(ROOT_SG1_D1, schemaList, 100); - - // Method 1 to add tablet data - long timestamp = System.currentTimeMillis(); - - for (long row = 0; row < 100; row++) { - int rowIndex = tablet.rowSize++; - tablet.addTimestamp(rowIndex, timestamp); - for (int s = 0; s < 3; s++) { - long value = random.nextLong(); - tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); - } - if (tablet.rowSize == tablet.getMaxRowNumber()) { - session.insertTablet(tablet, true); - tablet.reset(); - } - timestamp++; - } - - if (tablet.rowSize != 0) { - session.insertTablet(tablet); - tablet.reset(); - } - - // Method 2 to add tablet data - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - - for (long time = 0; time < 100; time++) { - int row = tablet.rowSize++; - timestamps[row] = time; - for (int i = 0; i < 3; i++) { - long[] sensor = (long[]) values[i]; - sensor[row] = i; - } - if (tablet.rowSize == tablet.getMaxRowNumber()) { - session.insertTablet(tablet, true); - tablet.reset(); - } - } - - if (tablet.rowSize != 0) { - session.insertTablet(tablet); - tablet.reset(); - } - } - - private static void insertTabletWithNullValues() - throws IoTDBConnectionException, StatementExecutionException { - /* - * A Tablet example: - * device1 - * time s1, s2, s3 - * 1, null, 1, 1 - * 2, 2, null, 2 - * 3, 3, 3, null - */ - // The schema of measurements of one device - // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); - schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); - schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); - schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); - - Tablet tablet = new Tablet(ROOT_SG1_D1, schemaList, 100); - - // Method 1 to add tablet data - insertTablet1(schemaList, tablet); - - // Method 2 to add tablet data - insertTablet2(schemaList, tablet); - } - - private static void insertTablet1(List schemaList, Tablet tablet) - throws IoTDBConnectionException, StatementExecutionException { - tablet.initBitMaps(); - - long timestamp = System.currentTimeMillis(); - for (long row = 0; row < 100; row++) { - int rowIndex = tablet.rowSize++; - tablet.addTimestamp(rowIndex, timestamp); - for (int s = 0; s < 3; s++) { - long value = random.nextLong(); - // mark null value - if (row % 3 == s) { - tablet.bitMaps[s].mark((int) row); - } - tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); - } - if (tablet.rowSize == tablet.getMaxRowNumber()) { - session.insertTablet(tablet, true); - tablet.reset(); - } - timestamp++; - } - - if (tablet.rowSize != 0) { - session.insertTablet(tablet); - tablet.reset(); - } - } - - private static void insertTablet2(List schemaList, Tablet tablet) - throws IoTDBConnectionException, StatementExecutionException { - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - BitMap[] bitMaps = new BitMap[schemaList.size()]; - for (int s = 0; s < 3; s++) { - bitMaps[s] = new BitMap(tablet.getMaxRowNumber()); - } - tablet.bitMaps = bitMaps; - - for (long time = 0; time < 100; time++) { - int row = tablet.rowSize++; - timestamps[row] = time; - for (int i = 0; i < 3; i++) { - long[] sensor = (long[]) values[i]; - // mark null value - if (row % 3 == i) { - bitMaps[i].mark(row); - } - sensor[row] = i; - } - if (tablet.rowSize == tablet.getMaxRowNumber()) { - session.insertTablet(tablet, true); - tablet.reset(); - } - } - - if (tablet.rowSize != 0) { - session.insertTablet(tablet); - tablet.reset(); - } - } - - private static void insertTablets() throws IoTDBConnectionException, StatementExecutionException { - // The schema of measurements of one device - // only measurementId and data type in MeasurementSchema take effects in Tablet - List schemaList = new ArrayList<>(); - schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); - schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); - schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); - - Tablet tablet1 = new Tablet(ROOT_SG1_D1, schemaList, 100); - Tablet tablet2 = new Tablet("root.sg1.d2", schemaList, 100); - Tablet tablet3 = new Tablet("root.sg1.d3", schemaList, 100); - - Map tabletMap = new HashMap<>(); - tabletMap.put(ROOT_SG1_D1, tablet1); - tabletMap.put("root.sg1.d2", tablet2); - tabletMap.put("root.sg1.d3", tablet3); - - // Method 1 to add tablet data - long timestamp = System.currentTimeMillis(); - for (long row = 0; row < 100; row++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; - int row3 = tablet3.rowSize++; - tablet1.addTimestamp(row1, timestamp); - tablet2.addTimestamp(row2, timestamp); - tablet3.addTimestamp(row3, timestamp); - for (int i = 0; i < 3; i++) { - long value = random.nextLong(); - tablet1.addValue(schemaList.get(i).getMeasurementId(), row1, value); - tablet2.addValue(schemaList.get(i).getMeasurementId(), row2, value); - tablet3.addValue(schemaList.get(i).getMeasurementId(), row3, value); - } - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { - session.insertTablets(tabletMap, true); - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); - } - timestamp++; - } - - if (tablet1.rowSize != 0) { - session.insertTablets(tabletMap, true); - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); - } - - // Method 2 to add tablet data - long[] timestamps1 = tablet1.timestamps; - Object[] values1 = tablet1.values; - long[] timestamps2 = tablet2.timestamps; - Object[] values2 = tablet2.values; - long[] timestamps3 = tablet3.timestamps; - Object[] values3 = tablet3.values; - - for (long time = 0; time < 100; time++) { - int row1 = tablet1.rowSize++; - int row2 = tablet2.rowSize++; - int row3 = tablet3.rowSize++; - timestamps1[row1] = time; - timestamps2[row2] = time; - timestamps3[row3] = time; - for (int i = 0; i < 3; i++) { - long[] sensor1 = (long[]) values1[i]; - sensor1[row1] = i; - long[] sensor2 = (long[]) values2[i]; - sensor2[row2] = i; - long[] sensor3 = (long[]) values3[i]; - sensor3[row3] = i; - } - if (tablet1.rowSize == tablet1.getMaxRowNumber()) { - session.insertTablets(tabletMap, true); - - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); - } - } - - if (tablet1.rowSize != 0) { - session.insertTablets(tabletMap, true); - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); - } - } - - /** - * This example shows how to insert data of TSDataType.TEXT. You can use the session interface to - * write data of String type or Binary type. - */ - private static void insertText() throws IoTDBConnectionException, StatementExecutionException { - String device = "root.sg1.text"; - // the first data is String type and the second data is Binary type - List datas = Arrays.asList("String", new Binary("Binary", TSFileConfig.STRING_CHARSET)); - // insertRecord example - for (int i = 0; i < datas.size(); i++) { - // write data of String type or Binary type - session.insertRecord( - device, - i, - Collections.singletonList("s1"), - Collections.singletonList(TSDataType.TEXT), - datas.get(i)); - } - - // insertTablet example - List schemaList = new ArrayList<>(); - schemaList.add(new MeasurementSchema("s2", TSDataType.TEXT)); - Tablet tablet = new Tablet(device, schemaList, 100); - for (int i = 0; i < datas.size(); i++) { - int rowIndex = tablet.rowSize++; - tablet.addTimestamp(rowIndex, i); - // write data of String type or Binary type - tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, datas.get(i)); - } - session.insertTablet(tablet); - try (SessionDataSet dataSet = session.executeQueryStatement("select s1, s2 from " + device)) { - System.out.println(dataSet.getColumnNames()); - while (dataSet.hasNext()) { - System.out.println(dataSet.next()); - } - } - } - - private static void selectInto() throws IoTDBConnectionException, StatementExecutionException { - session.executeNonQueryStatement( - "select s1, s2, s3 into into_s1, into_s2, into_s3 from root.sg1.d1"); - - try (SessionDataSet dataSet = - session.executeQueryStatement("select into_s1, into_s2, into_s3 from root.sg1.d1")) { - System.out.println(dataSet.getColumnNames()); - while (dataSet.hasNext()) { - System.out.println(dataSet.next()); - } - } - } - - private static void deleteData() throws IoTDBConnectionException, StatementExecutionException { - String path = ROOT_SG1_D1_S1; - long deleteTime = 99; - session.deleteData(path, deleteTime); - } - - private static void deleteTimeseries() - throws IoTDBConnectionException, StatementExecutionException { - List paths = new ArrayList<>(); - paths.add(ROOT_SG1_D1_S1); - paths.add(ROOT_SG1_D1_S2); - paths.add(ROOT_SG1_D1_S3); - session.deleteTimeseries(paths); - } - - private static void query() throws IoTDBConnectionException, StatementExecutionException { - try (SessionDataSet dataSet = session.executeQueryStatement(SELECT_D1)) { - System.out.println(dataSet.getColumnNames()); - dataSet.setFetchSize(1024); // default is 10000 - while (dataSet.hasNext()) { - System.out.println(dataSet.next()); - } - } - } - - private static void query4Redirect() - throws IoTDBConnectionException, StatementExecutionException { - String selectPrefix = "select * from root.redirect"; - for (int i = 0; i < 6; i++) { - try (SessionDataSet dataSet = - sessionEnableRedirect.executeQueryStatement(selectPrefix + i + ".d1")) { - - System.out.println(dataSet.getColumnNames()); - dataSet.setFetchSize(1024); // default is 10000 - while (dataSet.hasNext()) { - System.out.println(dataSet.next()); - } - } - } - - for (int i = 0; i < 6; i++) { - try (SessionDataSet dataSet = - sessionEnableRedirect.executeQueryStatement( - selectPrefix + i + ".d1 where time >= 1 and time < 10")) { - - System.out.println(dataSet.getColumnNames()); - dataSet.setFetchSize(1024); // default is 10000 - while (dataSet.hasNext()) { - System.out.println(dataSet.next()); - } - } - } - - for (int i = 0; i < 6; i++) { - try (SessionDataSet dataSet = - sessionEnableRedirect.executeQueryStatement( - selectPrefix + i + ".d1 where time >= 1 and time < 10 align by device")) { - - System.out.println(dataSet.getColumnNames()); - dataSet.setFetchSize(1024); // default is 10000 - while (dataSet.hasNext()) { - System.out.println(dataSet.next()); - } - } - } - - for (int i = 0; i < 6; i++) { - try (SessionDataSet dataSet = - sessionEnableRedirect.executeQueryStatement( - selectPrefix - + i - + ".d1 where time >= 1 and time < 10 and root.redirect" - + i - + ".d1.s1 > 1")) { - System.out.println(dataSet.getColumnNames()); - dataSet.setFetchSize(1024); // default is 10000 - while (dataSet.hasNext()) { - System.out.println(dataSet.next()); - } - } - } - } - - private static void queryWithTimeout() - throws IoTDBConnectionException, StatementExecutionException { - try (SessionDataSet dataSet = session.executeQueryStatement(SELECT_D1, 2000)) { - System.out.println(dataSet.getColumnNames()); - dataSet.setFetchSize(1024); // default is 10000 - while (dataSet.hasNext()) { - System.out.println(dataSet.next()); - } - } - } - - private static void rawDataQuery() throws IoTDBConnectionException, StatementExecutionException { - List paths = new ArrayList<>(); - paths.add(ROOT_SG1_D1_S1); - paths.add(ROOT_SG1_D1_S2); - paths.add(ROOT_SG1_D1_S3); - long startTime = 10L; - long endTime = 200L; - long timeOut = 60000; - - try (SessionDataSet dataSet = session.executeRawDataQuery(paths, startTime, endTime, timeOut)) { - - System.out.println(dataSet.getColumnNames()); - dataSet.setFetchSize(1024); - while (dataSet.hasNext()) { - System.out.println(dataSet.next()); - } - } - } - - private static void lastDataQuery() throws IoTDBConnectionException, StatementExecutionException { - List paths = new ArrayList<>(); - paths.add(ROOT_SG1_D1_S1); - paths.add(ROOT_SG1_D1_S2); - paths.add(ROOT_SG1_D1_S3); - try (SessionDataSet sessionDataSet = session.executeLastDataQuery(paths, 3, 60000)) { - System.out.println(sessionDataSet.getColumnNames()); - sessionDataSet.setFetchSize(1024); - while (sessionDataSet.hasNext()) { - System.out.println(sessionDataSet.next()); - } - } - } - - private static void fastLastDataQueryForOneDevice() - throws IoTDBConnectionException, StatementExecutionException { - System.out.println("-------fastLastQuery------"); - List paths = new ArrayList<>(); - paths.add("s1"); - paths.add("s2"); - paths.add("s3"); - try (SessionDataSet sessionDataSet = - sessionEnableRedirect.executeLastDataQueryForOneDevice( - ROOT_SG1, ROOT_SG1_D1, paths, true)) { - System.out.println(sessionDataSet.getColumnNames()); - sessionDataSet.setFetchSize(1024); - while (sessionDataSet.hasNext()) { - System.out.println(sessionDataSet.next()); - } - } - } - - private static void aggregationQuery() - throws IoTDBConnectionException, StatementExecutionException { - List paths = new ArrayList<>(); - paths.add(ROOT_SG1_D1_S1); - paths.add(ROOT_SG1_D1_S2); - paths.add(ROOT_SG1_D1_S3); - - List aggregations = new ArrayList<>(); - aggregations.add(TAggregationType.COUNT); - aggregations.add(TAggregationType.SUM); - aggregations.add(TAggregationType.MAX_VALUE); - try (SessionDataSet sessionDataSet = session.executeAggregationQuery(paths, aggregations)) { - System.out.println(sessionDataSet.getColumnNames()); - sessionDataSet.setFetchSize(1024); - while (sessionDataSet.hasNext()) { - System.out.println(sessionDataSet.next()); - } - } - } - - private static void groupByQuery() throws IoTDBConnectionException, StatementExecutionException { - List paths = new ArrayList<>(); - paths.add(ROOT_SG1_D1_S1); - paths.add(ROOT_SG1_D1_S2); - paths.add(ROOT_SG1_D1_S3); - - List aggregations = new ArrayList<>(); - aggregations.add(TAggregationType.COUNT); - aggregations.add(TAggregationType.SUM); - aggregations.add(TAggregationType.MAX_VALUE); - try (SessionDataSet sessionDataSet = - session.executeAggregationQuery(paths, aggregations, 0, 100, 10, 20)) { - System.out.println(sessionDataSet.getColumnNames()); - sessionDataSet.setFetchSize(1024); - while (sessionDataSet.hasNext()) { - System.out.println(sessionDataSet.next()); - } - } - } - - private static void queryByIterator() - throws IoTDBConnectionException, StatementExecutionException { - try (SessionDataSet dataSet = session.executeQueryStatement(SELECT_D1)) { - - DataIterator iterator = dataSet.iterator(); - System.out.println(dataSet.getColumnNames()); - dataSet.setFetchSize(1024); // default is 10000 - while (iterator.next()) { - StringBuilder builder = new StringBuilder(); - // get time - builder.append(iterator.getLong(1)).append(","); - // get second column - if (!iterator.isNull(2)) { - builder.append(iterator.getLong(2)).append(","); - } else { - builder.append("null").append(","); - } - - // get third column - if (!iterator.isNull(ROOT_SG1_D1_S2)) { - builder.append(iterator.getLong(ROOT_SG1_D1_S2)).append(","); - } else { - builder.append("null").append(","); - } - - // get forth column - if (!iterator.isNull(4)) { - builder.append(iterator.getLong(4)).append(","); - } else { - builder.append("null").append(","); - } - - // get fifth column - if (!iterator.isNull(ROOT_SG1_D1_S4)) { - builder.append(iterator.getObject(ROOT_SG1_D1_S4)); - } else { - builder.append("null"); - } - - System.out.println(builder); - } - } - } - - private static void nonQuery() throws IoTDBConnectionException, StatementExecutionException { - session.executeNonQueryStatement("insert into root.sg1.d1(timestamp,s1) values(200, 1)"); - } - - private static void setTimeout() throws IoTDBConnectionException { - try (Session tempSession = new Session(LOCAL_HOST, 6667, "root", "root", 10000, 20000)) { - tempSession.setQueryTimeout(60000); - } - } -} diff --git a/iotdb-client/client-py/setup.py b/iotdb-client/client-py/setup.py deleted file mode 100644 index 731e9a14a62a..000000000000 --- a/iotdb-client/client-py/setup.py +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import setuptools -import io - - -try: - with io.open("README.md", encoding="utf-8") as f: - long_description = f.read() -except FileNotFoundError: - long_description = "" - - -print(long_description) - -setuptools.setup( - name="apache-iotdb", # Replace with your own username - version="1.3.3.dev0", - author=" Apache Software Foundation", - author_email="dev@iotdb.apache.org", - description="Apache IoTDB client API", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/apache/iotdb", - packages=setuptools.find_packages(), - install_requires=[ - "thrift>=0.14.1", - "pandas>=1.3.5", - "numpy>=1.21.4", - "sqlalchemy<1.5,>=1.4", - "sqlalchemy-utils>=0.37.8", - ], - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - python_requires=">=3.7", - license="Apache License, Version 2.0", - website="https://iotdb.apache.org", - entry_points={ - "sqlalchemy.dialects": [ - "iotdb = iotdb.sqlalchemy.IoTDBDialect:IoTDBDialect", - ], - }, -) From 94185259ead7de91349306e5b8e32c9b90d41c41 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 20 Nov 2024 14:34:14 +0800 Subject: [PATCH 03/12] add some ITs --- ...DBUncorrelatedSubqueryInWhereClauseIT.java | 269 ++++++++++++++++++ .../recent/subquery/SubqueryDataSetUtils.java | 110 +++++++ .../process/EnforceSingleRowOperator.java | 115 ++++++++ .../plan/planner/TableOperatorGenerator.java | 19 ++ .../TableDistributedPlanGenerator.java | 13 + .../PushPredicateIntoTableScan.java | 24 +- 6 files changed, 546 insertions(+), 4 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedSubqueryInWhereClauseIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedSubqueryInWhereClauseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedSubqueryInWhereClauseIT.java new file mode 100644 index 000000000000..a71b1964899a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedSubqueryInWhereClauseIT.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUncorrelatedSubqueryInWhereClauseIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testScalarSubqueryAfterComparisonInOneTable() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: s equals to the maximum value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 WHERE device_id = 'd01')"; + retArray = new String[] {"70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s not equals to the maximum value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != ((SELECT max(%s) FROM table1 WHERE device_id = 'd01'))"; + retArray = new String[] {"30,", "40,", "50,", "60,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the average value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= ((SELECT AVG(%s) FROM table1 WHERE device_id = 'd01'))"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the max value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > ((SELECT max(%s) FROM table1 WHERE device_id = 'd01'))"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s is less than the maximum value of s in table1 and greater than the minimum value + // of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < (SELECT max(%s) from table1 WHERE device_id = 'd01') and %s > (SELECT min(%s) from table1 WHERE device_id = 'd01') "; + retArray = new String[] {"40,", "50,", "60,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the avg value of s in table1 and s5 = true + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > ((SELECT avg(%s) FROM table1 WHERE device_id = 'd01' and s5 = true))"; + retArray = new String[] {"60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the count value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > (SELECT count(%s) FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s less than the sum value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < (SELECT sum(%s) FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testScalarSubqueryAfterComparisonInDifferentTables() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: s greater than the count value of s in table2 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > (SELECT count(%s) from table2)"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s less than the max value of s in table2 * the count value of s in table2 * 10 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < ((SELECT max(%s) from table2) * (SELECT count(%s) from table2)) * 10"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testNestedScalarSubqueryAfterComparison() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: nested scalar subquery in where clause + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 where %s = (SELECT max(%s) from table1 WHERE device_id = 'd01'))"; + retArray = new String[] {"70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: nested scalar subquery with table subquery + sql = + "SELECT %s from (SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 where %s = (SELECT max(%s) from table1 WHERE device_id = 'd01')))"; + retArray = new String[] {"70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testScalarSubqueryAfterComparisonLegalityCheck() { + // Legality check: subquery returns multiple rows (should fail) + tableAssertTestFail( + "select s1 from table1 where s1 = (select s1 from table1)", + "301: Scalar sub-query has returned multiple rows.", + DATABASE_NAME); + + // Legality check: subquery can not be parsed + tableAssertTestFail( + "select s1 from table1 where s1 = (select s1 from)", "mismatched input", DATABASE_NAME); + + // Legality check: subquery can not be parsed(without parentheses) + tableAssertTestFail( + "select s1 from table1 where s1 = select s1 from table1", + "mismatched input", + DATABASE_NAME); + + // Legality check: Main query can not be parsed + tableAssertTestFail( + "select s1 from table1 where s1 = (select max(s1) from table1) and", + "mismatched input", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java new file mode 100644 index 000000000000..1bd779aac65d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery; + +public class SubqueryDataSetUtils { + protected static final String DATABASE_NAME = "subqueryTest"; + protected static final String[] NUMERIC_MEASUREMENTS = new String[] {"s1", "s2", "s3", "s4"}; + protected static final String[] CREATE_SQLS = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + // table1 + "CREATE TABLE table1(province STRING ID, city STRING ID, region STRING ID, device_id STRING ID, color STRING ATTRIBUTE, type STRING ATTRIBUTE, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT, s5 BOOLEAN MEASUREMENT, s6 TEXT MEASUREMENT, s7 STRING MEASUREMENT, s8 BLOB MEASUREMENT, s9 TIMESTAMP MEASUREMENT, s10 DATE MEASUREMENT)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',40,40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',50,50,50.0,50.0,true,'shanghai_huangpu_red_A_d01_50','shanghai_huangpu_red_A_d01_50',X'cafebabe50',2024-09-24T06:15:00.000+00:00,'2024-09-25')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:16:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',60,60,60.0,60.0,false,'shanghai_huangpu_red_A_d01_60','shanghai_huangpu_red_A_d01_60',X'cafebabe60',2024-09-24T06:16:00.000+00:00,'2024-09-26')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:17:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',70,70,70.0,70.0,true,'shanghai_huangpu_red_A_d01_70','shanghai_huangpu_red_A_d01_70',X'cafebabe70',2024-09-24T06:17:00.000+00:00,'2024-09-27')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',36,true,'shanghai_huangpu_red_B_d02_36','shanghai_huangpu_red_B_d02_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',40,40.0,'shanghai_huangpu_red_B_d02_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',50000,'shanghai_huangpu_red_B_d02_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',36,36.0,'shanghai_huangpu_yellow_A_d03_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',41,41.0,false,'shanghai_huangpu_yellow_A_d03_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',46000,46.0,'shanghai_huangpu_yellow_A_d03_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',51.0,'shanghai_huangpu_yellow_A_d03_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',30.0,true,'shanghai_huangpu_yellow_B_d04_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',55,55.0,'shanghai_huangpu_yellow_B_d04_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong','d05','red','A',30,30.0,'shanghai_pudong_red_A_d05_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'shanghai','shanghai','pudong','d05','red','A',35000,35.0,35.0,'shanghai_pudong_red_A_d05_35','shanghai_pudong_red_A_d05_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d05','red','A',40,40.0,true,'shanghai_pudong_red_A_d05_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong','d05','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong','d05','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',36,true,'shanghai_pudong_red_B_d06_36','shanghai_pudong_red_B_d06_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',40,40.0,'shanghai_pudong_red_B_d06_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',50000,'shanghai_pudong_red_B_d06_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',36,36.0,'shanghai_pudong_yellow_A_d07_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',41,41.0,false,'shanghai_pudong_yellow_A_d07_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',46000,46.0,'shanghai_pudong_yellow_A_d07_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',51.0,'shanghai_pudong_yellow_A_d07_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',30.0,true,'shanghai_pudong_yellow_B_d08_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',55,55.0,'shanghai_pudong_yellow_B_d08_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang','d09','red','A',30,30.0,'beijing_chaoyang_red_A_d09_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','chaoyang','d09','red','A',35000,35.0,35.0,'beijing_chaoyang_red_A_d09_35','beijing_chaoyang_red_A_d09_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d09','red','A',40,40.0,true,'beijing_chaoyang_red_A_d09_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang','d09','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang','d09','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',36,true,'beijing_chaoyang_red_B_d10_36','beijing_chaoyang_red_B_d10_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',40,40.0,'beijing_chaoyang_red_B_d10_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',50000,'beijing_chaoyang_red_B_d10_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',36,36.0,'beijing_chaoyang_yellow_A_d11_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',41,41.0,false,'beijing_chaoyang_yellow_A_d11_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',46000,46.0,'beijing_chaoyang_yellow_A_d11_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',51.0,'beijing_chaoyang_yellow_A_d11_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',30.0,true,'beijing_chaoyang_yellow_B_d12_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',55,55.0,'beijing_chaoyang_yellow_B_d12_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian','d13','red','A',30,30.0,'beijing_haidian_red_A_d13_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','haidian','d13','red','A',35000,35.0,35.0,'beijing_haidian_red_A_d13_35','beijing_haidian_red_A_d13_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d13','red','A',40,40.0,true,'beijing_haidian_red_A_d13_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian','d13','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian','d13','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',36,true,'beijing_haidian_red_B_d14_36','beijing_haidian_red_B_d14_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',40,40.0,'beijing_haidian_red_B_d14_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',50000,'beijing_haidian_red_B_d14_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','haidian','d15','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian','d15','yellow','A',36,36.0,'beijing_haidian_yellow_A_d15_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','haidian','d15','yellow','A',41,41.0,false,'beijing_haidian_yellow_A_d15_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','haidian','d15','yellow','A',46000,46.0,'beijing_haidian_yellow_A_d15_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','haidian','d15','yellow','A',51.0,'beijing_haidian_yellow_A_d15_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',30.0,true,'beijing_haidian_yellow_B_d16_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',55,55.0,'beijing_haidian_yellow_B_d16_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + // table2 + "CREATE TABLE table2(device_id STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT, s5 BOOLEAN MEASUREMENT, s6 TEXT MEASUREMENT, s7 STRING MEASUREMENT, s8 BLOB MEASUREMENT, s9 TIMESTAMP MEASUREMENT, s10 DATE MEASUREMENT)", + "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) " + + " values(1, 'd1', 1, 11, 1.1, 11.1, true, 'text1', 'string1', X'cafebabe01', 1, '2024-10-01')", + "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5) " + + " values(2, 'd1', 2, 22, 2.2, 22.2, false)", + "INSERT INTO table2(time,device_id,s6,s7,s8,s9,s10) " + + " values(3, 'd1', 'text3', 'string3', X'cafebabe03', 3, '2024-10-03')", + "INSERT INTO table2(time,device_id,s6,s7,s8,s9,s10) " + + " values(4, 'd1', 'text4', 'string4', X'cafebabe04', 4, '2024-10-04')", + "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5) " + + " values(5, 'd1', 5, 55, 5.5, 55.5, false)", + "FLUSH", + "CLEAR ATTRIBUTE CACHE", + }; +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java new file mode 100644 index 000000000000..891612f5966c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process; + +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.queryengine.execution.operator.Operator; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.utils.RamUsageEstimator; + +public class EnforceSingleRowOperator implements ProcessOperator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(EnforceSingleRowOperator.class); + + private static final String ERROR_MESSAGE = "Scalar sub-query has returned multiple rows."; + + private final OperatorContext operatorContext; + private final Operator child; + + private boolean finished = false; + + public EnforceSingleRowOperator(OperatorContext operatorContext, Operator child) { + this.operatorContext = operatorContext; + this.child = child; + } + + @Override + public ListenableFuture isBlocked() { + return child.isBlocked(); + } + + @Override + public TsBlock next() throws Exception { + if (finished) { + throw new IllegalStateException(ERROR_MESSAGE); + } + TsBlock tsBlock = child.next(); + if (tsBlock != null && tsBlock.getPositionCount() > 1) { + throw new IllegalStateException(ERROR_MESSAGE); + } + if (tsBlock != null) { + finished = true; + } + return tsBlock; + } + + @Override + public boolean hasNext() throws Exception { + boolean hasNext = child.hasNext(); + if (finished && hasNext) { + throw new IllegalStateException(ERROR_MESSAGE); + } + return hasNext; + } + + @Override + public void close() throws Exception { + if (child != null) { + child.close(); + } + } + + @Override + public boolean isFinished() throws Exception { + return finished || child.isFinished(); + } + + @Override + public long calculateMaxPeekMemory() { + return child.calculateMaxPeekMemory(); + } + + @Override + public long calculateMaxReturnSize() { + return child.calculateMaxReturnSize() + / TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber(); + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + return child.calculateRetainedSizeAfterCallingNext(); + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(child) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext); + } + + @Override + public OperatorContext getOperatorContext() { + return operatorContext; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index c41063943c04..5941950d4973 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; import org.apache.iotdb.db.queryengine.execution.operator.process.CollectOperator; +import org.apache.iotdb.db.queryengine.execution.operator.process.EnforceSingleRowOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.FilterAndProjectOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.LimitOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.OffsetOperator; @@ -110,6 +111,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; @@ -176,6 +178,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.commons.conf.IoTDBConstant.TIME; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.MEASUREMENT; import static org.apache.iotdb.db.queryengine.common.DataNodeEndPoints.isSameNode; import static org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.MergeSortComparator.getComparatorForTable; @@ -1189,6 +1192,7 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { rightOutputSymbolIdx[i] = node.getRightChild().getOutputSymbols().indexOf(node.getRightOutputSymbols().get(i)); } + // cross join does not need time column if (node.isCrossJoin()) { return new SimpleNestedLoopCrossJoinOperator( @@ -1231,6 +1235,21 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { throw new IllegalStateException("Unsupported join type: " + node.getJoinType()); } + @Override + public Operator visitEnforceSingleRow( + EnforceSingleRowNode node, LocalExecutionPlanContext context) { + Operator child = node.getChild().accept(this, context); + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + LimitOperator.class.getSimpleName()); + + return new EnforceSingleRowOperator(operatorContext, child); + } + @Override public Operator visitCountMerge( final CountSchemaMergeNode node, final LocalExecutionPlanContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 884508534e01..3f1a08f10814 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; @@ -620,6 +621,18 @@ public List visitAggregationTableScan( return resultTableScanNodeList; } + @Override + public List visitEnforceSingleRow(EnforceSingleRowNode node, PlanContext context) { + List childrenNodes = node.getChild().accept(this, context); + OrderingScheme childOrdering = nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); + if (childOrdering != null) { + nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); + } + + node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); + return Collections.singletonList(node); + } + private void buildRegionNodeMap( AggregationTableScanNode originalAggTableScanNode, List> regionReplicaSetsList, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index a2166e1a1ca2..25f5cbf2159b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -585,10 +586,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { List equiJoinClauses = new ArrayList<>(); ImmutableList.Builder joinFilterBuilder = ImmutableList.builder(); for (Expression conjunct : extractConjuncts(newJoinPredicate)) { - if (joinEqualityExpression( - conjunct, - node.getLeftChild().getOutputSymbols(), - node.getRightChild().getOutputSymbols())) { + if (joinEqualityExpressionOnTimeColumn(conjunct, node)) { ComparisonExpression equality = (ComparisonExpression) conjunct; boolean alignedComparison = @@ -724,6 +722,24 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { return output; } + private boolean joinEqualityExpressionOnTimeColumn(Expression conjunct, JoinNode node) { + if (!joinEqualityExpression( + conjunct, + node.getLeftChild().getOutputSymbols(), + node.getRightChild().getOutputSymbols())) { + return false; + } + // conjunct must be a comparison expression + ComparisonExpression equality = (ComparisonExpression) conjunct; + // After Optimization, some subqueries are transformed into Join. + // For now, we only support join on time, so we need to use FilterOperator + CrossJoinOperator + // to simulate the join with equality criteria on columns other than time column. + // todo: after supporting join on other columns, we need to remove the following code, this is + // temporary walkaround. + return IoTDBConstant.TIME.equalsIgnoreCase(equality.getLeft().toString()) + || IoTDBConstant.TIME.equalsIgnoreCase(equality.getRight().toString()); + } + private Symbol symbolForExpression(Expression expression) { if (expression instanceof SymbolReference) { return Symbol.from(expression); From 7808a2a427b7a6e60dd21729ee0c0f156cd449ea Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 20 Nov 2024 18:16:07 +0800 Subject: [PATCH 04/12] add ser/deser methods of some PlanNodes --- .../join/SimpleNestedLoopCrossJoinOperator.java | 2 +- .../plan/planner/plan/node/PlanNodeType.java | 4 ++++ .../plan/relational/planner/node/ApplyNode.java | 4 ++++ .../relational/planner/node/CorrelatedJoinNode.java | 4 ++++ .../relational/planner/node/EnforceSingleRowNode.java | 10 ++++++++-- 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java index e4847e12fce5..ef53a438bf45 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java @@ -205,7 +205,7 @@ public boolean isFinished() throws Exception { return false; } - return (cachedProbeBlock != null && !cachedProbeBlock.isEmpty()) + return (cachedProbeBlock == null || cachedProbeBlock.isEmpty()) && probeSource.isFinished() && buildFinished; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 75deadd23387..99d1d54300a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -115,6 +115,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; @@ -275,6 +276,7 @@ public enum PlanNodeType { TABLE_AGGREGATION_NODE((short) 1015), TABLE_AGGREGATION_TABLE_SCAN_NODE((short) 1016), TABLE_GAP_FILL_NODE((short) 1017), + TABLE_ENFORCE_SINGLE_ROW_NODE((short) 1018), RELATIONAL_INSERT_TABLET((short) 2000), RELATIONAL_INSERT_ROW((short) 2001), @@ -619,6 +621,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { .deserialize(buffer); case 1017: return GapFillNode.deserialize(buffer); + case 1018: + return EnforceSingleRowNode.deserialize(buffer); case 2000: return RelationalInsertTabletNode.deserialize(buffer); case 2001: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ApplyNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ApplyNode.java index 4574bf9d254b..5a9c5817eaef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ApplyNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ApplyNode.java @@ -147,11 +147,15 @@ public List getOutputColumnNames() { @Override protected void serializeAttributes(ByteBuffer byteBuffer) { + // ApplyNode should be transformed to other nodes after planning, so serialization is not + // expected. throw new UnsupportedOperationException(); } @Override protected void serializeAttributes(DataOutputStream stream) throws IOException { + // ApplyNode should be transformed to other nodes after planning, so serialization is not + // expected. throw new UnsupportedOperationException(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CorrelatedJoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CorrelatedJoinNode.java index 64e14b2d15a1..bbee7e5d68fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CorrelatedJoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CorrelatedJoinNode.java @@ -156,11 +156,15 @@ public List getOutputColumnNames() { @Override protected void serializeAttributes(ByteBuffer byteBuffer) { + // CorrelatedJoinNode should be transformed to other nodes after planning, so serialization is + // not expected. throw new UnsupportedOperationException(); } @Override protected void serializeAttributes(DataOutputStream stream) throws IOException { + // CorrelatedJoinNode should be transformed to other nodes after planning, so serialization is + // not expected. throw new UnsupportedOperationException(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/EnforceSingleRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/EnforceSingleRowNode.java index d67a47a1eba3..734f6c1a2323 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/EnforceSingleRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/EnforceSingleRowNode.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; @@ -70,11 +71,16 @@ public List getOutputColumnNames() { @Override protected void serializeAttributes(ByteBuffer byteBuffer) { - throw new UnsupportedOperationException(); + PlanNodeType.TABLE_ENFORCE_SINGLE_ROW_NODE.serialize(byteBuffer); } @Override protected void serializeAttributes(DataOutputStream stream) throws IOException { - throw new UnsupportedOperationException(); + PlanNodeType.TABLE_ENFORCE_SINGLE_ROW_NODE.serialize(stream); + } + + public static EnforceSingleRowNode deserialize(ByteBuffer byteBuffer) { + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); + return new EnforceSingleRowNode(planNodeId, null); } } From 6ba9c6d8e5e411cfc84985f49dd561d885f6fb6e Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 27 Nov 2024 15:20:30 +0800 Subject: [PATCH 05/12] self check and fix some typos --- .../operator/process/EnforceSingleRowOperator.java | 12 ++++++------ .../join/SimpleNestedLoopCrossJoinOperator.java | 4 ++-- .../plan/planner/TableOperatorGenerator.java | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java index dc468f1143e2..e9f6fba26f68 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java @@ -35,8 +35,7 @@ public class EnforceSingleRowOperator implements ProcessOperator { private static final String MULTIPLE_ROWS_ERROR_MESSAGE = "Scalar sub-query has returned multiple rows."; - private static final String NO_RESULT_ERROR_MESSAGE = - "Scalar sub-query has returned multiple rows."; + private static final String NO_RESULT_ERROR_MESSAGE = "Scalar sub-query does not have output."; private final OperatorContext operatorContext; private final Operator child; @@ -56,12 +55,13 @@ public ListenableFuture isBlocked() { @Override public TsBlock next() throws Exception { TsBlock tsBlock = child.next(); - if (tsBlock != null && (tsBlock.getPositionCount() > 1 || finished)) { - throw new IllegalStateException(MULTIPLE_ROWS_ERROR_MESSAGE); + if (tsBlock == null || tsBlock.isEmpty()) { + return tsBlock; } - if (tsBlock != null) { - finished = true; + if (tsBlock.getPositionCount() > 1 || finished) { + throw new IllegalStateException(MULTIPLE_ROWS_ERROR_MESSAGE); } + finished = true; return tsBlock; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java index ef53a438bf45..54e99570cfe7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java @@ -51,10 +51,10 @@ public class SimpleNestedLoopCrossJoinOperator extends AbstractOperator { private final Operator probeSource; - // cache the result of buildSource, for now, we assume that the buildChild produces a small number - // of TsBlocks private final Operator buildSource; + // cache the result of buildSource, for now, we assume that the buildChild produces a small number + // of TsBlocks private final List buildBlocks; private final TsBlockBuilder resultBuilder; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 97d2ca93613e..0d84a7fd16e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -1301,7 +1301,7 @@ public Operator visitEnforceSingleRow( .addOperatorContext( context.getNextOperatorId(), node.getPlanNodeId(), - LimitOperator.class.getSimpleName()); + EnforceSingleRowOperator.class.getSimpleName()); return new EnforceSingleRowOperator(operatorContext, child); } From d312d96f3a84038fcbf9e4a119e9e5306558e0cd Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 27 Nov 2024 17:57:55 +0800 Subject: [PATCH 06/12] fix ITs --- .../IoTDBMultiIDsWithAttributesTableIT.java | 23 +++++++--------- .../query/recent/IoTDBTableAggregationIT.java | 2 +- .../SimpleNestedLoopCrossJoinOperator.java | 27 +++++++++---------- 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java index e0a7dad4dc51..6bac8a5e0780 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java @@ -34,7 +34,6 @@ import java.sql.Statement; import java.util.Arrays; -import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.fail; @@ -154,14 +153,6 @@ public class IoTDBMultiIDsWithAttributesTableIT { String[] retArray; static String sql; - // public static void main(String[] args) { - // for (String[] sqlList : Arrays.asList(sql4, sql5)) { - // for (String sql : sqlList) { - // System.out.println(sql); - // } - // } - // } - @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(1024 * 1024L); @@ -1694,17 +1685,21 @@ public void fourTableJoinTest() { + "order by s.student_id, t.teacher_id, c.course_id,g.grade_id"; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = new String[] {"region", "name", "teacher_id", "course_name", "score"}; + + retArray = + new String[] { + "haidian,Lucy,1005,数学,99,", + }; sql = "select s.region, s.name," + " t.teacher_id," + " c.course_name," + " g.score " + "from students s, teachers t, courses c, grades g " - + "where s.time=c.time and c.time=g.time"; - tableAssertTestFail( - sql, - "701: Cross join is not supported in current version, each table must have at least one equiJoinClause", - DATABASE_NAME); + + "where s.time=c.time and c.time=g.time and t.teacher_id = 1005 limit 1"; + + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index 5ae3ce3862c6..ae32bb241f64 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -3713,7 +3713,7 @@ public void modeTest() { public void exceptionTest() { tableAssertTestFail( "select s1 from table1 where s2 in (select s2 from table1)", - "701: Only TableSubquery is supported now", + "Not a valid IR expression", DATABASE_NAME); tableAssertTestFail( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java index 54e99570cfe7..24f497a444c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java @@ -122,9 +122,7 @@ public TsBlock next() throws Exception { while (probeIndex < cachedProbeBlock.getPositionCount() && System.nanoTime() - start < maxRuntime) { for (TsBlock buildBlock : buildBlocks) { - for (int i = 0; i < buildBlock.getPositionCount(); i++) { - appendValueToResult(probeIndex, buildBlock); - } + appendValueToResult(probeIndex, buildBlock); } probeIndex++; } @@ -141,26 +139,27 @@ public TsBlock next() throws Exception { } private void appendValueToResult(int probeIndex, TsBlock buildBlock) { - for (int i = 0; i < buildBlock.getPositionCount(); i++) { - for (int j = 0; j < leftOutputSymbolIdx.length; j++) { - ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(j); - if (cachedProbeBlock.getColumn(leftOutputSymbolIdx[j]).isNull(probeIndex)) { + for (int i = 0; i < leftOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(i); + for (int j = 0; j < buildBlock.getPositionCount(); j++) { + if (cachedProbeBlock.getColumn(leftOutputSymbolIdx[i]).isNull(probeIndex)) { columnBuilder.appendNull(); } else { - columnBuilder.write(cachedProbeBlock.getColumn(leftOutputSymbolIdx[j]), probeIndex); + columnBuilder.write(cachedProbeBlock.getColumn(leftOutputSymbolIdx[i]), probeIndex); } } - for (int j = 0; j < rightOutputSymbolIdx.length; j++) { - ColumnBuilder columnBuilder = - resultBuilder.getColumnBuilder(leftOutputSymbolIdx.length + j); - if (buildBlock.getColumn(rightOutputSymbolIdx[j]).isNull(i)) { + } + for (int i = 0; i < rightOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(i + leftOutputSymbolIdx.length); + for (int j = 0; j < buildBlock.getPositionCount(); j++) { + if (buildBlock.getColumn(rightOutputSymbolIdx[i]).isNull(j)) { columnBuilder.appendNull(); } else { - columnBuilder.write(buildBlock.getColumn(rightOutputSymbolIdx[j]), i); + columnBuilder.write(buildBlock.getColumn(rightOutputSymbolIdx[i]), j); } } } - resultBuilder.declarePosition(); + resultBuilder.declarePositions(buildBlock.getPositionCount()); } @Override From ed5f5e254b39fad0b6e1899a54bd3c230ff83f07 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 27 Nov 2024 19:42:18 +0800 Subject: [PATCH 07/12] remove temporary workaround in pushAggregationIntoTableScan --- .../planner/optimizations/PushAggregationIntoTableScan.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java index 1b7a7f5db402..8bdef50cdbeb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java @@ -152,10 +152,6 @@ private PushDownLevel calculatePushDownLevel( // calculate DataSet part boolean singleDeviceEntry = tableScanNode.getDeviceEntries().size() < 2; if (groupingKeys.isEmpty()) { - // fixme: we don't expect ProjectNode here. Temporary workaround. - if (projectNode != null) { - return PushDownLevel.NOOP; - } // GlobalAggregation if (singleDeviceEntry) { return PushDownLevel.COMPLETE; From 99565fede171f5a38e1a35691337cc553311d68c Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Fri, 29 Nov 2024 11:46:03 +0800 Subject: [PATCH 08/12] fix some of the comments --- .../TableDistributedPlanGenerator.java | 8 +++++-- .../optimizations/LogicalOptimizeFactory.java | 4 +++- .../optimizations/QueryCardinalityUtil.java | 21 ++++++++++++++----- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index ca24d3642f71..853afa8a6da0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -433,8 +433,12 @@ public List visitJoin(JoinNode node, PlanContext context) { } // For CrossJoinNode, we need to merge children nodes(It's safe for other JoinNodes here since // the size of their children is always 1.) - node.setLeftChild(mergeChildrenViaCollectOrMergeSort(null, leftChildrenNodes)); - node.setRightChild(mergeChildrenViaCollectOrMergeSort(null, rightChildrenNodes)); + node.setLeftChild( + mergeChildrenViaCollectOrMergeSort( + nodeOrderingMap.get(node.getLeftChild().getPlanNodeId()), leftChildrenNodes)); + node.setRightChild( + mergeChildrenViaCollectOrMergeSort( + nodeOrderingMap.get(node.getRightChild().getPlanNodeId()), rightChildrenNodes)); return Collections.singletonList(node); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 5c166d0493ff..304917842877 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -154,7 +154,7 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new InlineProjections(plannerContext), new RemoveRedundantIdentityProjections(), new MergeLimits(), - new RemoveTrivialFilters() + new RemoveTrivialFilters(), // new RemoveRedundantLimit(), // new RemoveRedundantOffset(), // new RemoveRedundantSort(), @@ -164,6 +164,7 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { // new ReplaceRedundantJoinWithSource(), // new RemoveRedundantJoin(), // new ReplaceRedundantJoinWithProject(), + new RemoveRedundantEnforceSingleRowNode() // new RemoveRedundantExists(), // new RemoveRedundantWindow(), // new SingleDistinctAggregationToGroupBy(), @@ -205,6 +206,7 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new RemoveRedundantEnforceSingleRowNode(), new TransformUncorrelatedSubqueryToJoin())), new CheckSubqueryNodesAreRewritten(), + simplifyOptimizer, new PushPredicateIntoTableScan(), // redo columnPrune and inlineProjections after pushPredicateIntoTableScan columnPruningOptimizer, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java index 2d7d7a923496..d92764c6a3d7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java @@ -30,9 +30,13 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; import com.google.common.collect.Range; +import java.util.List; +import java.util.stream.Collectors; + import static com.google.common.collect.Iterables.getOnlyElement; import static java.lang.Math.max; import static java.lang.Math.min; @@ -184,11 +188,18 @@ public Range visitLimit(LimitNode node, Void context) { return applyLimit(node.getChild(), node.getCount()); } - // @Override - // public Range visitTopN(TopNNode node, Void context) - // { - // return applyLimit(node.getSource(), node.getCount()); - // } + @Override + public Range visitTopK(TopKNode node, Void context) { + long limit = node.getCount(); + List> rangeList = + node.getChildren().stream() + .map(child -> applyLimit(child, limit)) + .collect(Collectors.toList()); + // merge rangeList + long lower = rangeList.stream().mapToLong(Range::lowerEndpoint).sum(); + long upper = rangeList.stream().mapToLong(Range::upperEndpoint).sum(); + return Range.closed(Math.min(lower, limit), Math.min(upper, limit)); + } // @Override // public Range visitWindow(WindowNode node, Void context) From 629737419a1e580900c537f8b79f3a82d90aa6b9 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Fri, 29 Nov 2024 17:32:08 +0800 Subject: [PATCH 09/12] add boolean joinKeyComparator --- .../AscBooleanTypeJoinKeyComparator.java | 75 +++++++++++++++++++ .../DescBooleanTypeJoinKeyComparator.java | 74 ++++++++++++++++++ .../comparator/JoinKeyComparatorFactory.java | 4 + .../TableFullOuterJoinOperator.java | 11 +++ 4 files changed, 164 insertions(+) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBooleanTypeJoinKeyComparator.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBooleanTypeJoinKeyComparator.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBooleanTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBooleanTypeJoinKeyComparator.java new file mode 100644 index 000000000000..29edcca65d6f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBooleanTypeJoinKeyComparator.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +// Not sure about whether we need this type of Comparator +public class AscBooleanTypeJoinKeyComparator implements JoinKeyComparator { + + private static final AscBooleanTypeJoinKeyComparator INSTANCE = + new AscBooleanTypeJoinKeyComparator(); + + private AscBooleanTypeJoinKeyComparator() {} + + public static AscBooleanTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return transformBooleanToInt(left.getColumn(leftColumnIndex).getBoolean(leftRowIndex)) + < transformBooleanToInt(right.getColumn(rightColumnIndex).getBoolean(rightRowIndex)); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getBoolean(leftRowIndex) + == right.getColumn(rightColumnIndex).getBoolean(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return transformBooleanToInt(left.getColumn(leftColumnIndex).getBoolean(leftRowIndex)) + <= transformBooleanToInt(right.getColumn(rightColumnIndex).getBoolean(rightRowIndex)); + } + + private int transformBooleanToInt(boolean value) { + return value ? 1 : 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBooleanTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBooleanTypeJoinKeyComparator.java new file mode 100644 index 000000000000..aeb047177fdf --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBooleanTypeJoinKeyComparator.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class DescBooleanTypeJoinKeyComparator implements JoinKeyComparator { + + private static final DescBooleanTypeJoinKeyComparator INSTANCE = + new DescBooleanTypeJoinKeyComparator(); + + private DescBooleanTypeJoinKeyComparator() {} + + public static DescBooleanTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return transformBooleanToInt(left.getColumn(leftColumnIndex).getBoolean(leftRowIndex)) + > transformBooleanToInt(right.getColumn(rightColumnIndex).getBoolean(rightRowIndex)); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getBoolean(leftRowIndex) + == right.getColumn(rightColumnIndex).getBoolean(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return transformBooleanToInt(left.getColumn(leftColumnIndex).getBoolean(leftRowIndex)) + >= transformBooleanToInt(right.getColumn(rightColumnIndex).getBoolean(rightRowIndex)); + } + + private int transformBooleanToInt(boolean value) { + return value ? 1 : 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java index a1d4b1bff02f..995d568802ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java @@ -43,6 +43,10 @@ public static JoinKeyComparator getComparator(Type type, boolean isAscending) { return isAscending ? AscDoubleTypeJoinKeyComparator.getInstance() : DescDoubleTypeJoinKeyComparator.getInstance(); + case BOOLEAN: + return isAscending + ? AscBooleanTypeJoinKeyComparator.getInstance() + : DescBooleanTypeJoinKeyComparator.getInstance(); case STRING: case BLOB: case TEXT: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java index 05a76c3c02e0..76b1a6c46797 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java @@ -28,6 +28,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.FloatColumn; import org.apache.tsfile.read.common.block.column.IntColumn; @@ -399,6 +400,16 @@ private void initLastMatchedRightBlock(TsBlock block, int columnIndex, int rowIn Optional.empty(), new double[] {block.getColumn(columnIndex).getDouble(rowIndex)})); break; + case BOOLEAN: + lastMatchedRightBlock = + new TsBlock( + 1, + TIME_COLUMN_TEMPLATE, + new BooleanColumn( + 1, + Optional.empty(), + new boolean[] {block.getColumn(columnIndex).getBoolean(rowIndex)})); + break; case STRING: case TEXT: case BLOB: From 951b50dc3cdd8bbced3014ab797a7bb6243e5fd1 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Fri, 29 Nov 2024 19:17:34 +0800 Subject: [PATCH 10/12] fix typo --- .../source/relational/TableFullOuterJoinOperator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java index 76b1a6c46797..2b1e170bda84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java @@ -222,7 +222,7 @@ protected void appendResult() { // notice: must examine `comparator.lessThan(getCurrentRightTime(), leftTime)` then examine // `comparator.lessThan(leftTime, getCurrentRightTime())` if (lastMatchedRightBlockIsNull - || comparator.lessThanOrEqual( + || comparator.lessThan( lastMatchedRightBlock, 0, 0, @@ -296,7 +296,7 @@ private void appendRightWithEmptyLeft() { while (rightBlockListIdx < rightBlockList.size()) { if (lastMatchedRightBlockIsNull - || comparator.lessThanOrEqual( + || comparator.lessThan( lastMatchedRightBlock, 0, 0, From ab0b645b41c36c2d4123b648c31652046c973cd1 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Tue, 3 Dec 2024 10:31:32 +0800 Subject: [PATCH 11/12] remove comments in ExpressionExtractor --- .../relational/planner/ExpressionExtractor.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java index 59d77280a3c6..371016ad0809 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java @@ -17,6 +17,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.SimplePlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; @@ -80,14 +81,13 @@ public Void visitGroupReference(GroupReference node, Void context) { return lookup.resolve(node).accept(this, context); } - /*@Override - public Void visitAggregation(AggregationNode node, Void context) - { - for (Aggregation aggregation : node.getAggregations().values()) { - aggregation.getArguments().forEach(consumer); - } - return super.visitAggregation(node, context); - }*/ + @Override + public Void visitAggregation(AggregationNode node, Void context) { + for (AggregationNode.Aggregation aggregation : node.getAggregations().values()) { + aggregation.getArguments().forEach(consumer); + } + return super.visitAggregation(node, context); + } @Override public Void visitFilter(FilterNode node, Void context) { From 04c138c0dd9c231c1a3b1b89a9a3d082280ae673 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Tue, 3 Dec 2024 13:54:16 +0800 Subject: [PATCH 12/12] fix some of the comments --- .../SimpleNestedLoopCrossJoinOperator.java | 43 +++++++++++++------ .../AscBinaryTypeJoinKeyComparator.java | 5 +-- .../DescBinaryTypeJoinKeyComparator.java | 5 +-- .../merge/comparator/JoinKeyComparator.java | 12 ++++++ .../relational/TableInnerJoinOperator.java | 9 +--- .../relational/planner/RelationPlanner.java | 10 ++--- 6 files changed, 51 insertions(+), 33 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java index 24f497a444c8..d5f37d1037fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java @@ -30,13 +30,14 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.RamUsageEstimator; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; -import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableInnerJoinOperator.buildResultTsBlock; +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator.TIME_COLUMN_TEMPLATE; /** * This Operator is used to implement the simple nested loop join algorithm for Cartesian product. @@ -102,18 +103,19 @@ public TsBlock next() throws Exception { long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS); long start = System.nanoTime(); if (!buildFinished) { - TsBlock block = buildSource.next(); - if (block != null && !block.isEmpty()) { - buildBlocks.add(block); - memoryReservationManager.reserveMemoryCumulatively(block.getRetainedSizeInBytes()); - } - if (!buildSource.hasNext()) { + if (!buildSource.hasNextWithTimer()) { buildFinished = true; + } else { + TsBlock block = buildSource.nextWithTimer(); + if (block != null && !block.isEmpty()) { + buildBlocks.add(block); + memoryReservationManager.reserveMemoryCumulatively(block.getRetainedSizeInBytes()); + } } // probeSource could still be blocked by now, so we need to check it again return null; } - cachedProbeBlock = cachedProbeBlock == null ? probeSource.next() : cachedProbeBlock; + cachedProbeBlock = cachedProbeBlock == null ? probeSource.nextWithTimer() : cachedProbeBlock; if (cachedProbeBlock == null || cachedProbeBlock.isEmpty()) { // TsBlock returned by probeSource is null or empty, we need to wait for another round cachedProbeBlock = null; @@ -134,7 +136,9 @@ public TsBlock next() throws Exception { return null; } - resultTsBlock = buildResultTsBlock(resultBuilder); + resultTsBlock = + resultBuilder.build( + new RunLengthEncodedColumn(TIME_COLUMN_TEMPLATE, resultBuilder.getPositionCount())); return checkTsBlockSizeAndGetResult(); } @@ -168,9 +172,11 @@ public boolean hasNext() throws Exception { return true; } if (!buildFinished) { - return buildSource.hasNext(); + return true; } - return probeSource.hasNext(); + return !buildBlocks.isEmpty() + && ((cachedProbeBlock != null && !cachedProbeBlock.isEmpty()) + || probeSource.hasNextWithTimer()); } @Override @@ -204,9 +210,18 @@ public boolean isFinished() throws Exception { return false; } - return (cachedProbeBlock == null || cachedProbeBlock.isEmpty()) - && probeSource.isFinished() - && buildFinished; + if (buildFinished) { // build side is finished + // no remaining rows in probe side is finished + if (buildBlocks.isEmpty()) { + // no rows in build side + return true; + } else { + return (cachedProbeBlock == null || cachedProbeBlock.isEmpty()) && probeSource.isFinished(); + } + } else { + // build side is not finished + return false; + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBinaryTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBinaryTypeJoinKeyComparator.java index 7980bf8ffada..9507bec86695 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBinaryTypeJoinKeyComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBinaryTypeJoinKeyComparator.java @@ -57,9 +57,8 @@ public boolean equalsTo( int rightColumnIndex, int rightRowIndex) { return left.getColumn(leftColumnIndex) - .getBinary(leftRowIndex) - .compareTo(right.getColumn(rightColumnIndex).getBinary(rightRowIndex)) - == 0; + .getBinary(leftRowIndex) + .equals(right.getColumn(rightColumnIndex).getBinary(rightRowIndex)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBinaryTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBinaryTypeJoinKeyComparator.java index 14cc5010c2fe..effc243137a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBinaryTypeJoinKeyComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBinaryTypeJoinKeyComparator.java @@ -57,9 +57,8 @@ public boolean equalsTo( int rightColumnIndex, int rightRowIndex) { return left.getColumn(leftColumnIndex) - .getBinary(leftRowIndex) - .compareTo(right.getColumn(rightColumnIndex).getBinary(rightRowIndex)) - == 0; + .getBinary(leftRowIndex) + .equals(right.getColumn(rightColumnIndex).getBinary(rightRowIndex)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparator.java index 0cae74b597a3..cc47419376a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparator.java @@ -23,6 +23,10 @@ public interface JoinKeyComparator { + /** + * Get values at the given position from the TsBlocks and then compare these two values. Return + * true if the left value is less than the right value. + */ boolean lessThan( TsBlock left, int leftColumnIndex, @@ -31,6 +35,10 @@ boolean lessThan( int rightColumnIndex, int rightRowIndex); + /** + * Get values at the given position from the TsBlocks and then compare these two values. Return + * true if the left value equals to the right value. + */ boolean equalsTo( TsBlock left, int leftColumnIndex, @@ -39,6 +47,10 @@ boolean equalsTo( int rightColumnIndex, int rightRowIndex); + /** + * Get values at the given position from the TsBlocks and then compare these two values. Return + * true if the left value is less than or equals to the right value. + */ boolean lessThanOrEqual( TsBlock left, int leftColumnIndex, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableInnerJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableInnerJoinOperator.java index 067c0db43de1..393b50c0eeae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableInnerJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableInnerJoinOperator.java @@ -167,15 +167,8 @@ else if (allLeftLessThanRight()) { } while (!resultBuilder.isFull()) { - TsBlock lastRightTsBlock = rightBlockList.get(rightBlockList.size() - 1); // all right block value is not matched - if (!comparator.lessThanOrEqual( - leftBlock, - leftJoinKeyPosition, - leftIndex, - lastRightTsBlock, - rightJoinKeyPosition, - lastRightTsBlock.getPositionCount() - 1)) { + if (allRightLessThanLeft()) { resetRightBlockList(); break; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index fe8523ad9ac9..914a47fe0cef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -610,6 +610,11 @@ protected RelationPlan visitAliasedRelation(AliasedRelation node, Void context) return new RelationPlan(root, analysis.getScope(node), mappings, outerContext); } + @Override + protected RelationPlan visitSubqueryExpression(SubqueryExpression node, Void context) { + return process(node.getQuery(), context); + } + // ================================ Implemented later ===================================== @Override @@ -617,11 +622,6 @@ protected RelationPlan visitValues(Values node, Void context) { throw new IllegalStateException("Values is not supported in current version."); } - @Override - protected RelationPlan visitSubqueryExpression(SubqueryExpression node, Void context) { - return process(node.getQuery(), context); - } - @Override protected RelationPlan visitIntersect(Intersect node, Void context) { throw new IllegalStateException("Intersect is not supported in current version.");