diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java index f3bc0d295771..09f64dc50a11 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java @@ -34,7 +34,6 @@ import java.sql.Statement; import java.util.Arrays; -import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.fail; @@ -157,14 +156,6 @@ public class IoTDBMultiIDsWithAttributesTableIT { String[] retArray; static String sql; - // public static void main(String[] args) { - // for (String[] sqlList : Arrays.asList(sql1, sql2)) { - // for (String sql : sqlList) { - // System.out.println(sql+";"); - // } - // } - // } - @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(1024 * 1024L); @@ -1775,17 +1766,21 @@ public void fourTableJoinTest() { + "order by s.student_id, t.teacher_id, c.course_id,g.grade_id"; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = new String[] {"region", "name", "teacher_id", "course_name", "score"}; + + retArray = + new String[] { + "haidian,Lucy,1005,数学,99,", + }; sql = "select s.region, s.name," + " t.teacher_id," + " c.course_name," + " g.score " + "from students s, teachers t, courses c, grades g " - + "where s.time=c.time and c.time=g.time"; - tableAssertTestFail( - sql, - "701: Cross join is not supported in current version, each table must have at least one equiJoinClause", - DATABASE_NAME); + + "where s.time=c.time and c.time=g.time and t.teacher_id = 1005 limit 1"; + + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index 5ae3ce3862c6..ae32bb241f64 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -3713,7 +3713,7 @@ public void modeTest() { public void exceptionTest() { tableAssertTestFail( "select s1 from table1 where s2 in (select s2 from table1)", - "701: Only TableSubquery is supported now", + "Not a valid IR expression", DATABASE_NAME); tableAssertTestFail( diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedSubqueryInWhereClauseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedSubqueryInWhereClauseIT.java new file mode 100644 index 000000000000..b8d8371bf8f7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedSubqueryInWhereClauseIT.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUncorrelatedSubqueryInWhereClauseIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testScalarSubqueryAfterComparisonInOneTable() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: s equals to the maximum value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 WHERE device_id = 'd01')"; + retArray = new String[] {"70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s not equals to the maximum value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != ((SELECT max(%s) FROM table1 WHERE device_id = 'd01'))"; + retArray = new String[] {"30,", "40,", "50,", "60,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the average value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= ((SELECT AVG(%s) FROM table1 WHERE device_id = 'd01'))"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the max value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > ((SELECT max(%s) FROM table1 WHERE device_id = 'd01'))"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s is less than the maximum value of s in table1 and greater than the minimum value + // of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < (SELECT max(%s) from table1 WHERE device_id = 'd01') and %s > (SELECT min(%s) from table1 WHERE device_id = 'd01') "; + retArray = new String[] {"40,", "50,", "60,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the avg value of s in table1 and s5 = true + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > ((SELECT avg(%s) FROM table1 WHERE device_id = 'd01' and s5 = true))"; + retArray = new String[] {"60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s greater than the count value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > (SELECT count(%s) FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s less than the sum value of s in table1 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < (SELECT sum(%s) FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: subquery is not aggregation but returns exactly one row + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT %s FROM table1 WHERE device_id = 'd01' and %s = 30)"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testScalarSubqueryAfterComparisonInDifferentTables() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: s greater than the count value of s in table2 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > (SELECT count(%s) from table2)"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: s less than the max value of s in table2 * the count value of s in table2 * 10 + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < ((SELECT max(%s) from table2) * (SELECT count(%s) from table2)) * 10"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testNestedScalarSubqueryAfterComparison() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: nested scalar subquery in where clause + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 where %s = (SELECT max(%s) from table1 WHERE device_id = 'd01'))"; + retArray = new String[] {"70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: nested scalar subquery with table subquery + sql = + "SELECT %s from (SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 where %s = (SELECT max(%s) from table1 WHERE device_id = 'd01')))"; + retArray = new String[] {"70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testScalarSubqueryAfterComparisonLegalityCheck() { + // Legality check: subquery returns multiple rows (should fail) + tableAssertTestFail( + "select s1 from table1 where s1 = (select s1 from table1)", + "301: Scalar sub-query has returned multiple rows.", + DATABASE_NAME); + + // Legality check: subquery can not be parsed + tableAssertTestFail( + "select s1 from table1 where s1 = (select s1 from)", "mismatched input", DATABASE_NAME); + + // Legality check: subquery can not be parsed(without parentheses) + tableAssertTestFail( + "select s1 from table1 where s1 = select s1 from table1", + "mismatched input", + DATABASE_NAME); + + // Legality check: Main query can not be parsed + tableAssertTestFail( + "select s1 from table1 where s1 = (select max(s1) from table1) and", + "mismatched input", + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java new file mode 100644 index 000000000000..1bd779aac65d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery; + +public class SubqueryDataSetUtils { + protected static final String DATABASE_NAME = "subqueryTest"; + protected static final String[] NUMERIC_MEASUREMENTS = new String[] {"s1", "s2", "s3", "s4"}; + protected static final String[] CREATE_SQLS = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + // table1 + "CREATE TABLE table1(province STRING ID, city STRING ID, region STRING ID, device_id STRING ID, color STRING ATTRIBUTE, type STRING ATTRIBUTE, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT, s5 BOOLEAN MEASUREMENT, s6 TEXT MEASUREMENT, s7 STRING MEASUREMENT, s8 BLOB MEASUREMENT, s9 TIMESTAMP MEASUREMENT, s10 DATE MEASUREMENT)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',40,40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',50,50,50.0,50.0,true,'shanghai_huangpu_red_A_d01_50','shanghai_huangpu_red_A_d01_50',X'cafebabe50',2024-09-24T06:15:00.000+00:00,'2024-09-25')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:16:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',60,60,60.0,60.0,false,'shanghai_huangpu_red_A_d01_60','shanghai_huangpu_red_A_d01_60',X'cafebabe60',2024-09-24T06:16:00.000+00:00,'2024-09-26')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:17:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',70,70,70.0,70.0,true,'shanghai_huangpu_red_A_d01_70','shanghai_huangpu_red_A_d01_70',X'cafebabe70',2024-09-24T06:17:00.000+00:00,'2024-09-27')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',36,true,'shanghai_huangpu_red_B_d02_36','shanghai_huangpu_red_B_d02_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',40,40.0,'shanghai_huangpu_red_B_d02_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','huangpu','d02','red','BBBBBBBBBBBBBBBB',50000,'shanghai_huangpu_red_B_d02_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',36,36.0,'shanghai_huangpu_yellow_A_d03_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',41,41.0,false,'shanghai_huangpu_yellow_A_d03_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',46000,46.0,'shanghai_huangpu_yellow_A_d03_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','huangpu','d03','yellow','A',51.0,'shanghai_huangpu_yellow_A_d03_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',30.0,true,'shanghai_huangpu_yellow_B_d04_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','huangpu','d04','yellow','BBBBBBBBBBBBBBBB',55,55.0,'shanghai_huangpu_yellow_B_d04_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong','d05','red','A',30,30.0,'shanghai_pudong_red_A_d05_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'shanghai','shanghai','pudong','d05','red','A',35000,35.0,35.0,'shanghai_pudong_red_A_d05_35','shanghai_pudong_red_A_d05_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d05','red','A',40,40.0,true,'shanghai_pudong_red_A_d05_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong','d05','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong','d05','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',36,true,'shanghai_pudong_red_B_d06_36','shanghai_pudong_red_B_d06_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',40,40.0,'shanghai_pudong_red_B_d06_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'shanghai','shanghai','pudong','d06','red','BBBBBBBBBBBBBBBB',50000,'shanghai_pudong_red_B_d06_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',36,36.0,'shanghai_pudong_yellow_A_d07_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',41,41.0,false,'shanghai_pudong_yellow_A_d07_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',46000,46.0,'shanghai_pudong_yellow_A_d07_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'shanghai','shanghai','pudong','d07','yellow','A',51.0,'shanghai_pudong_yellow_A_d07_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',30.0,true,'shanghai_pudong_yellow_B_d08_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'shanghai','shanghai','pudong','d08','yellow','BBBBBBBBBBBBBBBB',55,55.0,'shanghai_pudong_yellow_B_d08_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang','d09','red','A',30,30.0,'beijing_chaoyang_red_A_d09_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','chaoyang','d09','red','A',35000,35.0,35.0,'beijing_chaoyang_red_A_d09_35','beijing_chaoyang_red_A_d09_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d09','red','A',40,40.0,true,'beijing_chaoyang_red_A_d09_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang','d09','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang','d09','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',36,true,'beijing_chaoyang_red_B_d10_36','beijing_chaoyang_red_B_d10_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',40,40.0,'beijing_chaoyang_red_B_d10_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','chaoyang','d10','red','BBBBBBBBBBBBBBBB',50000,'beijing_chaoyang_red_B_d10_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',36,36.0,'beijing_chaoyang_yellow_A_d11_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',41,41.0,false,'beijing_chaoyang_yellow_A_d11_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',46000,46.0,'beijing_chaoyang_yellow_A_d11_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','chaoyang','d11','yellow','A',51.0,'beijing_chaoyang_yellow_A_d11_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',30.0,true,'beijing_chaoyang_yellow_B_d12_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','chaoyang','d12','yellow','BBBBBBBBBBBBBBBB',55,55.0,'beijing_chaoyang_yellow_B_d12_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian','d13','red','A',30,30.0,'beijing_haidian_red_A_d13_30', X'cafebabe30',2024-09-24T06:15:30.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10) values (2024-09-24T06:15:35.000+00:00,'beijing','beijing','haidian','d13','red','A',35000,35.0,35.0,'beijing_haidian_red_A_d13_35','beijing_haidian_red_A_d13_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d13','red','A',40,40.0,true,'beijing_haidian_red_A_d13_40',2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s5,s9,s10) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian','d13','red','A',50000,false,2024-09-24T06:15:50.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian','d13','red','A',55,55.0,X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s5,s6,s7,s9) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',36,true,'beijing_haidian_red_B_d14_36','beijing_haidian_red_B_d14_36',2024-09-24T06:15:36.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',40,40.0,'beijing_haidian_red_B_d14_40',2024-09-24T06:15:40.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s7,s8,s9) values (2024-09-24T06:15:50.000+00:00,'beijing','beijing','haidian','d14','red','BBBBBBBBBBBBBBBB',50000,'beijing_haidian_red_B_d14_50',X'cafebabe50',2024-09-24T06:15:50.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s8,s9) values (2024-09-24T06:15:31.000+00:00,'beijing','beijing','haidian','d15','yellow','A',31000,X'cafebabe31',2024-09-24T06:15:31.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s7,s9,s10) values (2024-09-24T06:15:36.000+00:00,'beijing','beijing','haidian','d15','yellow','A',36,36.0,'beijing_haidian_yellow_A_d15_36',2024-09-24T06:15:36.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s3,s5,s6,s8,s9) values (2024-09-24T06:15:41.000+00:00,'beijing','beijing','haidian','d15','yellow','A',41,41.0,false,'beijing_haidian_yellow_A_d15_41',X'cafebabe41',2024-09-24T06:15:41.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s4,s7,s9) values (2024-09-24T06:15:46.000+00:00,'beijing','beijing','haidian','d15','yellow','A',46000,46.0,'beijing_haidian_yellow_A_d15_46',2024-09-24T06:15:46.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s6,s9) values (2024-09-24T06:15:51.000+00:00,'beijing','beijing','haidian','d15','yellow','A',51.0,'beijing_haidian_yellow_A_d15_51',2024-09-24T06:15:51.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s3,s5,s7,s9,s10) values (2024-09-24T06:15:30.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',30.0,true,'beijing_haidian_yellow_B_d16_30',2024-09-24T06:15:30.000+00:00,'2024-09-24')", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s2,s9) values (2024-09-24T06:15:40.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',40000,2024-09-24T06:15:40.000+00:00)", + "INSERT INTO table1(time,province,city,region,device_id,color,type,s1,s4,s6,s8,s9) values (2024-09-24T06:15:55.000+00:00,'beijing','beijing','haidian','d16','yellow','BBBBBBBBBBBBBBBB',55,55.0,'beijing_haidian_yellow_B_d16_55',X'cafebabe55',2024-09-24T06:15:55.000+00:00)", + // table2 + "CREATE TABLE table2(device_id STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT, s5 BOOLEAN MEASUREMENT, s6 TEXT MEASUREMENT, s7 STRING MEASUREMENT, s8 BLOB MEASUREMENT, s9 TIMESTAMP MEASUREMENT, s10 DATE MEASUREMENT)", + "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) " + + " values(1, 'd1', 1, 11, 1.1, 11.1, true, 'text1', 'string1', X'cafebabe01', 1, '2024-10-01')", + "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5) " + + " values(2, 'd1', 2, 22, 2.2, 22.2, false)", + "INSERT INTO table2(time,device_id,s6,s7,s8,s9,s10) " + + " values(3, 'd1', 'text3', 'string3', X'cafebabe03', 3, '2024-10-03')", + "INSERT INTO table2(time,device_id,s6,s7,s8,s9,s10) " + + " values(4, 'd1', 'text4', 'string4', X'cafebabe04', 4, '2024-10-04')", + "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5) " + + " values(5, 'd1', 5, 55, 5.5, 55.5, false)", + "FLUSH", + "CLEAR ATTRIBUTE CACHE", + }; +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java new file mode 100644 index 000000000000..e9f6fba26f68 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process; + +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.queryengine.execution.operator.Operator; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.utils.RamUsageEstimator; + +public class EnforceSingleRowOperator implements ProcessOperator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(EnforceSingleRowOperator.class); + + private static final String MULTIPLE_ROWS_ERROR_MESSAGE = + "Scalar sub-query has returned multiple rows."; + + private static final String NO_RESULT_ERROR_MESSAGE = "Scalar sub-query does not have output."; + + private final OperatorContext operatorContext; + private final Operator child; + + private boolean finished = false; + + public EnforceSingleRowOperator(OperatorContext operatorContext, Operator child) { + this.operatorContext = operatorContext; + this.child = child; + } + + @Override + public ListenableFuture isBlocked() { + return child.isBlocked(); + } + + @Override + public TsBlock next() throws Exception { + TsBlock tsBlock = child.next(); + if (tsBlock == null || tsBlock.isEmpty()) { + return tsBlock; + } + if (tsBlock.getPositionCount() > 1 || finished) { + throw new IllegalStateException(MULTIPLE_ROWS_ERROR_MESSAGE); + } + finished = true; + return tsBlock; + } + + @Override + public boolean hasNext() throws Exception { + return !isFinished(); + } + + @Override + public void close() throws Exception { + if (child != null) { + child.close(); + } + } + + @Override + public boolean isFinished() throws Exception { + boolean childFinished = child.isFinished(); + if (childFinished && !finished) { + // finished == false means the child has no result returned up to now, but we need at least + // one result. + throw new IllegalStateException(NO_RESULT_ERROR_MESSAGE); + } + // Even if finished == true, we can not return true here, we need to call child.next() to check + // if child has more data. + // For example, if the child is a TableScanOperator with a filter, childFinished = false does + // not mean that the child can produce more data.(see the isFinished() of TableScanOperator) + return childFinished; + } + + @Override + public long calculateMaxPeekMemory() { + return child.calculateMaxPeekMemory(); + } + + @Override + public long calculateMaxReturnSize() { + return child.calculateMaxReturnSize() + / TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber(); + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + return child.calculateRetainedSizeAfterCallingNext(); + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(child) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext); + } + + @Override + public OperatorContext getOperatorContext() { + return operatorContext; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java new file mode 100644 index 000000000000..d5f37d1037fd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/SimpleNestedLoopCrossJoinOperator.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join; + +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.queryengine.execution.operator.AbstractOperator; +import org.apache.iotdb.db.queryengine.execution.operator.Operator; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator.TIME_COLUMN_TEMPLATE; + +/** + * This Operator is used to implement the simple nested loop join algorithm for Cartesian product. + * It is used to join two tables, one is the probe table and the other is the build table. For now, + * the build table is assumed to be small enough to be cached in memory.(Produced by a scalar + * subquery.) Scalar subquery is always the right child of PlanNode, so we can use right child of + * JoinNode as the build table. + */ +public class SimpleNestedLoopCrossJoinOperator extends AbstractOperator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(SimpleNestedLoopCrossJoinOperator.class); + + private final Operator probeSource; + + private final Operator buildSource; + + // cache the result of buildSource, for now, we assume that the buildChild produces a small number + // of TsBlocks + private final List buildBlocks; + + private final TsBlockBuilder resultBuilder; + + private final MemoryReservationManager memoryReservationManager; + + private final int[] leftOutputSymbolIdx; + + private final int[] rightOutputSymbolIdx; + + private TsBlock cachedProbeBlock; + + private int probeIndex; + + private boolean buildFinished = false; + + public SimpleNestedLoopCrossJoinOperator( + OperatorContext operatorContext, + Operator probeSource, + Operator buildSource, + int[] leftOutputSymbolIdx, + int[] rightOutputSymbolIdx, + List dataTypes) { + this.operatorContext = operatorContext; + this.probeSource = probeSource; + this.buildSource = buildSource; + this.leftOutputSymbolIdx = leftOutputSymbolIdx; + this.rightOutputSymbolIdx = rightOutputSymbolIdx; + this.buildBlocks = new ArrayList<>(); + this.resultBuilder = new TsBlockBuilder(dataTypes); + this.memoryReservationManager = + operatorContext + .getDriverContext() + .getFragmentInstanceContext() + .getMemoryReservationContext(); + } + + @Override + public TsBlock next() throws Exception { + if (retainedTsBlock != null) { + getResultFromRetainedTsBlock(); + } + resultBuilder.reset(); + // start stopwatch + long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS); + long start = System.nanoTime(); + if (!buildFinished) { + if (!buildSource.hasNextWithTimer()) { + buildFinished = true; + } else { + TsBlock block = buildSource.nextWithTimer(); + if (block != null && !block.isEmpty()) { + buildBlocks.add(block); + memoryReservationManager.reserveMemoryCumulatively(block.getRetainedSizeInBytes()); + } + } + // probeSource could still be blocked by now, so we need to check it again + return null; + } + cachedProbeBlock = cachedProbeBlock == null ? probeSource.nextWithTimer() : cachedProbeBlock; + if (cachedProbeBlock == null || cachedProbeBlock.isEmpty()) { + // TsBlock returned by probeSource is null or empty, we need to wait for another round + cachedProbeBlock = null; + return null; + } + while (probeIndex < cachedProbeBlock.getPositionCount() + && System.nanoTime() - start < maxRuntime) { + for (TsBlock buildBlock : buildBlocks) { + appendValueToResult(probeIndex, buildBlock); + } + probeIndex++; + } + if (probeIndex == cachedProbeBlock.getPositionCount()) { + probeIndex = 0; + cachedProbeBlock = null; + } + if (resultBuilder.isEmpty()) { + return null; + } + + resultTsBlock = + resultBuilder.build( + new RunLengthEncodedColumn(TIME_COLUMN_TEMPLATE, resultBuilder.getPositionCount())); + return checkTsBlockSizeAndGetResult(); + } + + private void appendValueToResult(int probeIndex, TsBlock buildBlock) { + for (int i = 0; i < leftOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(i); + for (int j = 0; j < buildBlock.getPositionCount(); j++) { + if (cachedProbeBlock.getColumn(leftOutputSymbolIdx[i]).isNull(probeIndex)) { + columnBuilder.appendNull(); + } else { + columnBuilder.write(cachedProbeBlock.getColumn(leftOutputSymbolIdx[i]), probeIndex); + } + } + } + for (int i = 0; i < rightOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(i + leftOutputSymbolIdx.length); + for (int j = 0; j < buildBlock.getPositionCount(); j++) { + if (buildBlock.getColumn(rightOutputSymbolIdx[i]).isNull(j)) { + columnBuilder.appendNull(); + } else { + columnBuilder.write(buildBlock.getColumn(rightOutputSymbolIdx[i]), j); + } + } + } + resultBuilder.declarePositions(buildBlock.getPositionCount()); + } + + @Override + public boolean hasNext() throws Exception { + if (retainedTsBlock != null) { + return true; + } + if (!buildFinished) { + return true; + } + return !buildBlocks.isEmpty() + && ((cachedProbeBlock != null && !cachedProbeBlock.isEmpty()) + || probeSource.hasNextWithTimer()); + } + + @Override + public ListenableFuture isBlocked() { + if (buildFinished) { + return probeSource.isBlocked(); + } + return buildSource.isBlocked(); + } + + @Override + public void close() throws Exception { + if (probeSource != null) { + probeSource.close(); + } + if (buildSource != null) { + buildSource.close(); + } + for (TsBlock block : buildBlocks) { + memoryReservationManager.releaseMemoryCumulatively(block.getRetainedSizeInBytes()); + } + buildBlocks.clear(); + cachedProbeBlock = null; + resultTsBlock = null; + retainedTsBlock = null; + } + + @Override + public boolean isFinished() throws Exception { + if (retainedTsBlock != null) { + return false; + } + + if (buildFinished) { // build side is finished + // no remaining rows in probe side is finished + if (buildBlocks.isEmpty()) { + // no rows in build side + return true; + } else { + return (cachedProbeBlock == null || cachedProbeBlock.isEmpty()) && probeSource.isFinished(); + } + } else { + // build side is not finished + return false; + } + } + + @Override + public long calculateMaxPeekMemory() { + return Math.max( + Math.max( + probeSource.calculateMaxPeekMemoryWithCounter(), + buildSource.calculateMaxPeekMemoryWithCounter()), + calculateRetainedSizeAfterCallingNext() + calculateMaxReturnSize()); + } + + @Override + public long calculateMaxReturnSize() { + return maxReturnSize; + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + return probeSource.calculateRetainedSizeAfterCallingNext() + + buildSource.calculateRetainedSizeAfterCallingNext() + // cachedProbeBlock + one build block assumed + + maxReturnSize * 2; + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(probeSource) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(buildSource) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) + + resultBuilder.getRetainedSizeInBytes(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBinaryTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBinaryTypeJoinKeyComparator.java new file mode 100644 index 000000000000..9507bec86695 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBinaryTypeJoinKeyComparator.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class AscBinaryTypeJoinKeyComparator implements JoinKeyComparator { + + private static final AscBinaryTypeJoinKeyComparator INSTANCE = + new AscBinaryTypeJoinKeyComparator(); + + private AscBinaryTypeJoinKeyComparator() { + // hide constructor + } + + public static AscBinaryTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex) + .getBinary(leftRowIndex) + .compareTo(right.getColumn(rightColumnIndex).getBinary(rightRowIndex)) + < 0; + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex) + .getBinary(leftRowIndex) + .equals(right.getColumn(rightColumnIndex).getBinary(rightRowIndex)); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex) + .getBinary(leftRowIndex) + .compareTo(right.getColumn(rightColumnIndex).getBinary(rightRowIndex)) + <= 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBooleanTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBooleanTypeJoinKeyComparator.java new file mode 100644 index 000000000000..29edcca65d6f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBooleanTypeJoinKeyComparator.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +// Not sure about whether we need this type of Comparator +public class AscBooleanTypeJoinKeyComparator implements JoinKeyComparator { + + private static final AscBooleanTypeJoinKeyComparator INSTANCE = + new AscBooleanTypeJoinKeyComparator(); + + private AscBooleanTypeJoinKeyComparator() {} + + public static AscBooleanTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return transformBooleanToInt(left.getColumn(leftColumnIndex).getBoolean(leftRowIndex)) + < transformBooleanToInt(right.getColumn(rightColumnIndex).getBoolean(rightRowIndex)); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getBoolean(leftRowIndex) + == right.getColumn(rightColumnIndex).getBoolean(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return transformBooleanToInt(left.getColumn(leftColumnIndex).getBoolean(leftRowIndex)) + <= transformBooleanToInt(right.getColumn(rightColumnIndex).getBoolean(rightRowIndex)); + } + + private int transformBooleanToInt(boolean value) { + return value ? 1 : 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscDoubleTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscDoubleTypeJoinKeyComparator.java new file mode 100644 index 000000000000..e464a14580ec --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscDoubleTypeJoinKeyComparator.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class AscDoubleTypeJoinKeyComparator implements JoinKeyComparator { + + private static final AscDoubleTypeJoinKeyComparator INSTANCE = + new AscDoubleTypeJoinKeyComparator(); + + private AscDoubleTypeJoinKeyComparator() { + // hide constructor + } + + public static AscDoubleTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getDouble(leftRowIndex) + < right.getColumn(rightColumnIndex).getDouble(rightRowIndex); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getDouble(leftRowIndex) + == right.getColumn(rightColumnIndex).getDouble(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getDouble(leftRowIndex) + <= right.getColumn(rightColumnIndex).getDouble(rightRowIndex); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscFloatTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscFloatTypeJoinKeyComparator.java new file mode 100644 index 000000000000..3d39d0cff16b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscFloatTypeJoinKeyComparator.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class AscFloatTypeJoinKeyComparator implements JoinKeyComparator { + + private static final AscFloatTypeJoinKeyComparator INSTANCE = new AscFloatTypeJoinKeyComparator(); + + private AscFloatTypeJoinKeyComparator() { + // hide constructor + } + + public static AscFloatTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getFloat(leftRowIndex) + < right.getColumn(rightColumnIndex).getFloat(rightRowIndex); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getFloat(leftRowIndex) + == right.getColumn(rightColumnIndex).getFloat(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getFloat(leftRowIndex) + <= right.getColumn(rightColumnIndex).getFloat(rightRowIndex); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscIntTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscIntTypeJoinKeyComparator.java new file mode 100644 index 000000000000..50f87d7cffd5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscIntTypeJoinKeyComparator.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class AscIntTypeJoinKeyComparator implements JoinKeyComparator { + + private static final AscIntTypeJoinKeyComparator INSTANCE = new AscIntTypeJoinKeyComparator(); + + private AscIntTypeJoinKeyComparator() { + // hide constructor + } + + public static AscIntTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getInt(leftRowIndex) + < right.getColumn(rightColumnIndex).getInt(rightRowIndex); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getInt(leftRowIndex) + == right.getColumn(rightColumnIndex).getInt(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getInt(leftRowIndex) + <= right.getColumn(rightColumnIndex).getInt(rightRowIndex); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscLongTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscLongTypeJoinKeyComparator.java new file mode 100644 index 000000000000..935d4fe61d18 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscLongTypeJoinKeyComparator.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class AscLongTypeJoinKeyComparator implements JoinKeyComparator { + + private static final AscLongTypeJoinKeyComparator INSTANCE = new AscLongTypeJoinKeyComparator(); + + private AscLongTypeJoinKeyComparator() { + // hide constructor + } + + public static AscLongTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getLong(leftRowIndex) + < right.getColumn(rightColumnIndex).getLong(rightRowIndex); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getLong(leftRowIndex) + == right.getColumn(rightColumnIndex).getLong(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getLong(leftRowIndex) + <= right.getColumn(rightColumnIndex).getLong(rightRowIndex); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBinaryTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBinaryTypeJoinKeyComparator.java new file mode 100644 index 000000000000..effc243137a2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBinaryTypeJoinKeyComparator.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class DescBinaryTypeJoinKeyComparator implements JoinKeyComparator { + + private static final DescBinaryTypeJoinKeyComparator INSTANCE = + new DescBinaryTypeJoinKeyComparator(); + + private DescBinaryTypeJoinKeyComparator() { + // hide constructor + } + + public static DescBinaryTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex) + .getBinary(leftRowIndex) + .compareTo(right.getColumn(rightColumnIndex).getBinary(rightRowIndex)) + > 0; + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex) + .getBinary(leftRowIndex) + .equals(right.getColumn(rightColumnIndex).getBinary(rightRowIndex)); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex) + .getBinary(leftRowIndex) + .compareTo(right.getColumn(rightColumnIndex).getBinary(rightRowIndex)) + >= 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBooleanTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBooleanTypeJoinKeyComparator.java new file mode 100644 index 000000000000..aeb047177fdf --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescBooleanTypeJoinKeyComparator.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class DescBooleanTypeJoinKeyComparator implements JoinKeyComparator { + + private static final DescBooleanTypeJoinKeyComparator INSTANCE = + new DescBooleanTypeJoinKeyComparator(); + + private DescBooleanTypeJoinKeyComparator() {} + + public static DescBooleanTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return transformBooleanToInt(left.getColumn(leftColumnIndex).getBoolean(leftRowIndex)) + > transformBooleanToInt(right.getColumn(rightColumnIndex).getBoolean(rightRowIndex)); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getBoolean(leftRowIndex) + == right.getColumn(rightColumnIndex).getBoolean(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return transformBooleanToInt(left.getColumn(leftColumnIndex).getBoolean(leftRowIndex)) + >= transformBooleanToInt(right.getColumn(rightColumnIndex).getBoolean(rightRowIndex)); + } + + private int transformBooleanToInt(boolean value) { + return value ? 1 : 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescDoubleTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescDoubleTypeJoinKeyComparator.java new file mode 100644 index 000000000000..b841810f7df1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescDoubleTypeJoinKeyComparator.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class DescDoubleTypeJoinKeyComparator implements JoinKeyComparator { + + private static final DescDoubleTypeJoinKeyComparator INSTANCE = + new DescDoubleTypeJoinKeyComparator(); + + private DescDoubleTypeJoinKeyComparator() { + // hide constructor + } + + public static DescDoubleTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getDouble(leftRowIndex) + > right.getColumn(rightColumnIndex).getDouble(rightRowIndex); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getDouble(leftRowIndex) + == right.getColumn(rightColumnIndex).getDouble(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getDouble(leftRowIndex) + >= right.getColumn(rightColumnIndex).getDouble(rightRowIndex); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescFloatTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescFloatTypeJoinKeyComparator.java new file mode 100644 index 000000000000..6f7d9be22ded --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescFloatTypeJoinKeyComparator.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class DescFloatTypeJoinKeyComparator implements JoinKeyComparator { + + private static final DescFloatTypeJoinKeyComparator INSTANCE = + new DescFloatTypeJoinKeyComparator(); + + private DescFloatTypeJoinKeyComparator() { + // hide constructor + } + + public static DescFloatTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getFloat(leftRowIndex) + > right.getColumn(rightColumnIndex).getFloat(rightRowIndex); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getFloat(leftRowIndex) + == right.getColumn(rightColumnIndex).getFloat(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getFloat(leftRowIndex) + >= right.getColumn(rightColumnIndex).getFloat(rightRowIndex); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescIntTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescIntTypeJoinKeyComparator.java new file mode 100644 index 000000000000..246af1e5ca61 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescIntTypeJoinKeyComparator.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class DescIntTypeJoinKeyComparator implements JoinKeyComparator { + + private static final DescIntTypeJoinKeyComparator INSTANCE = new DescIntTypeJoinKeyComparator(); + + private DescIntTypeJoinKeyComparator() { + // hide constructor + } + + public static DescIntTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getInt(leftRowIndex) + > right.getColumn(rightColumnIndex).getInt(rightRowIndex); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getInt(leftRowIndex) + == right.getColumn(rightColumnIndex).getInt(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getInt(leftRowIndex) + >= right.getColumn(rightColumnIndex).getInt(rightRowIndex); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescLongTypeJoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescLongTypeJoinKeyComparator.java new file mode 100644 index 000000000000..7ea67c147acd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/DescLongTypeJoinKeyComparator.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public class DescLongTypeJoinKeyComparator implements JoinKeyComparator { + + private static final DescLongTypeJoinKeyComparator INSTANCE = new DescLongTypeJoinKeyComparator(); + + private DescLongTypeJoinKeyComparator() { + // hide constructor + } + + public static DescLongTypeJoinKeyComparator getInstance() { + return INSTANCE; + } + + @Override + public boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getLong(leftRowIndex) + > right.getColumn(rightColumnIndex).getLong(rightRowIndex); + } + + @Override + public boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getLong(leftRowIndex) + == right.getColumn(rightColumnIndex).getLong(rightRowIndex); + } + + @Override + public boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex) { + return left.getColumn(leftColumnIndex).getLong(leftRowIndex) + >= right.getColumn(rightColumnIndex).getLong(rightRowIndex); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparator.java new file mode 100644 index 000000000000..cc47419376a0 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparator.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public interface JoinKeyComparator { + + /** + * Get values at the given position from the TsBlocks and then compare these two values. Return + * true if the left value is less than the right value. + */ + boolean lessThan( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex); + + /** + * Get values at the given position from the TsBlocks and then compare these two values. Return + * true if the left value equals to the right value. + */ + boolean equalsTo( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex); + + /** + * Get values at the given position from the TsBlocks and then compare these two values. Return + * true if the left value is less than or equals to the right value. + */ + boolean lessThanOrEqual( + TsBlock left, + int leftColumnIndex, + int leftRowIndex, + TsBlock right, + int rightColumnIndex, + int rightRowIndex); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java new file mode 100644 index 000000000000..995d568802ca --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator; + +import org.apache.tsfile.read.common.type.Type; + +public class JoinKeyComparatorFactory { + + public static JoinKeyComparator getComparator(Type type, boolean isAscending) { + switch (type.getTypeEnum()) { + case INT32: + case DATE: + return isAscending + ? AscIntTypeJoinKeyComparator.getInstance() + : DescIntTypeJoinKeyComparator.getInstance(); + case INT64: + case TIMESTAMP: + return isAscending + ? AscLongTypeJoinKeyComparator.getInstance() + : DescLongTypeJoinKeyComparator.getInstance(); + case FLOAT: + return isAscending + ? AscFloatTypeJoinKeyComparator.getInstance() + : DescFloatTypeJoinKeyComparator.getInstance(); + case DOUBLE: + return isAscending + ? AscDoubleTypeJoinKeyComparator.getInstance() + : DescDoubleTypeJoinKeyComparator.getInstance(); + case BOOLEAN: + return isAscending + ? AscBooleanTypeJoinKeyComparator.getInstance() + : DescBooleanTypeJoinKeyComparator.getInstance(); + case STRING: + case BLOB: + case TEXT: + return isAscending + ? AscBinaryTypeJoinKeyComparator.getInstance() + : DescBinaryTypeJoinKeyComparator.getInstance(); + default: + // other types are not supported. + throw new UnsupportedOperationException("Unsupported data type: " + type); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java index e4ac3e6058e7..2b1e170bda84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java @@ -22,44 +22,60 @@ import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; -import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.TimeComparator; +import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator.JoinKeyComparator; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.FloatColumn; +import org.apache.tsfile.read.common.block.column.IntColumn; +import org.apache.tsfile.read.common.block.column.LongColumn; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import java.util.List; +import java.util.Optional; import java.util.concurrent.TimeUnit; +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator.TIME_COLUMN_TEMPLATE; + public class TableFullOuterJoinOperator extends TableInnerJoinOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(TableFullOuterJoinOperator.class); private boolean leftFinished; private boolean rightFinished; - private long lastMatchedRightTime = Long.MIN_VALUE; + + private TsBlock lastMatchedRightBlock; + + private boolean lastMatchedRightBlockIsNull = true; public TableFullOuterJoinOperator( OperatorContext operatorContext, Operator leftChild, - int leftTimeColumnPosition, + int leftJoinKeyPosition, int[] leftOutputSymbolIdx, Operator rightChild, - int rightTimeColumnPosition, + int rightJoinKeyPosition, int[] rightOutputSymbolIdx, - TimeComparator timeComparator, - List dataTypes) { + JoinKeyComparator joinKeyComparator, + List dataTypes, + Type joinKeyType) { super( operatorContext, leftChild, - leftTimeColumnPosition, + leftJoinKeyPosition, leftOutputSymbolIdx, rightChild, - rightTimeColumnPosition, + rightJoinKeyPosition, rightOutputSymbolIdx, - timeComparator, - dataTypes); + joinKeyComparator, + dataTypes, + joinKeyType); } @Override @@ -101,7 +117,7 @@ public TsBlock next() throws Exception { } // all the rightTsBlock is less than leftTsBlock, append right with empty left - if (comparator.lessThan(getRightEndTime(), getCurrentLeftTime())) { + if (allRightLessThanLeft()) { appendRightWithEmptyLeft(); resetRightBlockList(); resultTsBlock = buildResultTsBlock(resultBuilder); @@ -109,7 +125,7 @@ public TsBlock next() throws Exception { } // all the leftTsBlock is less than rightTsBlock, append left with empty right - else if (comparator.lessThan(getLeftEndTime(), getCurrentRightTime())) { + else if (allLeftLessThanRight()) { appendLeftWithEmptyRight(); leftBlock = null; leftIndex = 0; @@ -117,25 +133,28 @@ else if (comparator.lessThan(getLeftEndTime(), getCurrentRightTime())) { return checkTsBlockSizeAndGetResult(); } - long leftProbeTime = getCurrentLeftTime(); while (!resultBuilder.isFull()) { - // all right block time is not matched - if (!comparator.canContinueInclusive(leftProbeTime, getRightEndTime())) { + TsBlock lastRightBlock = rightBlockList.get(rightBlockList.size() - 1); + if (!comparator.lessThanOrEqual( + leftBlock, + leftJoinKeyPosition, + leftIndex, + lastRightBlock, + rightJoinKeyPosition, + lastRightBlock.getPositionCount() - 1)) { appendRightWithEmptyLeft(); resetRightBlockList(); break; } - appendResult(leftProbeTime); + appendResult(); if (leftIndex >= leftBlock.getPositionCount()) { leftBlock = null; leftIndex = 0; break; } - - leftProbeTime = getCurrentLeftTime(); } if (resultBuilder.isEmpty()) { @@ -188,16 +207,28 @@ protected boolean prepareInput(long start, long maxRuntime) throws Exception { || (leftFinished && rightBlockNotEmpty() && hasCachedNextRightBlock); } - @Override - protected void appendResult(long leftTime) { + protected void appendResult() { - while (comparator.lessThan(getCurrentRightTime(), leftTime)) { + while (comparator.lessThan( + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPosition, + rightIndex, + leftBlock, + leftJoinKeyPosition, + leftIndex)) { // getCurrentRightTime() can only be greater than lastMatchedRightTime // if greater than, then put right // if equals, it has been put in last round // notice: must examine `comparator.lessThan(getCurrentRightTime(), leftTime)` then examine // `comparator.lessThan(leftTime, getCurrentRightTime())` - if (getCurrentRightTime() > lastMatchedRightTime) { + if (lastMatchedRightBlockIsNull + || comparator.lessThan( + lastMatchedRightBlock, + 0, + 0, + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPosition, + rightIndex)) { appendOneRightRowWithEmptyLeft(); } @@ -206,17 +237,29 @@ protected void appendResult(long leftTime) { } } - if (comparator.lessThan(leftTime, getCurrentRightTime())) { + if (comparator.lessThan( + leftBlock, + leftJoinKeyPosition, + leftIndex, + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPosition, + rightIndex)) { appendOneLeftRowWithEmptyRight(); leftIndex++; return; } int tmpBlockIdx = rightBlockListIdx, tmpIdx = rightIndex; - while (leftTime == getRightTime(tmpBlockIdx, tmpIdx)) { + while (comparator.equalsTo( + leftBlock, + leftJoinKeyPosition, + leftIndex, + rightBlockList.get(tmpBlockIdx), + rightJoinKeyPosition, + tmpIdx)) { // lastMatchedRightBlockListIdx = rightBlockListIdx; // lastMatchedRightIdx = rightIndex; - lastMatchedRightTime = leftTime; + initLastMatchedRightBlock(leftBlock, leftJoinKeyPosition, leftIndex); appendValueToResult(tmpBlockIdx, tmpIdx); resultBuilder.declarePosition(); @@ -252,7 +295,14 @@ private void appendLeftWithEmptyRight() { private void appendRightWithEmptyLeft() { while (rightBlockListIdx < rightBlockList.size()) { - if (getCurrentRightTime() > lastMatchedRightTime) { + if (lastMatchedRightBlockIsNull + || comparator.lessThan( + lastMatchedRightBlock, + 0, + 0, + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPosition, + rightIndex)) { for (int i = 0; i < leftOutputSymbolIdx.length; i++) { ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(i); columnBuilder.appendNull(); @@ -305,6 +355,78 @@ private void appendOneLeftRowWithEmptyRight() { resultBuilder.declarePosition(); } + private void initLastMatchedRightBlock(TsBlock block, int columnIndex, int rowIndex) { + lastMatchedRightBlockIsNull = false; + switch (joinKeyType.getTypeEnum()) { + case INT32: + case DATE: + lastMatchedRightBlock = + new TsBlock( + 1, + TIME_COLUMN_TEMPLATE, + new IntColumn( + 1, + Optional.empty(), + new int[] {block.getColumn(columnIndex).getInt(rowIndex)})); + break; + case INT64: + case TIMESTAMP: + lastMatchedRightBlock = + new TsBlock( + 1, + TIME_COLUMN_TEMPLATE, + new LongColumn( + 1, + Optional.empty(), + new long[] {block.getColumn(columnIndex).getLong(rowIndex)})); + break; + case FLOAT: + lastMatchedRightBlock = + new TsBlock( + 1, + TIME_COLUMN_TEMPLATE, + new FloatColumn( + 1, + Optional.empty(), + new float[] {block.getColumn(columnIndex).getFloat(rowIndex)})); + break; + case DOUBLE: + lastMatchedRightBlock = + new TsBlock( + 1, + TIME_COLUMN_TEMPLATE, + new DoubleColumn( + 1, + Optional.empty(), + new double[] {block.getColumn(columnIndex).getDouble(rowIndex)})); + break; + case BOOLEAN: + lastMatchedRightBlock = + new TsBlock( + 1, + TIME_COLUMN_TEMPLATE, + new BooleanColumn( + 1, + Optional.empty(), + new boolean[] {block.getColumn(columnIndex).getBoolean(rowIndex)})); + break; + case STRING: + case TEXT: + case BLOB: + lastMatchedRightBlock = + new TsBlock( + 1, + TIME_COLUMN_TEMPLATE, + new BinaryColumn( + 1, + Optional.empty(), + new Binary[] {block.getColumn(columnIndex).getBinary(rowIndex)})); + break; + default: + throw new UnsupportedOperationException("Unsupported data type: " + joinKeyType); + } + } + @Override public long calculateMaxPeekMemory() { return Math.max( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableInnerJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableInnerJoinOperator.java index 768d525fff82..393b50c0eeae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableInnerJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableInnerJoinOperator.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.AbstractOperator; import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; -import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.TimeComparator; +import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator.JoinKeyComparator; import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; import com.google.common.util.concurrent.ListenableFuture; @@ -33,6 +33,7 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.RamUsageEstimator; import java.util.ArrayList; @@ -50,21 +51,23 @@ public class TableInnerJoinOperator extends AbstractOperator { protected final Operator leftChild; protected TsBlock leftBlock; protected int leftIndex; // start index of leftTsBlock - protected final int leftTimeColumnPosition; + protected final int leftJoinKeyPosition; protected final int[] leftOutputSymbolIdx; protected final Operator rightChild; protected final List rightBlockList = new ArrayList<>(); - protected final int rightTimeColumnPosition; + protected final int rightJoinKeyPosition; protected int rightBlockListIdx; protected int rightIndex; // start index of rightTsBlock protected final int[] rightOutputSymbolIdx; protected TsBlock cachedNextRightBlock; protected boolean hasCachedNextRightBlock; - protected final TimeComparator comparator; + protected final JoinKeyComparator comparator; protected final TsBlockBuilder resultBuilder; + protected final Type joinKeyType; + protected MemoryReservationManager memoryReservationManager; protected long maxUsedMemory; @@ -73,22 +76,24 @@ public class TableInnerJoinOperator extends AbstractOperator { public TableInnerJoinOperator( OperatorContext operatorContext, Operator leftChild, - int leftTimeColumnPosition, + int leftJoinKeyPosition, int[] leftOutputSymbolIdx, Operator rightChild, - int rightTimeColumnPosition, + int rightJoinKeyPosition, int[] rightOutputSymbolIdx, - TimeComparator timeComparator, - List dataTypes) { + JoinKeyComparator joinKeyComparator, + List dataTypes, + Type joinKeyType) { this.operatorContext = operatorContext; this.leftChild = leftChild; - this.leftTimeColumnPosition = leftTimeColumnPosition; + this.leftJoinKeyPosition = leftJoinKeyPosition; this.leftOutputSymbolIdx = leftOutputSymbolIdx; this.rightChild = rightChild; - this.rightTimeColumnPosition = rightTimeColumnPosition; + this.rightJoinKeyPosition = rightJoinKeyPosition; this.rightOutputSymbolIdx = rightOutputSymbolIdx; - this.comparator = timeComparator; + this.comparator = joinKeyComparator; + this.joinKeyType = joinKeyType; this.resultBuilder = new TsBlockBuilder(dataTypes); this.memoryReservationManager = @@ -148,38 +153,33 @@ public TsBlock next() throws Exception { } // all the rightTsBlock is less than leftTsBlock, just skip right - if (comparator.lessThan(getRightEndTime(), getCurrentLeftTime())) { - // releaseMemory(); + if (allRightLessThanLeft()) { + // release memory; resetRightBlockList(); return null; } // all the leftTsBlock is less than rightTsBlock, just skip left - else if (comparator.lessThan(getLeftEndTime(), getRightTime(rightBlockListIdx, rightIndex))) { + else if (allLeftLessThanRight()) { leftBlock = null; leftIndex = 0; return null; } - long leftProbeTime = getCurrentLeftTime(); while (!resultBuilder.isFull()) { - - // all right block time is not matched - if (!comparator.canContinueInclusive(leftProbeTime, getRightEndTime())) { - // releaseMemory(); + // all right block value is not matched + if (allRightLessThanLeft()) { resetRightBlockList(); break; } - appendResult(leftProbeTime); + appendResult(); if (leftIndex >= leftBlock.getPositionCount()) { leftBlock = null; leftIndex = 0; break; } - - leftProbeTime = getCurrentLeftTime(); } if (resultBuilder.isEmpty()) { @@ -190,6 +190,66 @@ else if (comparator.lessThan(getLeftEndTime(), getRightTime(rightBlockListIdx, r return checkTsBlockSizeAndGetResult(); } + protected boolean allRightLessThanLeft() { + // check if the last value of the right is less than left + return comparator.lessThan( + rightBlockList.get(rightBlockList.size() - 1), + rightJoinKeyPosition, + rightBlockList.get(rightBlockList.size() - 1).getPositionCount() - 1, + leftBlock, + leftJoinKeyPosition, + leftIndex); + } + + protected boolean allLeftLessThanRight() { + return comparator.lessThan( + leftBlock, + leftJoinKeyPosition, + leftBlock.getPositionCount() - 1, + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPosition, + rightIndex); + } + + private void appendResult() { + while (comparator.lessThan( + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPosition, + rightIndex, + leftBlock, + leftJoinKeyPosition, + leftIndex)) { + if (rightBlockFinish()) { + return; + } + } + + int tmpBlockIdx = rightBlockListIdx, tmpIdx = rightIndex; + while (comparator.equalsTo( + leftBlock, + leftJoinKeyPosition, + leftIndex, + rightBlockList.get(tmpBlockIdx), + rightJoinKeyPosition, + tmpIdx)) { + appendValueToResult(tmpBlockIdx, tmpIdx); + + resultBuilder.declarePosition(); + + tmpIdx++; + if (tmpIdx >= rightBlockList.get(tmpBlockIdx).getPositionCount()) { + tmpIdx = 0; + tmpBlockIdx++; + } + + if (tmpBlockIdx >= rightBlockList.size()) { + break; + } + } + + leftIndex++; + } + protected boolean prepareInput(long start, long maxRuntime) throws Exception { if ((leftBlock == null || leftBlock.getPositionCount() == leftIndex) && leftChild.hasNextWithTimer()) { @@ -226,7 +286,13 @@ protected void tryCachedNextRightTsBlock() throws Exception { if (rightChild.hasNextWithTimer()) { TsBlock block = rightChild.nextWithTimer(); if (block != null) { - if (block.getColumn(rightTimeColumnPosition).getLong(0) == getRightEndTime()) { + if (comparator.equalsTo( + block, + rightJoinKeyPosition, + 0, + rightBlockList.get(rightBlockList.size() - 1), + rightJoinKeyPosition, + rightBlockList.get(rightBlockList.size() - 1).getPositionCount() - 1)) { reserveMemory(block.getRetainedSizeInBytes()); rightBlockList.add(block); } else { @@ -240,57 +306,6 @@ protected void tryCachedNextRightTsBlock() throws Exception { } } - protected long getCurrentLeftTime() { - return leftBlock.getColumn(leftTimeColumnPosition).getLong(leftIndex); - } - - protected long getLeftEndTime() { - return leftBlock.getColumn(leftTimeColumnPosition).getLong(leftBlock.getPositionCount() - 1); - } - - protected long getRightTime(int blockIdx, int rowIdx) { - return rightBlockList.get(blockIdx).getColumn(rightTimeColumnPosition).getLong(rowIdx); - } - - protected long getCurrentRightTime() { - return getRightTime(rightBlockListIdx, rightIndex); - } - - protected long getRightEndTime() { - TsBlock lastRightTsBlock = rightBlockList.get(rightBlockList.size() - 1); - return lastRightTsBlock - .getColumn(rightTimeColumnPosition) - .getLong(lastRightTsBlock.getPositionCount() - 1); - } - - protected void appendResult(long leftTime) { - - while (comparator.lessThan(getCurrentRightTime(), leftTime)) { - if (rightBlockFinish()) { - return; - } - } - - int tmpBlockIdx = rightBlockListIdx, tmpIdx = rightIndex; - while (leftTime == getRightTime(tmpBlockIdx, tmpIdx)) { - appendValueToResult(tmpBlockIdx, tmpIdx); - - resultBuilder.declarePosition(); - - tmpIdx++; - if (tmpIdx >= rightBlockList.get(tmpBlockIdx).getPositionCount()) { - tmpIdx = 0; - tmpBlockIdx++; - } - - if (tmpBlockIdx >= rightBlockList.size()) { - break; - } - } - - leftIndex++; - } - /** * @return true if right block is consumed up */ diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 6c3ea3c6852e..e04b2d91fc2b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -38,6 +38,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; import org.apache.iotdb.db.queryengine.execution.operator.process.CollectOperator; +import org.apache.iotdb.db.queryengine.execution.operator.process.EnforceSingleRowOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.FilterAndProjectOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.LimitOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.OffsetOperator; @@ -61,6 +62,8 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.gapfill.GapFillWGroupWoMoOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.gapfill.GapFillWoGroupWMoOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.gapfill.GapFillWoGroupWoMoOperator; +import org.apache.iotdb.db.queryengine.execution.operator.process.join.SimpleNestedLoopCrossJoinOperator; +import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator.JoinKeyComparatorFactory; import org.apache.iotdb.db.queryengine.execution.operator.schema.CountMergeOperator; import org.apache.iotdb.db.queryengine.execution.operator.schema.SchemaCountOperator; import org.apache.iotdb.db.queryengine.execution.operator.schema.SchemaQueryScanOperator; @@ -108,6 +111,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; @@ -184,7 +188,6 @@ import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.AccumulatorFactory.createAccumulator; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.AccumulatorFactory.createGroupedAccumulator; import static org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils.convertPredicateToFilter; -import static org.apache.iotdb.db.queryengine.plan.planner.OperatorTreeGenerator.ASC_TIME_COMPARATOR; import static org.apache.iotdb.db.queryengine.plan.planner.OperatorTreeGenerator.IDENTITY_FILL; import static org.apache.iotdb.db.queryengine.plan.planner.OperatorTreeGenerator.UNKNOWN_DATATYPE; import static org.apache.iotdb.db.queryengine.plan.planner.OperatorTreeGenerator.getLinearFill; @@ -1185,10 +1188,6 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { ImmutableMap leftColumnNamesMap = makeLayoutFromOutputSymbols(node.getLeftChild().getOutputSymbols()); - Integer leftTimeColumnPosition = leftColumnNamesMap.get(node.getCriteria().get(0).getLeft()); - if (leftTimeColumnPosition == null) { - throw new IllegalStateException("Left child of JoinNode doesn't contain time column"); - } int[] leftOutputSymbolIdx = new int[node.getLeftOutputSymbols().size()]; for (int i = 0; i < leftOutputSymbolIdx.length; i++) { Integer index = leftColumnNamesMap.get(node.getLeftOutputSymbols().get(i)); @@ -1202,10 +1201,6 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { ImmutableMap rightColumnNamesMap = makeLayoutFromOutputSymbols(node.getRightChild().getOutputSymbols()); - Integer rightTimeColumnPosition = rightColumnNamesMap.get(node.getCriteria().get(0).getRight()); - if (rightTimeColumnPosition == null) { - throw new IllegalStateException("Right child of JoinNode doesn't contain time column"); - } int[] rightOutputSymbolIdx = new int[node.getRightOutputSymbols().size()]; for (int i = 0; i < rightOutputSymbolIdx.length; i++) { Integer index = rightColumnNamesMap.get(node.getRightOutputSymbols().get(i)); @@ -1217,6 +1212,42 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { rightOutputSymbolIdx[i] = index; } + // cross join does not need time column + if (node.isCrossJoin()) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + SimpleNestedLoopCrossJoinOperator.class.getSimpleName()); + return new SimpleNestedLoopCrossJoinOperator( + operatorContext, + leftChild, + rightChild, + leftOutputSymbolIdx, + rightOutputSymbolIdx, + dataTypes); + } + + Integer leftJoinKeyPosition = leftColumnNamesMap.get(node.getCriteria().get(0).getLeft()); + if (leftJoinKeyPosition == null) { + throw new IllegalStateException("Left child of JoinNode doesn't contain left join key."); + } + + Integer rightJoinKeyPosition = rightColumnNamesMap.get(node.getCriteria().get(0).getRight()); + if (rightJoinKeyPosition == null) { + throw new IllegalStateException("Right child of JoinNode doesn't contain right join key."); + } + + Type leftJoinKeyType = + context.getTypeProvider().getTableModelType(node.getCriteria().get(0).getLeft()); + + checkArgument( + leftJoinKeyType + == context.getTypeProvider().getTableModelType(node.getCriteria().get(0).getRight()), + "Join key type mismatch."); + if (requireNonNull(node.getJoinType()) == JoinNode.JoinType.INNER) { OperatorContext operatorContext = context @@ -1228,13 +1259,14 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { return new TableInnerJoinOperator( operatorContext, leftChild, - leftTimeColumnPosition, + leftJoinKeyPosition, leftOutputSymbolIdx, rightChild, - rightTimeColumnPosition, + rightJoinKeyPosition, rightOutputSymbolIdx, - ASC_TIME_COMPARATOR, - dataTypes); + JoinKeyComparatorFactory.getComparator(leftJoinKeyType, true), + dataTypes, + leftJoinKeyType); } else if (requireNonNull(node.getJoinType()) == JoinNode.JoinType.FULL) { OperatorContext operatorContext = context @@ -1246,18 +1278,34 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { return new TableFullOuterJoinOperator( operatorContext, leftChild, - leftTimeColumnPosition, + leftJoinKeyPosition, leftOutputSymbolIdx, rightChild, - rightTimeColumnPosition, + rightJoinKeyPosition, rightOutputSymbolIdx, - ASC_TIME_COMPARATOR, - dataTypes); + JoinKeyComparatorFactory.getComparator(leftJoinKeyType, true), + dataTypes, + leftJoinKeyType); } throw new IllegalStateException("Unsupported join type: " + node.getJoinType()); } + @Override + public Operator visitEnforceSingleRow( + EnforceSingleRowNode node, LocalExecutionPlanContext context) { + Operator child = node.getChild().accept(this, context); + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + EnforceSingleRowOperator.class.getSimpleName()); + + return new EnforceSingleRowOperator(operatorContext, child); + } + @Override public Operator visitCountMerge( final CountSchemaMergeNode node, final LocalExecutionPlanContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java index 1497928ffddc..1939f6c6d75d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java @@ -65,6 +65,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.CrossSeriesAggregationDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.DeviceViewIntoPathDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.IntoPathDescriptor; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; @@ -808,6 +809,13 @@ public List visitLimit( return render(node, boxValue, context); } + @Override + public List visitEnforceSingleRow(EnforceSingleRowNode node, GraphContext context) { + List boxValue = new ArrayList<>(); + boxValue.add(String.format("EnforceSingleRow-%s", node.getPlanNodeId().getId())); + return render(node, boxValue, context); + } + @Override public List visitOffset( org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode node, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index d949cc1be5b3..3e53bc696909 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -116,6 +116,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; @@ -278,6 +279,7 @@ public enum PlanNodeType { TABLE_GAP_FILL_NODE((short) 1017), TABLE_EXCHANGE_NODE((short) 1018), TABLE_EXPLAIN_ANALYZE_NODE((short) 1019), + TABLE_ENFORCE_SINGLE_ROW_NODE((short) 1020), RELATIONAL_INSERT_TABLET((short) 2000), RELATIONAL_INSERT_ROW((short) 2001), @@ -633,6 +635,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { .deserialize(buffer); case 1019: throw new UnsupportedOperationException("ExplainAnalyzeNode should not be deserialized"); + case 1020: + return EnforceSingleRowNode.deserialize(buffer); case 2000: return RelationalInsertTabletNode.deserialize(buffer); case 2001: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index ef0fe78f3379..682f2a65202a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -642,6 +642,23 @@ public R visitFilter( return visitSingleChildProcess(node, context); } + public R visitApply( + org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode node, C context) { + return visitTwoChildProcess(node, context); + } + + public R visitEnforceSingleRow( + org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode node, + C context) { + return visitSingleChildProcess(node, context); + } + + public R visitCorrelatedJoin( + org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode node, + C context) { + return visitTwoChildProcess(node, context); + } + public R visitTableScan(TableScanNode node, C context) { return visitPlan(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 96196da6dced..b9214507821d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -141,6 +141,8 @@ public class Analysis implements IAnalysis { private final Map, Expression> joins = new LinkedHashMap<>(); private final Map, JoinUsingAnalysis> joinUsing = new LinkedHashMap<>(); private final Map, SubqueryAnalysis> subQueries = new LinkedHashMap<>(); + private final Map, PredicateCoercions> predicateCoercions = + new LinkedHashMap<>(); private final Map, TableEntry> tables = new LinkedHashMap<>(); @@ -680,6 +682,14 @@ public Map getTableColumnSchema(QualifiedObjectName qualif return tableColumnSchemas.get(qualifiedObjectName); } + public void addPredicateCoercions(Map, PredicateCoercions> coercions) { + predicateCoercions.putAll(coercions); + } + + public PredicateCoercions getPredicateCoercions(Expression expression) { + return predicateCoercions.get(NodeRef.of(expression)); + } + public boolean hasValueFilter() { return hasValueFilter; } @@ -1108,6 +1118,35 @@ public List getQuantifiedComparisonSubqueries() } } + /** + * Analysis for predicates such as x IN (subquery) or x = SOME (subquery) + * + */ + public static class PredicateCoercions { + private final Type valueType; + private final Optional valueCoercion; + private final Optional subqueryCoercion; + + public PredicateCoercions( + Type valueType, Optional valueCoercion, Optional subqueryCoercion) { + this.valueType = requireNonNull(valueType, "valueType is null"); + this.valueCoercion = requireNonNull(valueCoercion, "valueCoercion is null"); + this.subqueryCoercion = requireNonNull(subqueryCoercion, "subqueryCoercion is null"); + } + + public Type getValueType() { + return valueType; + } + + public Optional getValueCoercion() { + return valueCoercion; + } + + public Optional getSubqueryCoercion() { + return subqueryCoercion; + } + } + public static class FillAnalysis { protected final FillPolicy fillMethod; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index ecfb0faaa574..404c88b2e1e8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -137,6 +137,8 @@ public class ExpressionAnalyzer { private final Set> existsSubqueries = new LinkedHashSet<>(); private final Set> subqueryInPredicates = new LinkedHashSet<>(); + private final Map, Analysis.PredicateCoercions> predicateCoercions = + new LinkedHashMap<>(); private final Map, ResolvedField> columnReferences = new LinkedHashMap<>(); private final Map, Type> expressionTypes = new LinkedHashMap<>(); @@ -236,6 +238,10 @@ public Set> getSubqueryInPredicates() { return unmodifiableSet(subqueryInPredicates); } + public Map, Analysis.PredicateCoercions> getPredicateCoercions() { + return unmodifiableMap(predicateCoercions); + } + public Map, ResolvedField> getColumnReferences() { return unmodifiableMap(columnReferences); } @@ -1005,6 +1011,20 @@ private Type analyzePredicateWithSubquery( valueRowType, subqueryType)); } + Optional valueCoercion = Optional.empty(); + // if (!valueRowType.equals(commonType.get())) { + // valueCoercion = commonType; + // } + + Optional subQueryCoercion = Optional.empty(); + // if (!subqueryType.equals(commonType.get())) { + // subQueryCoercion = commonType; + // } + + predicateCoercions.put( + NodeRef.of(node), + new Analysis.PredicateCoercions(valueRowType, valueCoercion, subQueryCoercion)); + return subqueryType; } @@ -1466,6 +1486,7 @@ private static void updateAnalysis( analysis.addColumnReferences(analyzer.getColumnReferences()); analysis.addTableColumnReferences( accessControl, session.getIdentity(), analyzer.getTableColumnReferences()); + analysis.addPredicateCoercions(analyzer.getPredicateCoercions()); } public static ExpressionAnalyzer createConstantAnalyzer( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java index 928c4f5c17a2..371016ad0809 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java @@ -15,8 +15,11 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.SimplePlanVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@ -73,20 +76,18 @@ public Void visitPlan(PlanNode node, Void context) { return null; } - /*@Override - public Void visitGroupReference(GroupReference node, Void context) - { - return lookup.resolve(node).accept(this, context); + @Override + public Void visitGroupReference(GroupReference node, Void context) { + return lookup.resolve(node).accept(this, context); } @Override - public Void visitAggregation(AggregationNode node, Void context) - { - for (Aggregation aggregation : node.getAggregations().values()) { - aggregation.getArguments().forEach(consumer); - } - return super.visitAggregation(node, context); - }*/ + public Void visitAggregation(AggregationNode node, Void context) { + for (AggregationNode.Aggregation aggregation : node.getAggregations().values()) { + aggregation.getArguments().forEach(consumer); + } + return super.visitAggregation(node, context); + } @Override public Void visitFilter(FilterNode node, Void context) { @@ -100,14 +101,13 @@ public Void visitProject(ProjectNode node, Void context) { return super.visitProject(node, context); } - /*@Override - public Void visitJoin(JoinNode node, Void context) - { - node.getFilter().ifPresent(consumer); - return super.visitJoin(node, context); + @Override + public Void visitJoin(JoinNode node, Void context) { + node.getFilter().ifPresent(consumer); + return super.visitJoin(node, context); } - @Override + /*@Override public Void visitValues(ValuesNode node, Void context) { node.getRows().ifPresent(list -> list.forEach(consumer)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 4c47bb90da81..7c2664b8fd13 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -161,7 +161,7 @@ public RelationPlan plan(Query query) { public RelationPlan plan(QuerySpecification node) { PlanBuilder builder = planFrom(node); - builder = filter(builder, analysis.getWhere(node)); + builder = filter(builder, analysis.getWhere(node), node); Expression wherePredicate = null; if (builder.getRoot() instanceof FilterNode) { wherePredicate = ((FilterNode) builder.getRoot()).getPredicate(); @@ -173,7 +173,7 @@ public RelationPlan plan(QuerySpecification node) { timeColumnForGapFill = builder.translate((Expression) gapFillColumn.getChildren().get(2)); } builder = aggregate(builder, node); - builder = filter(builder, analysis.getHaving(node)); + builder = filter(builder, analysis.getHaving(node), node); if (gapFillColumn != null) { if (wherePredicate == null) { @@ -189,13 +189,13 @@ public RelationPlan plan(QuerySpecification node) { } List selectExpressions = analysis.getSelectExpressions(node); + List expressions = + selectExpressions.stream() + .map(Analysis.SelectExpression::getExpression) + .collect(toImmutableList()); + builder = subqueryPlanner.handleSubqueries(builder, expressions, analysis.getSubqueries(node)); if (hasExpressionsToUnfold(selectExpressions)) { - List expressions = - selectExpressions.stream() - .map(Analysis.SelectExpression::getExpression) - .collect(toImmutableList()); - // pre-project the folded expressions to preserve any non-deterministic semantics of functions // that might be referenced builder = builder.appendProjections(expressions, symbolAllocator, queryContext); @@ -325,13 +325,12 @@ private PlanBuilder planFrom(QuerySpecification node) { } } - private PlanBuilder filter(PlanBuilder subPlan, Expression predicate) { + private PlanBuilder filter(PlanBuilder subPlan, Expression predicate, Node node) { if (predicate == null) { return subPlan; } - // planBuilder = subqueryPlanner.handleSubqueries(subPlan, predicate, - // analysis.getSubqueries(node)); + subPlan = subqueryPlanner.handleSubqueries(subPlan, predicate, analysis.getSubqueries(node)); return subPlan.withNewRoot( new FilterNode( queryIdAllocator.genPlanNodeId(), subPlan.getRoot(), subPlan.rewrite(predicate))); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 04fa9b2ee497..914a47fe0cef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -610,6 +610,11 @@ protected RelationPlan visitAliasedRelation(AliasedRelation node, Void context) return new RelationPlan(root, analysis.getScope(node), mappings, outerContext); } + @Override + protected RelationPlan visitSubqueryExpression(SubqueryExpression node, Void context) { + return process(node.getQuery(), context); + } + // ================================ Implemented later ===================================== @Override @@ -617,11 +622,6 @@ protected RelationPlan visitValues(Values node, Void context) { throw new IllegalStateException("Values is not supported in current version."); } - @Override - protected RelationPlan visitSubqueryExpression(SubqueryExpression node, Void context) { - throw new IllegalStateException("SubqueryExpression is not supported in current version."); - } - @Override protected RelationPlan visitIntersect(Intersect node, Void context) { throw new IllegalStateException("Intersect is not supported in current version."); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java index 573f0cb335d8..82f90ac36da9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java @@ -16,20 +16,31 @@ import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Field; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.RelationType; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Scope; import org.apache.iotdb.db.queryengine.plan.relational.planner.QueryPlanner.PlanAndMappings; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Cast; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -47,10 +58,15 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.collect.Streams.stream; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static org.apache.iotdb.db.queryengine.plan.relational.planner.PlanBuilder.newPlanBuilder; import static org.apache.iotdb.db.queryengine.plan.relational.planner.ScopeAware.scopeAwareKey; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression.Quantifier.ALL; +import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toSqlType; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; class SubqueryPlanner { @@ -95,18 +111,31 @@ public PlanBuilder handleSubqueries( public PlanBuilder handleSubqueries( PlanBuilder builder, Expression expression, Analysis.SubqueryAnalysis subqueries) { - /*for (Cluster cluster : cluster(builder.getScope(), selectSubqueries(builder, expression, subqueries.getInPredicatesSubqueries()))) { - builder = planInPredicate(builder, cluster, subqueries); + for (Cluster cluster : + cluster( + builder.getScope(), + selectSubqueries(builder, expression, subqueries.getInPredicatesSubqueries()))) { + builder = planInPredicate(builder, cluster, subqueries); } - for (Cluster cluster : cluster(builder.getScope(), selectSubqueries(builder, expression, subqueries.getSubqueries()))) { - builder = planScalarSubquery(builder, cluster); + for (Cluster cluster : + cluster( + builder.getScope(), + selectSubqueries(builder, expression, subqueries.getSubqueries()))) { + builder = planScalarSubquery(builder, cluster); } - for (Cluster cluster : cluster(builder.getScope(), selectSubqueries(builder, expression, subqueries.getExistsSubqueries()))) { - builder = planExists(builder, cluster); + for (Cluster cluster : + cluster( + builder.getScope(), + selectSubqueries(builder, expression, subqueries.getExistsSubqueries()))) { + builder = planExists(builder, cluster); + } + for (Cluster cluster : + cluster( + builder.getScope(), + selectSubqueries( + builder, expression, subqueries.getQuantifiedComparisonSubqueries()))) { + builder = planQuantifiedComparison(builder, cluster, subqueries); } - for (Cluster cluster : cluster(builder.getScope(), selectSubqueries(builder, expression, subqueries.getQuantifiedComparisonSubqueries()))) { - builder = planQuantifiedComparison(builder, cluster, subqueries); - }*/ return builder; } @@ -156,6 +185,142 @@ private Collection> cluster(Scope scope, List< .collect(toImmutableList()); } + private PlanBuilder planInPredicate( + PlanBuilder subPlan, Cluster cluster, Analysis.SubqueryAnalysis subqueries) { + // Plan one of the predicates from the cluster + InPredicate predicate = cluster.getRepresentative(); + + Expression value = predicate.getValue(); + SubqueryExpression subquery = (SubqueryExpression) predicate.getValueList(); + Symbol output = symbolAllocator.newSymbol(predicate, BOOLEAN); + + subPlan = handleSubqueries(subPlan, value, subqueries); + subPlan = + planInPredicate( + subPlan, value, subquery, output, predicate, analysis.getPredicateCoercions(predicate)); + + return new PlanBuilder( + subPlan + .getTranslations() + .withAdditionalMappings(mapAll(cluster, subPlan.getScope(), output)), + subPlan.getRoot()); + } + + /** + * Plans a correlated subquery for value IN (subQuery) + * + * @param originalExpression the original expression from which the IN predicate was derived. Used + * for subsequent translations. + */ + private PlanBuilder planInPredicate( + PlanBuilder subPlan, + Expression value, + SubqueryExpression subquery, + Symbol output, + Expression originalExpression, + Analysis.PredicateCoercions predicateCoercions) { + PlanAndMappings subqueryPlan = + planSubquery(subquery, predicateCoercions.getSubqueryCoercion(), subPlan.getTranslations()); + PlanAndMappings valuePlan = + planValue( + subPlan, + value, + predicateCoercions.getValueType(), + predicateCoercions.getValueCoercion()); + + return new PlanBuilder( + valuePlan.getSubPlan().getTranslations(), + new ApplyNode( + idAllocator.genPlanNodeId(), + valuePlan.getSubPlan().getRoot(), + subqueryPlan.getSubPlan().getRoot(), + ImmutableMap.of( + output, new ApplyNode.In(valuePlan.get(value), subqueryPlan.get(subquery))), + valuePlan.getSubPlan().getRoot().getOutputSymbols(), + originalExpression)); + } + + private PlanBuilder planScalarSubquery(PlanBuilder subPlan, Cluster cluster) { + // Plan one of the predicates from the cluster + SubqueryExpression scalarSubquery = cluster.getRepresentative(); + + RelationPlan relationPlan = planSubquery(scalarSubquery, subPlan.getTranslations()); + PlanBuilder subqueryPlan = newPlanBuilder(relationPlan, analysis); + + PlanNode root = new EnforceSingleRowNode(idAllocator.genPlanNodeId(), subqueryPlan.getRoot()); + + Type type = analysis.getType(scalarSubquery); + RelationType descriptor = relationPlan.getDescriptor(); + List fieldMappings = relationPlan.getFieldMappings(); + Symbol column; + if (descriptor.getVisibleFieldCount() > 1) { + column = symbolAllocator.newSymbol("row", type); + + ImmutableList.Builder fields = ImmutableList.builder(); + for (int i = 0; i < descriptor.getAllFieldCount(); i++) { + Field field = descriptor.getFieldByIndex(i); + if (!field.isHidden()) { + fields.add(fieldMappings.get(i).toSymbolReference()); + } + } + + Expression expression = new Cast(new Row(fields.build()), toSqlType(type)); + + root = new ProjectNode(idAllocator.genPlanNodeId(), root, Assignments.of(column, expression)); + } else { + column = getOnlyElement(fieldMappings); + } + + return appendCorrelatedJoin( + subPlan, + root, + scalarSubquery.getQuery(), + // Scalar subquery always contains EnforceSingleRowNode. Therefore, it's guaranteed + // that subquery will return single row. Hence, correlated join can be of INNER type. + JoinNode.JoinType.INNER, + TRUE_LITERAL, + mapAll(cluster, subPlan.getScope(), column)); + } + + public PlanBuilder appendCorrelatedJoin( + PlanBuilder subPlan, + PlanNode subquery, + Query query, + JoinNode.JoinType type, + Expression filterCondition, + Map, Symbol> mappings) { + return new PlanBuilder( + subPlan.getTranslations().withAdditionalMappings(mappings), + new CorrelatedJoinNode( + idAllocator.genPlanNodeId(), + subPlan.getRoot(), + subquery, + subPlan.getRoot().getOutputSymbols(), + type, + filterCondition, + query)); + } + + private PlanBuilder planExists(PlanBuilder subPlan, Cluster cluster) { + // Plan one of the predicates from the cluster + ExistsPredicate existsPredicate = cluster.getRepresentative(); + + Expression subquery = existsPredicate.getSubquery(); + Symbol exists = symbolAllocator.newSymbol("exists", BOOLEAN); + + return new PlanBuilder( + subPlan + .getTranslations() + .withAdditionalMappings(mapAll(cluster, subPlan.getScope(), exists)), + new ApplyNode( + idAllocator.genPlanNodeId(), + subPlan.getRoot(), + planSubquery(subquery, subPlan.getTranslations()).getRoot(), + ImmutableMap.of(exists, new ApplyNode.Exists()), + subPlan.getRoot().getOutputSymbols(), + subquery)); + } + private RelationPlan planSubquery(Expression subquery, TranslationMap outerContext) { return new RelationPlanner( analysis, @@ -167,6 +332,103 @@ private RelationPlan planSubquery(Expression subquery, TranslationMap outerConte .process(subquery, null); } + private PlanBuilder planQuantifiedComparison( + PlanBuilder subPlan, + Cluster cluster, + Analysis.SubqueryAnalysis subqueries) { + // Plan one of the predicates from the cluster + QuantifiedComparisonExpression quantifiedComparison = cluster.getRepresentative(); + + ComparisonExpression.Operator operator = quantifiedComparison.getOperator(); + QuantifiedComparisonExpression.Quantifier quantifier = quantifiedComparison.getQuantifier(); + Expression value = quantifiedComparison.getValue(); + SubqueryExpression subquery = (SubqueryExpression) quantifiedComparison.getSubquery(); + + subPlan = handleSubqueries(subPlan, value, subqueries); + + Symbol output = symbolAllocator.newSymbol(quantifiedComparison, BOOLEAN); + + Analysis.PredicateCoercions predicateCoercions = + analysis.getPredicateCoercions(quantifiedComparison); + + switch (operator) { + case EQUAL: + switch (quantifier) { + case ALL: + subPlan = + planQuantifiedComparison( + subPlan, operator, quantifier, value, subquery, output, predicateCoercions); + return new PlanBuilder( + subPlan + .getTranslations() + .withAdditionalMappings( + ImmutableMap.of( + scopeAwareKey(quantifiedComparison, analysis, subPlan.getScope()), + output)), + subPlan.getRoot()); + case ANY: + case SOME: + // A = ANY B <=> A IN B + subPlan = + planInPredicate( + subPlan, value, subquery, output, quantifiedComparison, predicateCoercions); + return new PlanBuilder( + subPlan + .getTranslations() + .withAdditionalMappings(mapAll(cluster, subPlan.getScope(), output)), + subPlan.getRoot()); + default: + throw new IllegalArgumentException(); + } + case NOT_EQUAL: + switch (quantifier) { + case ALL: + // A <> ALL B <=> !(A IN B) + return addNegation( + planInPredicate( + subPlan, value, subquery, output, quantifiedComparison, predicateCoercions), + cluster, + output); + case ANY: + case SOME: + // A <> ANY B <=> min B <> max B || A <> min B <=> !(min B = max B && A = min B) <=> !(A + // = ALL B) + // "A <> ANY B" is equivalent to "NOT (A = ALL B)" so add a rewrite for the initial + // quantifiedComparison to notAll + return addNegation( + planQuantifiedComparison( + subPlan, + ComparisonExpression.Operator.EQUAL, + ALL, + value, + subquery, + output, + predicateCoercions), + cluster, + output); + default: + throw new IllegalArgumentException(); + } + case LESS_THAN: + case LESS_THAN_OR_EQUAL: + case GREATER_THAN: + case GREATER_THAN_OR_EQUAL: + subPlan = + planQuantifiedComparison( + subPlan, operator, quantifier, value, subquery, output, predicateCoercions); + return new PlanBuilder( + subPlan + .getTranslations() + .withAdditionalMappings(mapAll(cluster, subPlan.getScope(), output)), + subPlan.getRoot()); + // Cannot be used with quantified comparison + case IS_DISTINCT_FROM: + default: + throw new IllegalArgumentException( + format("Unexpected quantified comparison: '%s %s'", operator.getValue(), quantifier)); + } + } + /** * Adds a negation of the given input and remaps the provided expression to the negated expression */ @@ -187,6 +449,74 @@ private PlanBuilder addNegation( .build())); } + private PlanBuilder planQuantifiedComparison( + PlanBuilder subPlan, + ComparisonExpression.Operator operator, + QuantifiedComparisonExpression.Quantifier quantifier, + Expression value, + Expression subquery, + Symbol assignment, + Analysis.PredicateCoercions predicateCoercions) { + PlanAndMappings subqueryPlan = + planSubquery(subquery, predicateCoercions.getSubqueryCoercion(), subPlan.getTranslations()); + PlanAndMappings valuePlan = + planValue( + subPlan, + value, + predicateCoercions.getValueType(), + predicateCoercions.getValueCoercion()); + + return new PlanBuilder( + valuePlan.getSubPlan().getTranslations(), + new ApplyNode( + idAllocator.genPlanNodeId(), + valuePlan.getSubPlan().getRoot(), + subqueryPlan.getSubPlan().getRoot(), + ImmutableMap.of( + assignment, + new ApplyNode.QuantifiedComparison( + mapOperator(operator), + mapQuantifier(quantifier), + valuePlan.get(value), + subqueryPlan.get(subquery))), + valuePlan.getSubPlan().getRoot().getOutputSymbols(), + subquery)); + } + + private static ApplyNode.Quantifier mapQuantifier( + QuantifiedComparisonExpression.Quantifier quantifier) { + switch (quantifier) { + case ALL: + return ApplyNode.Quantifier.ALL; + case ANY: + return ApplyNode.Quantifier.ANY; + case SOME: + return ApplyNode.Quantifier.SOME; + default: + throw new IllegalArgumentException(); + } + } + + private static ApplyNode.Operator mapOperator(ComparisonExpression.Operator operator) { + switch (operator) { + case EQUAL: + return ApplyNode.Operator.EQUAL; + case NOT_EQUAL: + return ApplyNode.Operator.NOT_EQUAL; + case LESS_THAN: + return ApplyNode.Operator.LESS_THAN; + case LESS_THAN_OR_EQUAL: + return ApplyNode.Operator.LESS_THAN_OR_EQUAL; + case GREATER_THAN: + return ApplyNode.Operator.GREATER_THAN; + case GREATER_THAN_OR_EQUAL: + return ApplyNode.Operator.GREATER_THAN_OR_EQUAL; + case IS_DISTINCT_FROM: + default: + throw new IllegalArgumentException(); + } + } + private PlanAndMappings planValue( PlanBuilder subPlan, Expression value, Type actualType, Optional coercion) { subPlan = subPlan.appendProjections(ImmutableList.of(value), symbolAllocator, plannerContext); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 9c151a19b5e0..b48abe29ef54 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; @@ -155,11 +156,6 @@ public List visitOutput(OutputNode node, PlanContext context) { nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); } - if (childrenNodes.size() == 1) { - node.setChild(childrenNodes.get(0)); - return Collections.singletonList(node); - } - node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); return Collections.singletonList(node); } @@ -175,11 +171,6 @@ public List visitFill(FillNode node, PlanContext context) { nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); } - if (childrenNodes.size() == 1) { - node.setChild(childrenNodes.get(0)); - return Collections.singletonList(node); - } - node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); return Collections.singletonList(node); } @@ -193,11 +184,6 @@ public List visitGapFill(GapFillNode node, PlanContext context) { nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); } - if (childrenNodes.size() == 1) { - node.setChild(childrenNodes.get(0)); - return Collections.singletonList(node); - } - node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); return Collections.singletonList(node); } @@ -210,11 +196,6 @@ public List visitLimit(LimitNode node, PlanContext context) { nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); } - if (childrenNodes.size() == 1) { - node.setChild(childrenNodes.get(0)); - return Collections.singletonList(node); - } - // push down LimitNode in distributed plan optimize rule node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); return Collections.singletonList(node); @@ -228,11 +209,6 @@ public List visitOffset(OffsetNode node, PlanContext context) { nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); } - if (childrenNodes.size() == 1) { - node.setChild(childrenNodes.get(0)); - return Collections.singletonList(node); - } - node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); return Collections.singletonList(node); } @@ -442,18 +418,27 @@ public List visitFilter(FilterNode node, PlanContext context) { @Override public List visitJoin(JoinNode node, PlanContext context) { - // child of JoinNode must be SortNode, so after rewritten, the child must be MergeSortNode or - // SortNode - List leftChildrenNodes = node.getLeftChild().accept(this, context); - checkArgument( - leftChildrenNodes.size() == 1, "The size of left children node of JoinNode should be 1"); - node.setLeftChild(leftChildrenNodes.get(0)); + List leftChildrenNodes = node.getLeftChild().accept(this, context); List rightChildrenNodes = node.getRightChild().accept(this, context); - checkArgument( - rightChildrenNodes.size() == 1, "The size of right children node of JoinNode should be 1"); - node.setRightChild(rightChildrenNodes.get(0)); - + if (!node.isCrossJoin()) { + // child of JoinNode(excluding CrossJoin) must be SortNode, so after rewritten, the child must + // be MergeSortNode or + // SortNode + checkArgument( + leftChildrenNodes.size() == 1, "The size of left children node of JoinNode should be 1"); + checkArgument( + rightChildrenNodes.size() == 1, + "The size of right children node of JoinNode should be 1"); + } + // For CrossJoinNode, we need to merge children nodes(It's safe for other JoinNodes here since + // the size of their children is always 1.) + node.setLeftChild( + mergeChildrenViaCollectOrMergeSort( + nodeOrderingMap.get(node.getLeftChild().getPlanNodeId()), leftChildrenNodes)); + node.setRightChild( + mergeChildrenViaCollectOrMergeSort( + nodeOrderingMap.get(node.getRightChild().getPlanNodeId()), rightChildrenNodes)); return Collections.singletonList(node); } @@ -656,6 +641,18 @@ public List visitAggregationTableScan( return resultTableScanNodeList; } + @Override + public List visitEnforceSingleRow(EnforceSingleRowNode node, PlanContext context) { + List childrenNodes = node.getChild().accept(this, context); + OrderingScheme childOrdering = nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); + if (childOrdering != null) { + nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); + } + + node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); + return Collections.singletonList(node); + } + private void buildRegionNodeMap( AggregationTableScanNode originalAggTableScanNode, List> regionReplicaSetsList, @@ -707,6 +704,12 @@ private static OrderingScheme constructOrderingSchema(List symbols) { private PlanNode mergeChildrenViaCollectOrMergeSort( OrderingScheme childOrdering, List childrenNodes) { + checkArgument(!childrenNodes.isEmpty(), "childrenNodes should not be empty"); + + if (childrenNodes.size() == 1) { + return childrenNodes.get(0); + } + PlanNode firstChild = childrenNodes.get(0); // children has sort property, use MergeSort to merge children diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java index 8d8c56696ed1..d232101d6037 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentDatabase; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentUser; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; @@ -48,6 +49,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause; @@ -148,6 +150,16 @@ public Expression rewriteInListExpression( return rewriteExpression(node, context, treeRewriter); } + public Expression rewriteExists( + ExistsPredicate node, C context, ExpressionTreeRewriter treeRewriter) { + return rewriteExpression(node, context, treeRewriter); + } + + public Expression rewriteSubqueryExpression( + SubqueryExpression node, C context, ExpressionTreeRewriter treeRewriter) { + return rewriteExpression(node, context, treeRewriter); + } + public Expression rewriteFunctionCall( FunctionCall node, C context, ExpressionTreeRewriter treeRewriter) { return rewriteExpression(node, context, treeRewriter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java index 6b547bc8127d..cab412becfcb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataTypeParameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; @@ -51,6 +52,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TypeParameter; @@ -529,6 +531,39 @@ protected Expression visitInListExpression(InListExpression node, Context con } @Override + protected Expression visitExists(ExistsPredicate node, Context context) { + if (!context.isDefaultRewrite()) { + Expression result = + rewriter.rewriteExists(node, context.get(), ExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + Expression subquery = node.getSubquery(); + subquery = rewrite(subquery, context.get()); + + if (subquery != node.getSubquery()) { + return new ExistsPredicate(subquery); + } + + return node; + } + + @Override + public Expression visitSubqueryExpression(SubqueryExpression node, Context context) { + if (!context.isDefaultRewrite()) { + Expression result = + rewriter.rewriteSubqueryExpression(node, context.get(), ExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + // No default rewrite for SubqueryExpression since we do not want to traverse subqueries + return node; + } + protected Expression visitLiteral(final Literal node, final Context context) { if (!context.isDefaultRewrite()) { final Expression result = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinColumns.java new file mode 100644 index 000000000000..70d4f3021ddd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinColumns.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; + +import com.google.common.collect.ImmutableSet; + +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.Sets.intersection; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor.extractUnique; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.Util.restrictOutputs; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.correlatedJoin; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isAtMostScalar; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isScalar; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; + +public class PruneCorrelatedJoinColumns extends ProjectOffPushDownRule { + public PruneCorrelatedJoinColumns() { + super(correlatedJoin()); + } + + @Override + protected Optional pushDownProjectOff( + Context context, CorrelatedJoinNode correlatedJoinNode, Set referencedOutputs) { + PlanNode input = correlatedJoinNode.getInput(); + PlanNode subquery = correlatedJoinNode.getSubquery(); + + // remove unused correlated join node, retain input + if (intersection(ImmutableSet.copyOf(subquery.getOutputSymbols()), referencedOutputs) + .isEmpty()) { + // remove unused subquery of inner join + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.INNER + && isScalar(subquery, context.getLookup()) + && correlatedJoinNode.getFilter().equals(TRUE_LITERAL)) { + return Optional.of(input); + } + // remove unused subquery of left join + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.LEFT + && isAtMostScalar(subquery, context.getLookup())) { + return Optional.of(input); + } + } + + Set referencedAndCorrelationSymbols = + ImmutableSet.builder() + .addAll(referencedOutputs) + .addAll(correlatedJoinNode.getCorrelation()) + .build(); + + // remove unused input node, retain subquery + if (intersection(ImmutableSet.copyOf(input.getOutputSymbols()), referencedAndCorrelationSymbols) + .isEmpty()) { + // remove unused input of inner join + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.INNER + && isScalar(input, context.getLookup()) + && correlatedJoinNode.getFilter().equals(TRUE_LITERAL)) { + return Optional.of(subquery); + } + // remove unused input of right join + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.RIGHT + && isAtMostScalar(input, context.getLookup())) { + return Optional.of(subquery); + } + } + + Set filterSymbols = extractUnique(correlatedJoinNode.getFilter()); + + Set referencedAndFilterSymbols = + ImmutableSet.builder().addAll(referencedOutputs).addAll(filterSymbols).build(); + + Optional newSubquery = + restrictOutputs(context.getIdAllocator(), subquery, referencedAndFilterSymbols); + + Set referencedAndFilterAndCorrelationSymbols = + ImmutableSet.builder() + .addAll(referencedAndFilterSymbols) + .addAll(correlatedJoinNode.getCorrelation()) + .build(); + + Optional newInput = + restrictOutputs(context.getIdAllocator(), input, referencedAndFilterAndCorrelationSymbols); + + boolean pruned = newSubquery.isPresent() || newInput.isPresent(); + + if (pruned) { + return Optional.of( + new CorrelatedJoinNode( + correlatedJoinNode.getPlanNodeId(), + newInput.orElse(input), + newSubquery.orElse(subquery), + correlatedJoinNode.getCorrelation(), + correlatedJoinNode.getJoinType(), + correlatedJoinNode.getFilter(), + correlatedJoinNode.getOriginSubquery())); + } + + return Optional.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinCorrelation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinCorrelation.java new file mode 100644 index 000000000000..c8ec3adeea04 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneCorrelatedJoinCorrelation.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import java.util.List; +import java.util.Set; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor.extractUnique; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.correlatedJoin; + +public class PruneCorrelatedJoinCorrelation implements Rule { + private static final Pattern PATTERN = correlatedJoin(); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(CorrelatedJoinNode correlatedJoinNode, Captures captures, Context context) { + Set subquerySymbols = + extractUnique(correlatedJoinNode.getSubquery(), context.getLookup()); + List newCorrelation = + correlatedJoinNode.getCorrelation().stream() + .filter(subquerySymbols::contains) + .collect(toImmutableList()); + + if (newCorrelation.size() < correlatedJoinNode.getCorrelation().size()) { + return Result.ofPlanNode( + new CorrelatedJoinNode( + correlatedJoinNode.getPlanNodeId(), + correlatedJoinNode.getInput(), + correlatedJoinNode.getSubquery(), + newCorrelation, + correlatedJoinNode.getJoinType(), + correlatedJoinNode.getFilter(), + correlatedJoinNode.getOriginSubquery())); + } + + return Result.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneEnforceSingleRowColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneEnforceSingleRowColumns.java new file mode 100644 index 000000000000..17125ef9a802 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneEnforceSingleRowColumns.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; + +import java.util.Optional; +import java.util.Set; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.Util.restrictChildOutputs; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.enforceSingleRow; + +public class PruneEnforceSingleRowColumns extends ProjectOffPushDownRule { + public PruneEnforceSingleRowColumns() { + super(enforceSingleRow()); + } + + @Override + protected Optional pushDownProjectOff( + Context context, EnforceSingleRowNode enforceSingleRowNode, Set referencedOutputs) { + return restrictChildOutputs(context.getIdAllocator(), enforceSingleRowNode, referencedOutputs); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveRedundantEnforceSingleRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveRedundantEnforceSingleRowNode.java new file mode 100644 index 000000000000..4546dd4efc2c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveRedundantEnforceSingleRowNode.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.enforceSingleRow; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isScalar; + +public class RemoveRedundantEnforceSingleRowNode implements Rule { + private static final Pattern PATTERN = enforceSingleRow(); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(EnforceSingleRowNode node, Captures captures, Context context) { + if (isScalar(node.getSource(), context.getLookup())) { + return Result.ofPlanNode(node.getSource()); + } + return Result.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedSubqueryToJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedSubqueryToJoin.java new file mode 100644 index 000000000000..15369b37778a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedSubqueryToJoin.java @@ -0,0 +1,133 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Assignments; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Cast; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.apache.tsfile.read.common.type.Type; + +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.CorrelatedJoin.correlation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.correlatedJoin; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.extractCardinality; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toSqlType; +import static org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern.empty; + +public class TransformUncorrelatedSubqueryToJoin implements Rule { + private static final Pattern PATTERN = + correlatedJoin().with(empty(correlation())); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(CorrelatedJoinNode correlatedJoinNode, Captures captures, Context context) { + // handle INNER and LEFT correlated join + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.INNER + || correlatedJoinNode.getJoinType() == JoinNode.JoinType.LEFT) { + return Result.ofPlanNode( + rewriteToJoin( + correlatedJoinNode, + correlatedJoinNode.getJoinType(), + correlatedJoinNode.getFilter(), + context.getLookup())); + } + + checkState( + correlatedJoinNode.getJoinType() == JoinNode.JoinType.RIGHT + || correlatedJoinNode.getJoinType() == JoinNode.JoinType.FULL, + "unexpected CorrelatedJoin type: " + correlatedJoinNode.getType()); + + // handle RIGHT and FULL correlated join ON TRUE + JoinNode.JoinType type; + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.RIGHT) { + type = JoinNode.JoinType.INNER; + } else { + type = JoinNode.JoinType.LEFT; + } + JoinNode joinNode = rewriteToJoin(correlatedJoinNode, type, TRUE_LITERAL, context.getLookup()); + + if (correlatedJoinNode.getFilter().equals(TRUE_LITERAL)) { + return Result.ofPlanNode(joinNode); + } + + // handle RIGHT correlated join on condition other than TRUE + if (correlatedJoinNode.getJoinType() == JoinNode.JoinType.RIGHT) { + Assignments.Builder assignments = Assignments.builder(); + assignments.putIdentities( + Sets.intersection( + ImmutableSet.copyOf(correlatedJoinNode.getSubquery().getOutputSymbols()), + ImmutableSet.copyOf(correlatedJoinNode.getOutputSymbols()))); + for (Symbol inputSymbol : + Sets.intersection( + ImmutableSet.copyOf(correlatedJoinNode.getInput().getOutputSymbols()), + ImmutableSet.copyOf(correlatedJoinNode.getOutputSymbols()))) { + Type inputType = context.getSymbolAllocator().getTypes().getTableModelType(inputSymbol); + assignments.put( + inputSymbol, + new IfExpression( + correlatedJoinNode.getFilter(), + inputSymbol.toSymbolReference(), + new Cast(new NullLiteral(), toSqlType(inputType)))); + } + ProjectNode projectNode = + new ProjectNode(context.getIdAllocator().genPlanNodeId(), joinNode, assignments.build()); + + return Result.ofPlanNode(projectNode); + } + + // no support for FULL correlated join on condition other than TRUE + return Result.empty(); + } + + private JoinNode rewriteToJoin( + CorrelatedJoinNode parent, JoinNode.JoinType type, Expression filter, Lookup lookup) { + if (type == JoinNode.JoinType.LEFT + && extractCardinality(parent.getSubquery(), lookup).isAtLeastScalar() + && filter.equals(TRUE_LITERAL)) { + // input rows will always be matched against subquery rows + type = JoinNode.JoinType.INNER; + } + return new JoinNode( + parent.getPlanNodeId(), + type, + parent.getInput(), + parent.getSubquery(), + ImmutableList.of(), + parent.getInput().getOutputSymbols(), + parent.getSubquery().getOutputSymbols(), + filter.equals(TRUE_LITERAL) ? Optional.empty() : Optional.of(filter), + Optional.empty()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ApplyNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ApplyNode.java new file mode 100644 index 000000000000..5a9c5817eaef --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ApplyNode.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.TwoChildProcessNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; + +import com.google.common.collect.ImmutableList; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class ApplyNode extends TwoChildProcessNode { + + /** Correlation symbols, returned from input (outer plan) used in subquery (inner plan) */ + private final List correlation; + + /** + * Expressions that use subquery symbols. + * + *

Subquery expressions are different than other expressions in a sense that they might use an + * entire subquery result as an input (e.g: "x IN (subquery)", "x < ALL (subquery)"). Such + * expressions are invalid in linear operator context (e.g: ProjectNode) in logical plan, but are + * correct in ApplyNode context. + * + *

Example 1: - expression: input_symbol_X IN (subquery_symbol_Y) - meaning: if set consisting + * of all values for subquery_symbol_Y contains value represented by input_symbol_X + * + *

Example 2: - expression: input_symbol_X < ALL (subquery_symbol_Y) - meaning: if + * input_symbol_X is smaller than all subquery values represented by subquery_symbol_Y + * + *

+ */ + private final Map subqueryAssignments; + + /** HACK! Used for error reporting in case this ApplyNode is not supported */ + private final Node originSubquery; + + public ApplyNode( + PlanNodeId id, + // will be set as leftChild + PlanNode input, + // will be set as rightChild + PlanNode subquery, + Map subqueryAssignments, + List correlation, + Node originSubquery) { + super(id, input, subquery); + requireNonNull(subqueryAssignments, "subqueryAssignments is null"); + requireNonNull(correlation, "correlation is null"); + requireNonNull(originSubquery, "originSubquery is null"); + + if (input != null) { + checkArgument( + input.getOutputSymbols().containsAll(correlation), + "Input does not contain symbols from correlation"); + } + + this.subqueryAssignments = subqueryAssignments; + this.correlation = ImmutableList.copyOf(correlation); + this.originSubquery = originSubquery; + } + + public PlanNode getInput() { + return leftChild; + } + + public PlanNode getSubquery() { + return rightChild; + } + + public Map getSubqueryAssignments() { + return subqueryAssignments; + } + + public List getCorrelation() { + return correlation; + } + + public Node getOriginSubquery() { + return originSubquery; + } + + @Override + public List getOutputSymbols() { + return ImmutableList.builder() + .addAll(leftChild.getOutputSymbols()) + .addAll(subqueryAssignments.keySet()) + .build(); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitApply(this, context); + } + + @Override + public PlanNode replaceChildren(List newChildren) { + checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); + return new ApplyNode( + getPlanNodeId(), + newChildren.get(0), + newChildren.get(1), + subqueryAssignments, + correlation, + originSubquery); + } + + @Override + public PlanNode clone() { + // clone without children + return new ApplyNode( + getPlanNodeId(), null, null, subqueryAssignments, correlation, originSubquery); + } + + @Override + public List getOutputColumnNames() { + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + // ApplyNode should be transformed to other nodes after planning, so serialization is not + // expected. + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + // ApplyNode should be transformed to other nodes after planning, so serialization is not + // expected. + throw new UnsupportedOperationException(); + } + + public interface SetExpression { + List inputs(); + } + + public static class In implements SetExpression { + private final Symbol value; + private final Symbol reference; + + public In(Symbol value, Symbol reference) { + this.value = value; + this.reference = reference; + } + + public Symbol getValue() { + return value; + } + + public Symbol getReference() { + return reference; + } + + @Override + public List inputs() { + return ImmutableList.of(value, reference); + } + } + + public static class Exists implements SetExpression { + @Override + public List inputs() { + return ImmutableList.of(); + } + } + + public static class QuantifiedComparison implements SetExpression { + private final Operator operator; + private final Quantifier quantifier; + private final Symbol value; + private final Symbol reference; + + public QuantifiedComparison( + Operator operator, Quantifier quantifier, Symbol value, Symbol reference) { + this.operator = operator; + this.quantifier = quantifier; + this.value = value; + this.reference = reference; + } + + public Operator getOperator() { + return operator; + } + + public Quantifier getQuantifier() { + return quantifier; + } + + public Symbol getValue() { + return value; + } + + public Symbol getReference() { + return reference; + } + + @Override + public List inputs() { + return ImmutableList.of(value, reference); + } + } + + public enum Operator { + EQUAL, + NOT_EQUAL, + LESS_THAN, + LESS_THAN_OR_EQUAL, + GREATER_THAN, + GREATER_THAN_OR_EQUAL, + } + + public enum Quantifier { + ALL, + ANY, + SOME, + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CorrelatedJoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CorrelatedJoinNode.java new file mode 100644 index 000000000000..bbee7e5d68fe --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CorrelatedJoinNode.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.TwoChildProcessNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; + +import com.google.common.collect.ImmutableList; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +/** + * For every row from {@link #leftChild}(input) a {@link #rightChild}(subquery) relation is + * calculated. Then input row is cross joined with subquery relation and returned as a result. + * + *

INNER - does not return any row for input row when subquery relation is empty LEFT - does + * return input completed with NULL values when subquery relation is empty + */ +public class CorrelatedJoinNode extends TwoChildProcessNode { + + /** Correlation symbols, returned from input (outer plan) used in subquery (inner plan) */ + private final List correlation; + + private final JoinNode.JoinType type; + private final Expression filter; + + /** HACK! Used for error reporting in case this ApplyNode is not supported */ + private final Node originSubquery; + + public CorrelatedJoinNode( + PlanNodeId id, + PlanNode input, + PlanNode subquery, + List correlation, + JoinNode.JoinType type, + Expression filter, + Node originSubquery) { + super(id, input, subquery); + requireNonNull(correlation, "correlation is null"); + requireNonNull(filter, "filter is null"); + // The condition doesn't guarantee that filter is of type boolean, but was found to be a + // practical way to identify + // places where CorrelatedJoinNode could be created without appropriate coercions. + checkArgument( + !(filter instanceof NullLiteral), + "Filter must be an expression of boolean type: %s", + filter); + requireNonNull(originSubquery, "originSubquery is null"); + + if (input != null) { + checkArgument( + input.getOutputSymbols().containsAll(correlation), + "Input does not contain symbols from correlation"); + } + + this.correlation = ImmutableList.copyOf(correlation); + this.type = type; + this.filter = filter; + this.originSubquery = originSubquery; + } + + public PlanNode getInput() { + return leftChild; + } + + public PlanNode getSubquery() { + return rightChild; + } + + public List getCorrelation() { + return correlation; + } + + public JoinNode.JoinType getJoinType() { + return type; + } + + public Expression getFilter() { + return filter; + } + + public Node getOriginSubquery() { + return originSubquery; + } + + public List getSources() { + return ImmutableList.of(leftChild, rightChild); + } + + @Override + public List getOutputSymbols() { + return ImmutableList.builder() + .addAll(leftChild.getOutputSymbols()) + .addAll(rightChild.getOutputSymbols()) + .build(); + } + + @Override + public PlanNode replaceChildren(List newChildren) { + checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); + return new CorrelatedJoinNode( + getPlanNodeId(), + newChildren.get(0), + newChildren.get(1), + correlation, + type, + filter, + originSubquery); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCorrelatedJoin(this, context); + } + + @Override + public PlanNode clone() { + // clone without children + return new CorrelatedJoinNode( + getPlanNodeId(), null, null, correlation, type, filter, originSubquery); + } + + @Override + public List getOutputColumnNames() { + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + // CorrelatedJoinNode should be transformed to other nodes after planning, so serialization is + // not expected. + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + // CorrelatedJoinNode should be transformed to other nodes after planning, so serialization is + // not expected. + throw new UnsupportedOperationException(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/EnforceSingleRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/EnforceSingleRowNode.java new file mode 100644 index 000000000000..734f6c1a2323 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/EnforceSingleRowNode.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; + +import com.google.common.collect.Iterables; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +public class EnforceSingleRowNode extends SingleChildProcessNode { + + public EnforceSingleRowNode(PlanNodeId id, PlanNode source) { + super(id, source); + } + + @Override + public List getOutputSymbols() { + return child.getOutputSymbols(); + } + + public PlanNode getSource() { + return child; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitEnforceSingleRow(this, context); + } + + @Override + public PlanNode replaceChildren(List newChildren) { + return new EnforceSingleRowNode(getPlanNodeId(), Iterables.getOnlyElement(newChildren)); + } + + @Override + public PlanNode clone() { + // clone without children + return new EnforceSingleRowNode(getPlanNodeId(), null); + } + + @Override + public List getOutputColumnNames() { + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.TABLE_ENFORCE_SINGLE_ROW_NODE.serialize(byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.TABLE_ENFORCE_SINGLE_ROW_NODE.serialize(stream); + } + + public static EnforceSingleRowNode deserialize(ByteBuffer byteBuffer) { + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); + return new EnforceSingleRowNode(planNodeId, null); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java index 9cea03e14043..aea9fb529f87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java @@ -14,7 +14,9 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Property; @@ -34,22 +36,21 @@ public static Pattern aggregation() { return typeOf(AggregationNode.class); } - /*public static Pattern assignUniqueId() - { - return typeOf(AssignUniqueId.class); - } - - public static Pattern groupId() - { - return typeOf(GroupIdNode.class); - } + // public static Pattern assignUniqueId() + // { + // return typeOf(AssignUniqueId.class); + // } + // + // public static Pattern groupId() + // { + // return typeOf(GroupIdNode.class); + // } - public static Pattern applyNode() - { - return typeOf(ApplyNode.class); + public static Pattern applyNode() { + return typeOf(ApplyNode.class); } - public static Pattern tableExecute() + /* public static Pattern tableExecute() { return typeOf(TableExecuteNode.class); } @@ -72,13 +73,12 @@ public static Pattern exchange() public static Pattern explainAnalyze() { return typeOf(ExplainAnalyzeNode.class); - } - - public static Pattern enforceSingleRow() - { - return typeOf(EnforceSingleRowNode.class); }*/ + public static Pattern enforceSingleRow() { + return typeOf(EnforceSingleRowNode.class); + } + public static Pattern filter() { return typeOf(FilterNode.class); } @@ -105,13 +105,12 @@ public static Pattern dynamicFilterSource() public static Pattern spatialJoin() { return typeOf(SpatialJoinNode.class); - } - - public static Pattern correlatedJoin() - { - return typeOf(CorrelatedJoinNode.class); }*/ + public static Pattern correlatedJoin() { + return typeOf(CorrelatedJoinNode.class); + } + public static Pattern offset() { return typeOf(OffsetNode.class); } @@ -136,13 +135,13 @@ public static Pattern project() { /*public static Pattern sample() { return typeOf(SampleNode.class); - } - - public static Pattern semiJoin() - { - return typeOf(SemiJoinNode.class); }*/ + // public static Pattern semiJoin() + // { + // return typeOf(SemiJoinNode.class); + // } + public static Pattern gapFill() { return typeOf(GapFillNode.class); } @@ -279,72 +278,63 @@ public static Property step() { return property("step", AggregationNode::getStep); } - } + }*/ - public static final class Apply - { - public static Property> correlation() - { - return property("correlation", ApplyNode::getCorrelation); - } + public static final class Apply { + public static Property> correlation() { + return property("correlation", ApplyNode::getCorrelation); + } } - public static final class DistinctLimit + /*public static final class DistinctLimit { public static Property isPartial() { return property("isPartial", DistinctLimitNode::isPartial); } - } + }*/ - public static final class Exchange + /*public static final class Exchange { public static Property scope() { return property("scope", ExchangeNode::getScope); } - } + }*/ - public static final class Join - { - public static Property type() - { - return property("type", JoinNode::getType); - } + public static final class Join { + public static Property type() { + return property("type", JoinNode::getJoinType); + } - public static Property left() - { - return property("left", (JoinNode joinNode, Lookup lookup) -> lookup.resolve(joinNode.getLeft())); - } + public static Property left() { + return property( + "left", (JoinNode joinNode, Lookup lookup) -> lookup.resolve(joinNode.getLeftChild())); + } - public static Property right() - { - return property("right", (JoinNode joinNode, Lookup lookup) -> lookup.resolve(joinNode.getRight())); - } + public static Property right() { + return property( + "right", (JoinNode joinNode, Lookup lookup) -> lookup.resolve(joinNode.getRightChild())); + } } - public static final class CorrelatedJoin - { - public static Property> correlation() - { - return property("correlation", CorrelatedJoinNode::getCorrelation); - } + public static final class CorrelatedJoin { + public static Property> correlation() { + return property("correlation", CorrelatedJoinNode::getCorrelation); + } - public static Property subquery() - { - return property("subquery", (node, context) -> context.resolve(node.getSubquery())); - } + public static Property subquery() { + return property("subquery", (node, context) -> context.resolve(node.getSubquery())); + } - public static Property filter() - { - return property("filter", CorrelatedJoinNode::getFilter); - } + public static Property filter() { + return property("filter", CorrelatedJoinNode::getFilter); + } - public static Property type() - { - return property("type", CorrelatedJoinNode::getType); - } - }*/ + public static Property type() { + return property("type", CorrelatedJoinNode::getJoinType); + } + } public static final class Limit { public static Property count() { @@ -393,9 +383,9 @@ public static Property rowCount() { return property("rowCount", ValuesNode::getRowCount); } - } + }*/ - public static final class SemiJoin + /*public static final class SemiJoin { public static Property getSource() { @@ -410,9 +400,9 @@ public static Property getFilteringSource() "filteringSource", (SemiJoinNode semiJoin, Lookup lookup) -> lookup.resolve(semiJoin.getFilteringSource())); } - } + }*/ - public static final class Intersect + /*public static final class Intersect { public static Property distinct() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Cardinality.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Cardinality.java new file mode 100644 index 000000000000..2db8c0849d60 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Cardinality.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; + +import com.google.common.collect.Range; + +import static java.util.Objects.requireNonNull; + +public class Cardinality { + private final Range cardinalityRange; + + public Cardinality(Range cardinalityRange) { + this.cardinalityRange = requireNonNull(cardinalityRange, "cardinalityRange is null"); + } + + public boolean isEmpty() { + return isAtMost(0); + } + + public boolean isScalar() { + return Range.singleton(1L).encloses(cardinalityRange); + } + + public boolean isAtLeastScalar() { + return isAtLeast(1L); + } + + public boolean isAtMostScalar() { + return isAtMost(1L); + } + + public boolean isAtLeast(long minCardinality) { + return Range.atLeast(minCardinality).encloses(cardinalityRange); + } + + public boolean isAtMost(long maxCardinality) { + return Range.closed(0L, maxCardinality).encloses(cardinalityRange); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CheckSubqueryNodesAreRewritten.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CheckSubqueryNodesAreRewritten.java new file mode 100644 index 000000000000..eab54f6251ca --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CheckSubqueryNodesAreRewritten.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; + +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; + +import java.util.List; + +import static com.google.common.base.Preconditions.checkState; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.PlanNodeSearcher.searchFrom; +import static org.apache.iotdb.rpc.TSStatusCode.SEMANTIC_ERROR; + +public class CheckSubqueryNodesAreRewritten implements PlanOptimizer { + @Override + public PlanNode optimize(PlanNode plan, Context context) { + searchFrom(plan) + .where(ApplyNode.class::isInstance) + .findFirst() + .ifPresent( + node -> { + ApplyNode applyNode = (ApplyNode) node; + throw error(applyNode.getCorrelation()); + }); + + searchFrom(plan) + .where(CorrelatedJoinNode.class::isInstance) + .findFirst() + .ifPresent( + node -> { + CorrelatedJoinNode correlatedJoinNode = (CorrelatedJoinNode) node; + throw error(correlatedJoinNode.getCorrelation()); + }); + + return plan; + } + + private SemanticException error(List correlation) { + checkState( + !correlation.isEmpty(), + "All the non correlated subqueries should be rewritten at this point"); + throw new SemanticException( + "Given correlated subquery is not supported", SEMANTIC_ERROR.getStatusCode()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index eb7837def996..304917842877 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -32,6 +32,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.MergeLimits; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneAggregationColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneAggregationSourceColumns; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneCorrelatedJoinColumns; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneCorrelatedJoinCorrelation; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneEnforceSingleRowColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneFillColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneFilterColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneGapFillColumns; @@ -47,9 +50,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PushLimitThroughOffset; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PushLimitThroughProject; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveDuplicateConditions; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveRedundantEnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveRedundantIdentityProjections; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveTrivialFilters; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.SimplifyExpressions; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedSubqueryToJoin; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -72,6 +77,9 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { // TODO After ValuesNode introduced // new RemoveEmptyGlobalAggregation(), new PruneAggregationSourceColumns(), + new PruneCorrelatedJoinColumns(), + new PruneCorrelatedJoinCorrelation(), + new PruneEnforceSingleRowColumns(), new PruneFilterColumns(), new PruneGapFillColumns(), new PruneFillColumns(), @@ -146,7 +154,7 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new InlineProjections(plannerContext), new RemoveRedundantIdentityProjections(), new MergeLimits(), - new RemoveTrivialFilters() + new RemoveTrivialFilters(), // new RemoveRedundantLimit(), // new RemoveRedundantOffset(), // new RemoveRedundantSort(), @@ -156,6 +164,7 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { // new ReplaceRedundantJoinWithSource(), // new RemoveRedundantJoin(), // new ReplaceRedundantJoinWithProject(), + new RemoveRedundantEnforceSingleRowNode() // new RemoveRedundantExists(), // new RemoveRedundantWindow(), // new SingleDistinctAggregationToGroupBy(), @@ -190,6 +199,14 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new UnaliasSymbolReferences(plannerContext.getMetadata()), columnPruningOptimizer, inlineProjectionLimitFiltersOptimizer, + new IterativeOptimizer( + plannerContext, + ruleStats, + ImmutableSet.of( + new RemoveRedundantEnforceSingleRowNode(), + new TransformUncorrelatedSubqueryToJoin())), + new CheckSubqueryNodesAreRewritten(), + simplifyOptimizer, new PushPredicateIntoTableScan(), // redo columnPrune and inlineProjections after pushPredicateIntoTableScan columnPruningOptimizer, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index d388f7f19ca1..b3d0e3b88728 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -20,11 +20,11 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet; @@ -551,10 +551,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { List equiJoinClauses = new ArrayList<>(); ImmutableList.Builder joinFilterBuilder = ImmutableList.builder(); for (Expression conjunct : extractConjuncts(newJoinPredicate)) { - if (joinEqualityExpression( - conjunct, - node.getLeftChild().getOutputSymbols(), - node.getRightChild().getOutputSymbols())) { + if (joinEqualityExpressionOnTimeColumn(conjunct, node)) { ComparisonExpression equality = (ComparisonExpression) conjunct; boolean alignedComparison = @@ -608,12 +605,11 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { } if (node.getJoinType() == INNER && newJoinFilter.isPresent() && equiJoinClauses.isEmpty()) { - throw new IllegalStateException("INNER JOIN only support equiJoinClauses"); // if we do not have any equi conjunct we do not pushdown non-equality condition into // inner join, so we plan execution as nested-loops-join followed by filter instead // hash join. - // postJoinPredicate = combineConjuncts(postJoinPredicate, newJoinFilter.get()); - // newJoinFilter = Optional.empty(); + postJoinPredicate = combineConjuncts(postJoinPredicate, newJoinFilter.get()); + newJoinFilter = Optional.empty(); } boolean filtersEquivalent = @@ -646,37 +642,35 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { node.isSpillable()); } - if (((JoinNode) output).isCrossJoin()) { - throw new SemanticException( - "Cross join is not supported in current version, each table must have at least one equiJoinClause"); + // sort the left and right child of join node if it is not a cross join + if (!((JoinNode) output).isCrossJoin()) { + JoinNode.EquiJoinClause joinCriteria = ((JoinNode) output).getCriteria().get(0); + OrderingScheme leftOrderingScheme = + new OrderingScheme( + Collections.singletonList(joinCriteria.getLeft()), + Collections.singletonMap(joinCriteria.getLeft(), ASC_NULLS_LAST)); + OrderingScheme rightOrderingScheme = + new OrderingScheme( + Collections.singletonList(joinCriteria.getRight()), + Collections.singletonMap(joinCriteria.getRight(), ASC_NULLS_LAST)); + SortNode leftSortNode = + new SortNode( + queryId.genPlanNodeId(), + ((JoinNode) output).getLeftChild(), + leftOrderingScheme, + false, + false); + SortNode rightSortNode = + new SortNode( + queryId.genPlanNodeId(), + ((JoinNode) output).getRightChild(), + rightOrderingScheme, + false, + false); + ((JoinNode) output).setLeftChild(leftSortNode); + ((JoinNode) output).setRightChild(rightSortNode); } - JoinNode.EquiJoinClause joinCriteria = ((JoinNode) output).getCriteria().get(0); - OrderingScheme leftOrderingScheme = - new OrderingScheme( - Collections.singletonList(joinCriteria.getLeft()), - Collections.singletonMap(joinCriteria.getLeft(), ASC_NULLS_LAST)); - OrderingScheme rightOrderingScheme = - new OrderingScheme( - Collections.singletonList(joinCriteria.getRight()), - Collections.singletonMap(joinCriteria.getRight(), ASC_NULLS_LAST)); - SortNode leftSortNode = - new SortNode( - queryId.genPlanNodeId(), - ((JoinNode) output).getLeftChild(), - leftOrderingScheme, - false, - false); - SortNode rightSortNode = - new SortNode( - queryId.genPlanNodeId(), - ((JoinNode) output).getRightChild(), - rightOrderingScheme, - false, - false); - ((JoinNode) output).setLeftChild(leftSortNode); - ((JoinNode) output).setRightChild(rightSortNode); - if (!postJoinPredicate.equals(TRUE_LITERAL)) { output = new FilterNode(queryContext.getQueryId().genPlanNodeId(), output, postJoinPredicate); @@ -693,6 +687,34 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { return output; } + private boolean joinEqualityExpressionOnTimeColumn(Expression conjunct, JoinNode node) { + if (!joinEqualityExpression( + conjunct, + node.getLeftChild().getOutputSymbols(), + node.getRightChild().getOutputSymbols())) { + return false; + } + // conjunct must be a comparison expression + ComparisonExpression equality = (ComparisonExpression) conjunct; + // After Optimization, some subqueries are transformed into Join. + // For now, Users can only use join on time. And the join is implemented using MergeSortJoin. + // However, it's assumed that need to use Filter + NestedLoopJoin is better than MergeSortJoin + // for scalar subquery, since sorting the left table is not necessary. + // So, we want to find out whether the join is on time column. If it is not on time column, we + // will use Filter + NestedLoopJoin instead. + // Attention: For now, join on time column is assumed to hold the following condition: left + // and right both contains the substring time. + // todo: after supporting join on other columns for the user, we need to remove the following + // code, since the condition does not hold anymore. + // This is temporary workaround. + Expression left = equality.getLeft(); + Expression right = equality.getRight(); + return (left instanceof SymbolReference + && ((SymbolReference) left).getName().contains(IoTDBConstant.TIME)) + && (right instanceof SymbolReference + && ((SymbolReference) right).getName().contains(IoTDBConstant.TIME)); + } + private Symbol symbolForExpression(Expression expression) { if (expression instanceof SymbolReference) { return Symbol.from(expression); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java new file mode 100644 index 000000000000..d92764c6a3d7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ExchangeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; + +import com.google.common.collect.Range; + +import java.util.List; +import java.util.stream.Collectors; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup.noLookup; + +public final class QueryCardinalityUtil { + private QueryCardinalityUtil() {} + + public static boolean isScalar(PlanNode node) { + return isScalar(node, noLookup()); + } + + public static boolean isScalar(PlanNode node, Lookup lookup) { + return extractCardinality(node, lookup).isScalar(); + } + + public static boolean isAtMostScalar(PlanNode node) { + return isAtMostScalar(node, noLookup()); + } + + public static boolean isAtMostScalar(PlanNode node, Lookup lookup) { + return isAtMost(node, lookup, 1L); + } + + public static boolean isAtMost(PlanNode node, Lookup lookup, long maxCardinality) { + return extractCardinality(node, lookup).isAtMost(maxCardinality); + } + + public static boolean isAtLeastScalar(PlanNode node, Lookup lookup) { + return isAtLeast(node, lookup, 1L); + } + + public static boolean isAtLeast(PlanNode node, Lookup lookup, long minCardinality) { + return extractCardinality(node, lookup).isAtLeast(minCardinality); + } + + public static boolean isEmpty(PlanNode node, Lookup lookup) { + return isAtMost(node, lookup, 0); + } + + public static Cardinality extractCardinality(PlanNode node) { + return extractCardinality(node, noLookup()); + } + + public static Cardinality extractCardinality(PlanNode node, Lookup lookup) { + return new Cardinality(node.accept(new CardinalityExtractorPlanVisitor(lookup), null)); + } + + private static final class CardinalityExtractorPlanVisitor + extends PlanVisitor, Void> { + private final Lookup lookup; + + public CardinalityExtractorPlanVisitor(Lookup lookup) { + this.lookup = requireNonNull(lookup, "lookup is null"); + } + + @Override + public Range visitPlan(PlanNode node, Void context) { + return Range.atLeast(0L); + } + + @Override + public Range visitGroupReference(GroupReference node, Void context) { + return lookup.resolve(node).accept(this, context); + } + + @Override + public Range visitEnforceSingleRow(EnforceSingleRowNode node, Void context) { + return Range.singleton(1L); + } + + @Override + public Range visitAggregation(AggregationNode node, Void context) { + if (node.hasSingleGlobalAggregation()) { + // only single default aggregation which will produce exactly single row + return Range.singleton(1L); + } + + Range sourceCardinalityRange = node.getChild().accept(this, null); + + long lower; + if (node.hasDefaultOutput() || sourceCardinalityRange.lowerEndpoint() > 0) { + lower = 1; + } else { + lower = 0; + } + + if (sourceCardinalityRange.hasUpperBound()) { + long upper = Math.max(lower, sourceCardinalityRange.upperEndpoint()); + return Range.closed(lower, upper); + } + + return Range.atLeast(lower); + } + + @Override + public Range visitExchange(ExchangeNode node, Void context) { + if (node.getChildren().size() == 1) { + return getOnlyElement(node.getChildren()).accept(this, null); + } + return Range.atLeast(0L); + } + + @Override + public Range visitProject(ProjectNode node, Void context) { + return node.getChild().accept(this, null); + } + + @Override + public Range visitFilter(FilterNode node, Void context) { + Range sourceCardinalityRange = node.getChild().accept(this, null); + if (sourceCardinalityRange.hasUpperBound()) { + return Range.closed(0L, sourceCardinalityRange.upperEndpoint()); + } + return Range.atLeast(0L); + } + + // @Override + // public Range visitValues(ValuesNode node, Void context) + // { + // return Range.singleton((long) node.getRowCount()); + // } + + @Override + public Range visitOffset(OffsetNode node, Void context) { + Range sourceCardinalityRange = node.getChild().accept(this, null); + + long lower = max(sourceCardinalityRange.lowerEndpoint() - node.getCount(), 0L); + + if (sourceCardinalityRange.hasUpperBound()) { + return Range.closed( + lower, max(sourceCardinalityRange.upperEndpoint() - node.getCount(), 0L)); + } + return Range.atLeast(lower); + } + + @Override + public Range visitLimit(LimitNode node, Void context) { + if (node.isWithTies()) { + Range sourceCardinalityRange = node.getChild().accept(this, null); + long lower = min(node.getCount(), sourceCardinalityRange.lowerEndpoint()); + if (sourceCardinalityRange.hasUpperBound()) { + return Range.closed(lower, sourceCardinalityRange.upperEndpoint()); + } + return Range.atLeast(lower); + } + + return applyLimit(node.getChild(), node.getCount()); + } + + @Override + public Range visitTopK(TopKNode node, Void context) { + long limit = node.getCount(); + List> rangeList = + node.getChildren().stream() + .map(child -> applyLimit(child, limit)) + .collect(Collectors.toList()); + // merge rangeList + long lower = rangeList.stream().mapToLong(Range::lowerEndpoint).sum(); + long upper = rangeList.stream().mapToLong(Range::upperEndpoint).sum(); + return Range.closed(Math.min(lower, limit), Math.min(upper, limit)); + } + + // @Override + // public Range visitWindow(WindowNode node, Void context) + // { + // return node.getSource().accept(this, null); + // } + + private Range applyLimit(PlanNode source, long limit) { + Range sourceCardinalityRange = source.accept(this, null); + if (sourceCardinalityRange.hasUpperBound()) { + limit = min(sourceCardinalityRange.upperEndpoint(), limit); + } + long lower = min(limit, sourceCardinalityRange.lowerEndpoint()); + return Range.closed(lower, limit); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java index 1ec6499ff22b..623a9873d908 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.ir.ExpressionRewriter; import org.apache.iotdb.db.queryengine.plan.relational.planner.ir.ExpressionTreeRewriter; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@ -84,18 +85,23 @@ public Symbol map(Symbol symbol) { return mappingFunction.apply(symbol); } - /*public ApplyNode.SetExpression map(ApplyNode.SetExpression expression) - { - return switch (expression) { - case ApplyNode.Exists exists -> exists; - case ApplyNode.In in -> new ApplyNode.In(map(in.value()), map(in.reference())); - case ApplyNode.QuantifiedComparison comparison -> new ApplyNode.QuantifiedComparison( - comparison.operator(), - comparison.quantifier(), - map(comparison.value()), - map(comparison.reference())); - }; - }*/ + public ApplyNode.SetExpression map(ApplyNode.SetExpression expression) { + if (expression instanceof ApplyNode.Exists) { + return expression; + } else if (expression instanceof ApplyNode.In) { + ApplyNode.In in = (ApplyNode.In) expression; + return new ApplyNode.In(map(in.getValue()), map(in.getReference())); + } else if (expression instanceof ApplyNode.QuantifiedComparison) { + ApplyNode.QuantifiedComparison comparison = (ApplyNode.QuantifiedComparison) expression; + return new ApplyNode.QuantifiedComparison( + comparison.getOperator(), + comparison.getQuantifier(), + map(comparison.getValue()), + map(comparison.getReference())); + } else { + throw new IllegalArgumentException("Unexpected value: " + expression); + } + } public List map(List symbols) { return symbols.stream().map(this::map).collect(toImmutableList()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index b79f9465319d..aedd97bff6dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -24,6 +24,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolAllocator; import org.apache.iotdb.db.queryengine.plan.relational.planner.ir.DeterminismEvaluator; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; @@ -463,6 +466,136 @@ public PlanAndMappings visitOutput(OutputNode node, UnaliasContext context) { mapping); } + @Override + public PlanAndMappings visitEnforceSingleRow( + EnforceSingleRowNode node, UnaliasContext context) { + PlanAndMappings rewrittenSource = node.getSource().accept(this, context); + + return new PlanAndMappings( + node.replaceChildren(ImmutableList.of(rewrittenSource.getRoot())), + rewrittenSource.getMappings()); + } + + @Override + public PlanAndMappings visitApply(ApplyNode node, UnaliasContext context) { + // it is assumed that apart from correlation (and possibly outer correlation), symbols are + // distinct between Input and Subquery + // rewrite Input + PlanAndMappings rewrittenInput = node.getInput().accept(this, context); + Map inputMapping = new HashMap<>(rewrittenInput.getMappings()); + SymbolMapper mapper = symbolMapper(inputMapping); + + // rewrite correlation with mapping from Input + List rewrittenCorrelation = mapper.mapAndDistinct(node.getCorrelation()); + + // extract new mappings for correlation symbols to apply in Subquery + Set correlationSymbols = ImmutableSet.copyOf(node.getCorrelation()); + Map correlationMapping = new HashMap<>(); + for (Map.Entry entry : inputMapping.entrySet()) { + if (correlationSymbols.contains(entry.getKey())) { + correlationMapping.put(entry.getKey(), mapper.map(entry.getKey())); + } + } + + Map mappingForSubquery = new HashMap<>(); + mappingForSubquery.putAll(context.getCorrelationMapping()); + mappingForSubquery.putAll(correlationMapping); + + // rewrite Subquery + PlanAndMappings rewrittenSubquery = + node.getSubquery().accept(this, new UnaliasContext(mappingForSubquery)); + + // unify mappings from Input and Subquery to rewrite Subquery assignments + Map resultMapping = new HashMap<>(); + resultMapping.putAll(rewrittenInput.getMappings()); + resultMapping.putAll(rewrittenSubquery.getMappings()); + mapper = symbolMapper(resultMapping); + + ImmutableList.Builder> rewrittenAssignments = + ImmutableList.builder(); + for (Map.Entry assignment : + node.getSubqueryAssignments().entrySet()) { + rewrittenAssignments.add( + new SimpleEntry<>(mapper.map(assignment.getKey()), mapper.map(assignment.getValue()))); + } + + // deduplicate assignments + Map deduplicateAssignments = + rewrittenAssignments.build().stream() + .distinct() + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + + mapper = symbolMapper(resultMapping); + + // build new Assignments with canonical outputs + // duplicate entries will be removed by the Builder + ImmutableMap.Builder newAssignments = ImmutableMap.builder(); + for (Map.Entry assignment : + deduplicateAssignments.entrySet()) { + newAssignments.put(mapper.map(assignment.getKey()), assignment.getValue()); + } + + return new PlanAndMappings( + new ApplyNode( + node.getPlanNodeId(), + rewrittenInput.getRoot(), + rewrittenSubquery.getRoot(), + newAssignments.buildOrThrow(), + rewrittenCorrelation, + node.getOriginSubquery()), + resultMapping); + } + + @Override + public PlanAndMappings visitCorrelatedJoin(CorrelatedJoinNode node, UnaliasContext context) { + // it is assumed that apart from correlation (and possibly outer correlation), symbols are + // distinct between left and right CorrelatedJoin source + // rewrite Input + PlanAndMappings rewrittenInput = node.getInput().accept(this, context); + Map inputMapping = new HashMap<>(rewrittenInput.getMappings()); + SymbolMapper mapper = symbolMapper(inputMapping); + + // rewrite correlation with mapping from Input + List rewrittenCorrelation = mapper.mapAndDistinct(node.getCorrelation()); + + // extract new mappings for correlation symbols to apply in Subquery + Set correlationSymbols = ImmutableSet.copyOf(node.getCorrelation()); + Map correlationMapping = new HashMap<>(); + for (Map.Entry entry : inputMapping.entrySet()) { + if (correlationSymbols.contains(entry.getKey())) { + correlationMapping.put(entry.getKey(), mapper.map(entry.getKey())); + } + } + + Map mappingForSubquery = new HashMap<>(); + mappingForSubquery.putAll(context.getCorrelationMapping()); + mappingForSubquery.putAll(correlationMapping); + + // rewrite Subquery + PlanAndMappings rewrittenSubquery = + node.getSubquery().accept(this, new UnaliasContext(mappingForSubquery)); + + // unify mappings from Input and Subquery + Map resultMapping = new HashMap<>(); + resultMapping.putAll(rewrittenInput.getMappings()); + resultMapping.putAll(rewrittenSubquery.getMappings()); + + // rewrite filter with unified mapping + mapper = symbolMapper(resultMapping); + Expression newFilter = mapper.map(node.getFilter()); + + return new PlanAndMappings( + new CorrelatedJoinNode( + node.getPlanNodeId(), + rewrittenInput.getRoot(), + rewrittenSubquery.getRoot(), + rewrittenCorrelation, + node.getJoinType(), + newFilter, + node.getOriginSubquery()), + resultMapping); + } + @Override public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { // it is assumed that symbols are distinct between left and right join source. Only symbols diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SingleColumn.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SingleColumn.java index 9f4e3663e6f7..e4e3cf9f7455 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SingleColumn.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SingleColumn.java @@ -109,6 +109,10 @@ public boolean shallowEquals(Node other) { return false; } + if (alias == null) { + return ((SingleColumn) other).alias == null; + } + return alias.equals(((SingleColumn) other).alias); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java new file mode 100644 index 000000000000..652472b05918 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; +import org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.junit.Test; + +import java.util.Collections; +import java.util.Optional; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanAssert.assertPlan; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.aggregation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.aggregationFunction; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.aggregationTableScan; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.any; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.anyTree; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.collect; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.enforceSingleRow; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.exchange; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.filter; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.join; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.project; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.singleGroupingSet; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.tableScan; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.FINAL; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.INTERMEDIATE; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.PARTIAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.EQUAL; + +public class SubqueryTest { + @Test + public void testUncorrelatedScalarSubqueryInWhereClause() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT s1 FROM table1 where s1 = (select max(s1) from table1)"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + Expression filterPredicate = + new ComparisonExpression(EQUAL, new SymbolReference("s1"), new SymbolReference("max")); + + PlanMatchPattern tableScan = + tableScan("testdb.table1", ImmutableList.of("s1"), ImmutableSet.of("s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──FilterNode + * └──JoinNode + * |──TableScanNode + * ├──AggregationNode + * │ └──AggregationTableScanNode + + */ + assertPlan( + logicalQueryPlan, + output( + project( + filter( + filterPredicate, + join( + JoinNode.JoinType.INNER, + builder -> + builder + .left(tableScan) + .right( + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("max"), + aggregationFunction("max", ImmutableList.of("max_9"))), + Collections.emptyList(), + Optional.empty(), + FINAL, + aggregationTableScan( + singleGroupingSet(), + Collections.emptyList(), + Optional.empty(), + PARTIAL, + "testdb.table1", + ImmutableList.of("max_9"), + ImmutableSet.of("s1_6"))))))))); + + // Verify DistributionPlan + assertPlan( + planTester.getFragmentPlan(0), + output( + project( + filter( + filterPredicate, + join( + JoinNode.JoinType.INNER, + builder -> + builder + .left(collect(exchange(), tableScan, exchange())) + .right( + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("max"), + aggregationFunction("max", ImmutableList.of("max_10"))), + Collections.emptyList(), + Optional.empty(), + FINAL, + collect( + exchange(), + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("max_10"), + aggregationFunction( + "max", ImmutableList.of("max_9"))), + Collections.emptyList(), + Optional.empty(), + INTERMEDIATE, + aggregationTableScan( + singleGroupingSet(), + Collections.emptyList(), + Optional.empty(), + PARTIAL, + "testdb.table1", + ImmutableList.of("max_9"), + ImmutableSet.of("s1_6"))), + exchange())))))))); + + assertPlan(planTester.getFragmentPlan(1), tableScan); + + assertPlan(planTester.getFragmentPlan(2), tableScan); + + assertPlan( + planTester.getFragmentPlan(3), + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("max_10"), aggregationFunction("max", ImmutableList.of("max_9"))), + Collections.emptyList(), + Optional.empty(), + INTERMEDIATE, + aggregationTableScan( + singleGroupingSet(), + Collections.emptyList(), + Optional.empty(), + PARTIAL, + "testdb.table1", + ImmutableList.of("max_9"), + ImmutableSet.of("s1_6")))); + + assertPlan( + planTester.getFragmentPlan(4), + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("max_10"), aggregationFunction("max", ImmutableList.of("max_9"))), + Collections.emptyList(), + Optional.empty(), + INTERMEDIATE, + aggregationTableScan( + singleGroupingSet(), + Collections.emptyList(), + Optional.empty(), + PARTIAL, + "testdb.table1", + ImmutableList.of("max_9"), + ImmutableSet.of("s1_6")))); + } + + @Test + public void testUncorrelatedScalarSubqueryInWhereClauseWithEnforceSingleRowNode() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT s1 FROM table1 where s1 = (select s2 from table1)"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + PlanMatchPattern tableScan1 = + tableScan("testdb.table1", ImmutableList.of("s1"), ImmutableSet.of("s1")); + + // Verify LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──FilterNode + * └──JoinNode + * |──TableScanNode + * ├──EnforceSingleRowNode + * │ └──TableScanNode + + */ + assertPlan( + logicalQueryPlan, + output( + project( + anyTree( + join( + JoinNode.JoinType.INNER, + builder -> builder.left(tableScan1).right(enforceSingleRow(any()))))))); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java new file mode 100644 index 000000000000..b2cadeffd96d --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.db.queryengine.plan.relational.planner.assertions; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; + +import static java.util.Objects.requireNonNull; + +class EquiJoinClauseProvider implements ExpectedValueProvider { + private final SymbolAlias left; + private final SymbolAlias right; + + EquiJoinClauseProvider(SymbolAlias left, SymbolAlias right) { + this.left = requireNonNull(left, "left is null"); + this.right = requireNonNull(right, "right is null"); + } + + @Override + public JoinNode.EquiJoinClause getExpectedValue(SymbolAliases aliases) { + return new JoinNode.EquiJoinClause(left.toSymbol(aliases), right.toSymbol(aliases)); + } + + @Override + public String toString() { + return left + " = " + right; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java new file mode 100644 index 000000000000..b237d78a7646 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java @@ -0,0 +1,185 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.db.queryengine.plan.relational.planner.assertions; + +import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.CanIgnoreReturnValue; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.MatchResult.NO_MATCH; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.equiJoinClause; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.node; + +public final class JoinMatcher implements Matcher { + private final JoinNode.JoinType joinType; + private final List> equiCriteria; + private final boolean ignoreEquiCriteria; + private final Optional filter; + + JoinMatcher( + JoinNode.JoinType joinType, + List> equiCriteria, + boolean ignoreEquiCriteria, + Optional filter) { + this.joinType = requireNonNull(joinType, "joinType is null"); + this.equiCriteria = requireNonNull(equiCriteria, "equiCriteria is null"); + if (ignoreEquiCriteria && !equiCriteria.isEmpty()) { + throw new IllegalArgumentException("ignoreEquiCriteria passed with non-empty equiCriteria"); + } + this.ignoreEquiCriteria = ignoreEquiCriteria; + this.filter = requireNonNull(filter, "filter cannot be null"); + } + + @Override + public boolean shapeMatches(PlanNode node) { + if (!(node instanceof JoinNode)) { + return false; + } + + return ((JoinNode) node).getJoinType() == joinType; + } + + @Override + public MatchResult detailMatches( + PlanNode node, SessionInfo sessionInfo, Metadata metadata, SymbolAliases symbolAliases) { + checkState( + shapeMatches(node), + "Plan testing framework error: shapeMatches returned false in detailMatches in %s", + this.getClass().getName()); + + JoinNode joinNode = (JoinNode) node; + + if (!ignoreEquiCriteria && joinNode.getCriteria().size() != equiCriteria.size()) { + return NO_MATCH; + } + + if (filter.isPresent()) { + if (!joinNode.getFilter().isPresent()) { + return NO_MATCH; + } + if (!new ExpressionVerifier(symbolAliases) + .process(joinNode.getFilter().get(), filter.get())) { + return NO_MATCH; + } + } else { + if (joinNode.getFilter().isPresent()) { + return NO_MATCH; + } + } + + if (!ignoreEquiCriteria) { + /* + * Have to use order-independent comparison; there are no guarantees what order + * the equi criteria will have after planning and optimizing. + */ + Set actual = ImmutableSet.copyOf(joinNode.getCriteria()); + Set expected = + equiCriteria.stream() + .map(maker -> maker.getExpectedValue(symbolAliases)) + .collect(toImmutableSet()); + + if (!expected.equals(actual)) { + return NO_MATCH; + } + } + + return MatchResult.match(); + } + + @Override + public String toString() { + return toStringHelper(this) + .omitNullValues() + .add("type", joinType) + .add("equiCriteria", equiCriteria) + .add("filter", filter.orElse(null)) + .toString(); + } + + public static class Builder { + private final JoinNode.JoinType joinType; + private Optional>> equiCriteria = + Optional.empty(); + private PlanMatchPattern left; + private PlanMatchPattern right; + private Optional filter = Optional.empty(); + private boolean ignoreEquiCriteria; + + public Builder(JoinNode.JoinType joinType) { + this.joinType = joinType; + } + + @CanIgnoreReturnValue + public Builder equiCriteria( + List> expectedEquiCriteria) { + this.equiCriteria = Optional.of(expectedEquiCriteria); + + return this; + } + + @CanIgnoreReturnValue + public Builder equiCriteria(String left, String right) { + this.equiCriteria = Optional.of(ImmutableList.of(equiJoinClause(left, right))); + + return this; + } + + @CanIgnoreReturnValue + public Builder filter(Expression expectedFilter) { + this.filter = Optional.of(expectedFilter); + + return this; + } + + @CanIgnoreReturnValue + public Builder left(PlanMatchPattern left) { + this.left = left; + + return this; + } + + @CanIgnoreReturnValue + public Builder right(PlanMatchPattern right) { + this.right = right; + + return this; + } + + public Builder ignoreEquiCriteria() { + this.ignoreEquiCriteria = true; + return this; + } + + public PlanMatchPattern build() { + return node(JoinNode.class, left, right) + .with( + new JoinMatcher( + joinType, equiCriteria.orElse(ImmutableList.of()), ignoreEquiCriteria, filter)); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index 0b056a2fd6a9..1b5708c4d409 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -22,8 +22,10 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; @@ -48,6 +50,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; import java.util.function.Predicate; import static com.google.common.base.MoreObjects.toStringHelper; @@ -351,15 +354,15 @@ public static PlanMatchPattern patternRecognition(Consumer handler) - { - JoinMatcher.Builder builder = new JoinMatcher.Builder(type); - handler.accept(builder); - return builder.build(); }*/ + public static PlanMatchPattern join( + JoinNode.JoinType type, Consumer handler) { + JoinMatcher.Builder builder = new JoinMatcher.Builder(type); + handler.accept(builder); + return builder.build(); + } + public static PlanMatchPattern sort(PlanMatchPattern source) { return node(SortNode.class, source); } @@ -428,10 +431,10 @@ public static PlanMatchPattern exchange(PlanMatchPattern... sources) { return node(ExchangeNode.class, sources); } - /*public static ExpectedValueProvider equiJoinClause(String left, String right) - { - return new EquiJoinClauseProvider(new SymbolAlias(left), new SymbolAlias(right)); - }*/ + public static ExpectedValueProvider equiJoinClause( + String left, String right) { + return new EquiJoinClauseProvider(new SymbolAlias(left), new SymbolAlias(right)); + } public static SymbolAlias symbol(String alias) { return new SymbolAlias(alias); @@ -559,6 +562,10 @@ public static PlanMatchPattern exchange() { return node(ExchangeNode.class).with(new ExchangeNodeMatcher()); } + public static PlanMatchPattern enforceSingleRow(PlanMatchPattern source) { + return node(EnforceSingleRowNode.class, source); + } + public PlanMatchPattern(List sourcePatterns) { requireNonNull(sourcePatterns, "sourcePatterns are null");