diff --git a/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql b/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql index cb8bb5d9cb40b9..312a0a25fac45b 100644 --- a/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql @@ -342,4 +342,13 @@ CREATE TABLE Doris.Doris ( CREATE TABLE Doris.doris ( id varchar(128) -); \ No newline at end of file +); + +create table doris_test.compoundpredicate_test ( +pk int, +col_int_undef_signed int, +col_int_undef_signed2 int +); + +create table doris_test.text_push (pk varchar(10)); + diff --git a/docker/thirdparties/docker-compose/mysql/init/04-insert.sql b/docker/thirdparties/docker-compose/mysql/init/04-insert.sql index d71986b135492c..4580440e9bd4f4 100644 --- a/docker/thirdparties/docker-compose/mysql/init/04-insert.sql +++ b/docker/thirdparties/docker-compose/mysql/init/04-insert.sql @@ -1164,3 +1164,7 @@ insert into doris_test.test_zd (id,d_z) VALUES (1,'0000-00-00'),(2,'2022-01-01') insert into Doris.DORIS values ('DORIS'); insert into Doris.Doris values ('Doris'); insert into Doris.doris values ('doris'); + +insert into doris_test.compoundpredicate_test(pk,col_int_undef_signed,col_int_undef_signed2) values (0,null,23868),(1,68,-18),(2,19030,-125),(3,16539,null),(4,null,null),(5,null,-127),(6,14680,-26424),(7,-22270,12722),(8,null,null),(9,null,null),(10,null,7744),(11,null,-94),(12,16970,95),(13,null,7023),(14,null,1),(15,3679,-11),(16,null,-1079),(17,-22,null),(18,30995,null),(19,null,-79); + +insert into doris_test.text_push values('a'),('aa'),('aaa'); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java index ba7e684820ccc4..26df521d9e9843 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java @@ -61,6 +61,7 @@ public class JdbcScanNode extends ExternalScanNode { private final List columns = new ArrayList(); private final List filters = new ArrayList(); + private final List pushedDownConjuncts = new ArrayList<>(); private String tableName; private TOdbcTableType jdbcType; private String graphQueryString = ""; @@ -128,7 +129,7 @@ private void createJdbcFilters() { for (Expr individualConjunct : pushDownConjuncts) { String filter = conjunctExprToString(jdbcType, individualConjunct, tbl); filters.add(filter); - conjuncts.remove(individualConjunct); + pushedDownConjuncts.add(individualConjunct); } } @@ -165,7 +166,7 @@ private void createJdbcColumns() { } private boolean shouldPushDownLimit() { - return limit != -1 && conjuncts.isEmpty(); + return limit != -1 && conjuncts.size() == pushedDownConjuncts.size(); } private String getJdbcQueryStr() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java index 2f9aa4a83342ec..391f632761906e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java @@ -60,6 +60,7 @@ public class OdbcScanNode extends ExternalScanNode { private final List columns = new ArrayList(); private final List filters = new ArrayList(); + private final List pushedDownConjuncts = new ArrayList<>(); private String tblName; private String connectString; private TOdbcTableType odbcType; @@ -138,7 +139,7 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { // only all conjuncts be pushed down as filter, we can // push down limit operation to ODBC table private boolean shouldPushDownLimit() { - return limit != -1 && conjuncts.isEmpty(); + return limit != -1 && conjuncts.size() == pushedDownConjuncts.size(); } private String getOdbcQueryStr() { @@ -208,7 +209,7 @@ private void createOdbcFilters() { if (shouldPushDownConjunct(odbcType, p)) { String filter = JdbcScanNode.conjunctExprToString(odbcType, p, tbl); filters.add(filter); - conjuncts.remove(p); + pushedDownConjuncts.add(p); } } } diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out index b2686b6c362230..f2a05e3d0896be 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out @@ -247,15 +247,6 @@ workload_policy 张三6 11 124314567 123 321312 1999-02-13T00:00 中国 男 0 张三7 11 123445167 123 321312 1998-02-13T00:00 中国 男 0 --- !test_filter_not_old_plan -- -张三1 11 12345678 123 321312 1999-02-13T00:00 中国 男 0 -张三2 11 12345671 123 321312 1999-02-13T00:00 中国 男 0 -张三3 11 12345673 123 321312 1999-02-13T00:00 中国 男 0 -张三4 11 123456711 123 321312 1999-02-13T00:00 中国 男 0 -张三5 11 1232134567 123 321312 1999-02-13T00:00 中国 男 0 -张三6 11 124314567 123 321312 1999-02-13T00:00 中国 男 0 -张三7 11 123445167 123 321312 1998-02-13T00:00 中国 男 0 - -- !test_insert1 -- doris1 18 @@ -454,3 +445,22 @@ doris -- !sql -- 1 +-- !sql -- +10 \N 7744 +11 \N -94 +12 16970 95 +13 \N 7023 +14 \N 1 +15 3679 -11 +16 \N -1079 +17 -22 \N +18 30995 \N +19 \N -79 +5 \N -127 +6 14680 -26424 +7 -22270 12722 +8 \N \N +9 \N \N + +-- !sql -- + diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy index f74151e07ffcc9..bef3ba03bd7a26 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy @@ -133,102 +133,89 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc qt_sql """select current_catalog()""" sql """switch ${catalog_name}""" qt_sql """select current_catalog()""" - def res_dbs_log = sql "show databases;" - for(int i = 0;i < res_dbs_log.size();i++) { - def tbs = sql "show tables from `${res_dbs_log[i][0]}`" - log.info( "database = ${res_dbs_log[i][0]} => tables = "+tbs.toString()) - } - try { - - sql """ use ${ex_db_name}""" - - order_qt_ex_tb0 """ select id, name from ${ex_tb0} order by id; """ - sql """ insert into internal.${internal_db_name}.${inDorisTable} select id, name from ${ex_tb0}; """ - order_qt_in_tb """ select id, name from internal.${internal_db_name}.${inDorisTable} order by id; """ - - order_qt_ex_tb1 """ select * from ${ex_tb1} order by id; """ - order_qt_ex_tb2 """ select * from ${ex_tb2} order by id; """ - order_qt_ex_tb3 """ select * from ${ex_tb3} order by game_code; """ - order_qt_ex_tb4 """ select * from ${ex_tb4} order by products_id; """ - order_qt_ex_tb5 """ select * from ${ex_tb5} order by id; """ - order_qt_ex_tb6 """ select * from ${ex_tb6} order by id; """ - order_qt_ex_tb7 """ select * from ${ex_tb7} order by id; """ - order_qt_ex_tb8 """ select * from ${ex_tb8} order by uid; """ - order_qt_ex_tb9 """ select * from ${ex_tb9} order by c_date; """ - order_qt_ex_tb10 """ select * from ${ex_tb10} order by aa; """ - order_qt_ex_tb11 """ select * from ${ex_tb11} order by aa; """ - order_qt_ex_tb12 """ select * from ${ex_tb12} order by cc; """ - order_qt_ex_tb13 """ select * from ${ex_tb13} order by name; """ - order_qt_ex_tb14 """ select * from ${ex_tb14} order by tid; """ - order_qt_ex_tb15 """ select * from ${ex_tb15} order by col1; """ - order_qt_ex_tb16 """ select * from ${ex_tb16} order by id; """ - order_qt_ex_tb17 """ select * from ${ex_tb17} order by id; """ - order_qt_ex_tb18 """ select * from ${ex_tb18} order by num_tinyint; """ - order_qt_ex_tb19 """ select * from ${ex_tb19} order by date_value; """ - order_qt_ex_tb20 """ select * from ${ex_tb20} order by decimal_normal; """ - order_qt_ex_tb21_1 """ select `key`, `id` from ${ex_tb21} where `key` = 2 order by id;""" - order_qt_ex_tb21_2 """ select `key`, `id` from ${ex_tb21} where `key` like 2 order by id;""" - order_qt_ex_tb21_3 """ select `key`, `id` from ${ex_tb21} where `key` in (1,2) order by id;""" - order_qt_ex_tb21_4 """ select `key`, `id` from ${ex_tb21} where abs(`key`) = 2 order by id;""" - order_qt_ex_tb21_5 """ select `key`, `id` from ${ex_tb21} where `key` between 1 and 2 order by id;""" - order_qt_ex_tb21_6 """ select `key`, `id` from ${ex_tb21} where `key` = case when id = 1 then 1 else 0 end order by id;""" - order_qt_ex_tb21_7 """ select (`key` +1) as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;""" - order_qt_ex_tb21_8 """ select `key` as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;""" - order_qt_information_schema """ show tables from information_schema; """ - order_qt_dt """select * from ${dt}; """ - order_qt_dt_null """select * from ${dt_null} order by 1; """ - order_qt_test_dz """select * from ${test_zd} order by 1; """ - order_qt_test_filter_not """select * from ${ex_tb13} where name not like '%张三0%' order by 1; """ - explain { - sql("select `datetime` from all_types where to_date(`datetime`) = '2012-10-25';") - contains """ SELECT `datetime` FROM `doris_test`.`all_types` WHERE (date(`datetime`) = '2012-10-25')""" - } + sql """ use ${ex_db_name}""" - explain { - sql("select /*+ SET_VAR(enable_ext_func_pred_pushdown = false) */ `datetime` from all_types where to_date(`datetime`) = '2012-10-25';") - contains """SELECT `datetime` FROM `doris_test`.`all_types`""" - } + order_qt_ex_tb0 """ select id, name from ${ex_tb0} order by id; """ + sql """ insert into internal.${internal_db_name}.${inDorisTable} select id, name from ${ex_tb0}; """ + order_qt_in_tb """ select id, name from internal.${internal_db_name}.${inDorisTable} order by id; """ + + order_qt_ex_tb1 """ select * from ${ex_tb1} order by id; """ + order_qt_ex_tb2 """ select * from ${ex_tb2} order by id; """ + order_qt_ex_tb3 """ select * from ${ex_tb3} order by game_code; """ + order_qt_ex_tb4 """ select * from ${ex_tb4} order by products_id; """ + order_qt_ex_tb5 """ select * from ${ex_tb5} order by id; """ + order_qt_ex_tb6 """ select * from ${ex_tb6} order by id; """ + order_qt_ex_tb7 """ select * from ${ex_tb7} order by id; """ + order_qt_ex_tb8 """ select * from ${ex_tb8} order by uid; """ + order_qt_ex_tb9 """ select * from ${ex_tb9} order by c_date; """ + order_qt_ex_tb10 """ select * from ${ex_tb10} order by aa; """ + order_qt_ex_tb11 """ select * from ${ex_tb11} order by aa; """ + order_qt_ex_tb12 """ select * from ${ex_tb12} order by cc; """ + order_qt_ex_tb13 """ select * from ${ex_tb13} order by name; """ + order_qt_ex_tb14 """ select * from ${ex_tb14} order by tid; """ + order_qt_ex_tb15 """ select * from ${ex_tb15} order by col1; """ + order_qt_ex_tb16 """ select * from ${ex_tb16} order by id; """ + order_qt_ex_tb17 """ select * from ${ex_tb17} order by id; """ + order_qt_ex_tb18 """ select * from ${ex_tb18} order by num_tinyint; """ + order_qt_ex_tb19 """ select * from ${ex_tb19} order by date_value; """ + order_qt_ex_tb20 """ select * from ${ex_tb20} order by decimal_normal; """ + order_qt_ex_tb21_1 """ select `key`, `id` from ${ex_tb21} where `key` = 2 order by id;""" + order_qt_ex_tb21_2 """ select `key`, `id` from ${ex_tb21} where `key` like 2 order by id;""" + order_qt_ex_tb21_3 """ select `key`, `id` from ${ex_tb21} where `key` in (1,2) order by id;""" + order_qt_ex_tb21_4 """ select `key`, `id` from ${ex_tb21} where abs(`key`) = 2 order by id;""" + order_qt_ex_tb21_5 """ select `key`, `id` from ${ex_tb21} where `key` between 1 and 2 order by id;""" + order_qt_ex_tb21_6 """ select `key`, `id` from ${ex_tb21} where `key` = case when id = 1 then 1 else 0 end order by id;""" + order_qt_ex_tb21_7 """ select (`key` +1) as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;""" + order_qt_ex_tb21_8 """ select `key` as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;""" + order_qt_information_schema """ show tables from information_schema; """ + order_qt_dt """select * from ${dt}; """ + order_qt_dt_null """select * from ${dt_null} order by 1; """ + order_qt_test_dz """select * from ${test_zd} order by 1; """ + order_qt_test_filter_not """select * from ${ex_tb13} where name not like '%张三0%' order by 1; """ + explain { + sql("select `datetime` from all_types where to_date(`datetime`) = '2012-10-25';") + contains """ SELECT `datetime` FROM `doris_test`.`all_types` WHERE (date(`datetime`) = '2012-10-25')""" + } + + explain { + sql("select /*+ SET_VAR(enable_ext_func_pred_pushdown = false) */ `datetime` from all_types where to_date(`datetime`) = '2012-10-25';") + contains """SELECT `datetime` FROM `doris_test`.`all_types`""" + } - // test insert - String uuid1 = UUID.randomUUID().toString(); - connect(user=user, password="${pwd}", url=url) { - try { - sql """ insert into ${catalog_name}.${ex_db_name}.${test_insert} values ('${uuid1}', 'doris1', 18) """ - fail() - } catch (Exception e) { - log.info(e.getMessage()) - } + // test insert + String uuid1 = UUID.randomUUID().toString(); + connect(user=user, password="${pwd}", url=url) { + try { + sql """ insert into ${catalog_name}.${ex_db_name}.${test_insert} values ('${uuid1}', 'doris1', 18) """ + fail() + } catch (Exception e) { + log.info(e.getMessage()) } + } - sql """GRANT LOAD_PRIV ON ${catalog_name}.${ex_db_name}.${test_insert} TO ${user}""" + sql """GRANT LOAD_PRIV ON ${catalog_name}.${ex_db_name}.${test_insert} TO ${user}""" - connect(user=user, password="${pwd}", url=url) { - try { - sql """ insert into ${catalog_name}.${ex_db_name}.${test_insert} values ('${uuid1}', 'doris1', 18) """ - } catch (Exception e) { - fail(); - } + connect(user=user, password="${pwd}", url=url) { + try { + sql """ insert into ${catalog_name}.${ex_db_name}.${test_insert} values ('${uuid1}', 'doris1', 18) """ + } catch (Exception e) { + fail(); } - order_qt_test_insert1 """ select name, age from ${test_insert} where id = '${uuid1}' order by age """ - - String uuid2 = UUID.randomUUID().toString(); - sql """ insert into ${test_insert} values ('${uuid2}', 'doris2', 19), ('${uuid2}', 'doris3', 20) """ - order_qt_test_insert2 """ select name, age from ${test_insert} where id = '${uuid2}' order by age """ - - sql """ insert into ${test_insert} select * from ${test_insert} where id = '${uuid2}' """ - order_qt_test_insert3 """ select name, age from ${test_insert} where id = '${uuid2}' order by age """ - - String uuid3 = UUID.randomUUID().toString(); - sql """ INSERT INTO ${test_insert2} VALUES - ('${uuid3}', true, 'abcHa1.12345', '1.123450xkalowadawd', '2022-10-01', 3.14159, 1, 2, 0, 100000, 1.2345678, 24.000, '07:09:51', '2022', '2022-11-27 07:09:51', '2022-11-27 07:09:51'); """ - order_qt_test_insert4 """ select k1,k2,k3,k4,k5,k6,k7,k8,k9,k10,k11,k12,k13,k14,k15 from ${test_insert2} where id = '${uuid3}' """ - } finally { - res_dbs_log = sql "show databases;" - for(int i = 0;i < res_dbs_log.size();i++) { - def tbs = sql "show tables from `${res_dbs_log[i][0]}`" - log.info( "database = ${res_dbs_log[i][0]} => tables = "+tbs.toString()) - } - } + } + order_qt_test_insert1 """ select name, age from ${test_insert} where id = '${uuid1}' order by age """ + + String uuid2 = UUID.randomUUID().toString(); + sql """ insert into ${test_insert} values ('${uuid2}', 'doris2', 19), ('${uuid2}', 'doris3', 20) """ + order_qt_test_insert2 """ select name, age from ${test_insert} where id = '${uuid2}' order by age """ + + sql """ insert into ${test_insert} select * from ${test_insert} where id = '${uuid2}' """ + order_qt_test_insert3 """ select name, age from ${test_insert} where id = '${uuid2}' order by age """ + + String uuid3 = UUID.randomUUID().toString(); + sql """ INSERT INTO ${test_insert2} VALUES + ('${uuid3}', true, 'abcHa1.12345', '1.123450xkalowadawd', '2022-10-01', 3.14159, 1, 2, 0, 100000, 1.2345678, 24.000, '07:09:51', '2022', '2022-11-27 07:09:51', '2022-11-27 07:09:51'); """ + order_qt_test_insert4 """ select k1,k2,k3,k4,k5,k6,k7,k8,k9,k10,k11,k12,k13,k14,k15 from ${test_insert2} where id = '${uuid3}' """ + sql """ drop catalog if exists ${catalog_name} """ // test only_specified_database argument @@ -313,41 +300,29 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc "jdbc.driver_class" = "com.mysql.cj.jdbc.Driver"); """ sql """ switch ${catalog_name} """ - - res_dbs_log = sql "show databases;" - for(int i = 0;i < res_dbs_log.size();i++) { - def tbs = sql "show tables from `${res_dbs_log[i][0]}`" - log.info( "database = ${res_dbs_log[i][0]} => tables = "+tbs.toString()) - } - try { - sql """ use ${ex_db_name} """ - order_qt_ex_tb1 """ select * from ${ex_tb1} order by id; """ - - // test all types supported by MySQL - sql """use doris_test;""" - qt_mysql_all_types """select * from all_types order by tinyint_u;""" - - // test insert into internal.db.table select * from all_types - sql """ insert into internal.${internal_db_name}.${test_insert_all_types} select * from all_types; """ - order_qt_select_insert_all_types """ select * from internal.${internal_db_name}.${test_insert_all_types} order by tinyint_u; """ - - // test CTAS - sql """ drop table if exists internal.${internal_db_name}.${test_ctas} """ - sql """ create table internal.${internal_db_name}.${test_ctas} - PROPERTIES("replication_num" = "1") - AS select * from all_types; - """ - order_qt_ctas """select * from internal.${internal_db_name}.${test_ctas} order by tinyint_u;""" + sql """ use ${ex_db_name} """ + order_qt_ex_tb1 """ select * from ${ex_tb1} order by id; """ + + // test all types supported by MySQL + sql """use doris_test;""" + qt_mysql_all_types """select * from all_types order by tinyint_u;""" + + // test insert into internal.db.table select * from all_types + sql """ insert into internal.${internal_db_name}.${test_insert_all_types} select * from all_types; """ + order_qt_select_insert_all_types """ select * from internal.${internal_db_name}.${test_insert_all_types} order by tinyint_u; """ + + // test CTAS + sql """ drop table if exists internal.${internal_db_name}.${test_ctas} """ + sql """ create table internal.${internal_db_name}.${test_ctas} + PROPERTIES("replication_num" = "1") + AS select * from all_types; + """ + + order_qt_ctas """select * from internal.${internal_db_name}.${test_ctas} order by tinyint_u;""" + + order_qt_ctas_desc """desc internal.${internal_db_name}.${test_ctas};""" - order_qt_ctas_desc """desc internal.${internal_db_name}.${test_ctas};""" - } finally { - res_dbs_log = sql "show databases;" - for(int i = 0;i < res_dbs_log.size();i++) { - def tbs = sql "show tables from `${res_dbs_log[i][0]}`" - log.info( "database = ${res_dbs_log[i][0]} => tables = "+tbs.toString()) - } - } sql """ drop catalog if exists ${catalog_name} """ // test mysql view @@ -624,6 +599,22 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc sql """drop catalog if exists mysql_rename2;""" + sql """drop catalog if exists mysql_conjuncts;""" + + sql """create catalog if not exists mysql_conjuncts properties( + "type"="jdbc", + "user"="root", + "password"="123456", + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false&zeroDateTimeBehavior=convertToNull", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver" + );""" + + order_qt_sql """SELECT * FROM mysql_conjuncts.doris_test.compoundpredicate_test WHERE (pk > 4) OR ((pk < 6 OR pk > 7) AND col_int_undef_signed < 1);""" + + order_qt_sql """select * from mysql_conjuncts.doris_test.text_push where pk <=7;""" + + sql """drop catalog if exists mysql_conjuncts;""" } }