From c814ff004e4632cd4e6bf77fc5fb1a80c8780a27 Mon Sep 17 00:00:00 2001 From: "zhongjian.xzj" Date: Wed, 26 Jun 2024 15:33:39 +0800 Subject: [PATCH] [opt](nereids) recover adoptive bucket shuffle --- .../properties/ChildrenPropertiesRegulator.java | 2 +- .../correctness_p0/test_bucket_shuffle_join.groovy | 11 ++++++----- .../bs_downgrade_shape/query13.groovy | 2 +- .../bs_downgrade_shape/query19.groovy | 2 +- .../bs_downgrade_shape/query44.groovy | 2 +- .../bs_downgrade_shape/query45.groovy | 2 +- .../bs_downgrade_shape/query54.groovy | 2 +- .../bs_downgrade_shape/query56.groovy | 2 +- .../bs_downgrade_shape/query6.groovy | 2 +- .../bs_downgrade_shape/query61.groovy | 2 +- .../bs_downgrade_shape/query68.groovy | 2 +- .../bs_downgrade_shape/query8.groovy | 2 +- .../bs_downgrade_shape/query91.groovy | 2 +- .../bs_downgrade_shape/query95.groovy | 2 +- 14 files changed, 19 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java index 58dceaa78d814ce..2d366b85d7e3579 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java @@ -219,7 +219,7 @@ private boolean isBucketShuffleDownGrade(Plan oneSidePlan, DistributionSpecHash } else { int bucketNum = candidate.getTable().getDefaultDistributionInfo().getBucketNum(); int backEndNum = Math.max(1, ConnectContext.get().getEnv().getClusterInfo() - .getAllBackendIds(true).size()); + .getBackendsNumber(true)); int paraNum = Math.min(4, Math.max(1, ConnectContext.get() .getSessionVariable().getParallelExecInstanceNum())); return bucketNum < backEndNum * paraNum; diff --git a/regression-test/suites/correctness_p0/test_bucket_shuffle_join.groovy b/regression-test/suites/correctness_p0/test_bucket_shuffle_join.groovy index b05a4765f56cd12..1ca2d6845de9390 100644 --- a/regression-test/suites/correctness_p0/test_bucket_shuffle_join.groovy +++ b/regression-test/suites/correctness_p0/test_bucket_shuffle_join.groovy @@ -18,6 +18,7 @@ suite("test_bucket_shuffle_join") { sql "set disable_join_reorder=true" + sql "set parallel_pipeline_task_num=1" sql """ DROP TABLE IF EXISTS `test_colo1` """ sql """ DROP TABLE IF EXISTS `test_colo2` """ @@ -30,7 +31,7 @@ suite("test_bucket_shuffle_join") { ) ENGINE=OLAP DUPLICATE KEY(`id`,`name`) COMMENT 'OLAP' - DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 32 + DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 4 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "in_memory" = "false", @@ -46,7 +47,7 @@ suite("test_bucket_shuffle_join") { ) ENGINE=OLAP DUPLICATE KEY(`id`,`name`) COMMENT 'OLAP' - DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 40 + DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 5 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "in_memory" = "false", @@ -63,7 +64,7 @@ suite("test_bucket_shuffle_join") { ) ENGINE=OLAP DUPLICATE KEY(`id`,`name`) COMMENT 'OLAP' - DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 48 + DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 6 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "in_memory" = "false", @@ -99,7 +100,7 @@ suite("test_bucket_shuffle_join") { sql """ create table shuffle_join_t1 ( a varchar(10) not null ) ENGINE=OLAP - DISTRIBUTED BY HASH(a) BUCKETS 40 + DISTRIBUTED BY HASH(a) BUCKETS 5 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "in_memory" = "false", @@ -110,7 +111,7 @@ suite("test_bucket_shuffle_join") { sql """ create table shuffle_join_t2 ( a varchar(5) not null, b string not null, c char(3) not null ) ENGINE=OLAP - DISTRIBUTED BY HASH(a) BUCKETS 40 + DISTRIBUTED BY HASH(a) BUCKETS 5 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "in_memory" = "false", diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.groovy index 02586d5c8acf391..2367b0fe7d618cf 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.groovy @@ -25,7 +25,7 @@ suite("query13") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.groovy index 5d8e7b72b9644fa..df34248c37059a4 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.groovy @@ -25,7 +25,7 @@ suite("query19") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.groovy index 2545983724a84f7..8d9aa785b209468 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.groovy @@ -25,7 +25,7 @@ suite("query44") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.groovy index 7e7510ff6a0b22a..6e421616a0f8278 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.groovy @@ -25,7 +25,7 @@ suite("query45") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.groovy index 9d22cc30b66b7d8..57a875d02b54122 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.groovy @@ -25,7 +25,7 @@ suite("query54") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.groovy index b1c7169b0f49ea3..2133f123b57ac7c 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.groovy @@ -25,7 +25,7 @@ suite("query56") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.groovy index 5708cdf3572bc76..d40d6351245119d 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.groovy @@ -25,7 +25,7 @@ suite("query6") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.groovy index c158a01bdead39c..0c1dd43c0b873cb 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.groovy @@ -25,7 +25,7 @@ suite("query61") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.groovy index 0214be7a485d13b..1d6a42dee38a815 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.groovy @@ -25,7 +25,7 @@ suite("query68") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.groovy index 78597d354eb383b..234724a4dbed247 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.groovy @@ -25,7 +25,7 @@ suite("query8") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.groovy index c8d78357912f63f..e07a1aff8708be6 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.groovy @@ -25,7 +25,7 @@ suite("query91") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.groovy index b3439e40240e11a..78385cf81a487f4 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.groovy @@ -25,7 +25,7 @@ suite("query95") { sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' sql 'set parallel_fragment_exec_instance_num=8; ' - sql 'set parallel_pipeline_task_num=8; ' + sql 'set parallel_pipeline_task_num=1; ' sql 'set forbid_unknown_col_stats=true' sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false'