Skip to content

Commit

Permalink
[opt](nereids) recover adoptive bucket shuffle
Browse files Browse the repository at this point in the history
  • Loading branch information
zhongjian.xzj authored and zhongjian.xzj committed Jun 26, 2024
1 parent 94ba2e2 commit c814ff0
Show file tree
Hide file tree
Showing 14 changed files with 19 additions and 18 deletions.
2 changes: 1 addition & 1 deletion ...e-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ private boolean isBucketShuffleDownGrade(Plan oneSidePlan, DistributionSpecHash
} else {
int bucketNum = candidate.getTable().getDefaultDistributionInfo().getBucketNum();
int backEndNum = Math.max(1, ConnectContext.get().getEnv().getClusterInfo()
.getAllBackendIds(true).size());
.getBackendsNumber(true));
int paraNum = Math.min(4, Math.max(1, ConnectContext.get()
.getSessionVariable().getParallelExecInstanceNum()));
return bucketNum < backEndNum * paraNum;
Expand Down
11 changes: 6 additions & 5 deletions regression-test/suites/correctness_p0/test_bucket_shuffle_join.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
suite("test_bucket_shuffle_join") {

sql "set disable_join_reorder=true"
sql "set parallel_pipeline_task_num=1"

sql """ DROP TABLE IF EXISTS `test_colo1` """
sql """ DROP TABLE IF EXISTS `test_colo2` """
Expand All @@ -30,7 +31,7 @@ suite("test_bucket_shuffle_join") {
) ENGINE=OLAP
DUPLICATE KEY(`id`,`name`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 32
DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 4
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
Expand All @@ -46,7 +47,7 @@ suite("test_bucket_shuffle_join") {
) ENGINE=OLAP
DUPLICATE KEY(`id`,`name`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 40
DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 5
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
Expand All @@ -63,7 +64,7 @@ suite("test_bucket_shuffle_join") {
) ENGINE=OLAP
DUPLICATE KEY(`id`,`name`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 48
DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 6
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
Expand Down Expand Up @@ -99,7 +100,7 @@ suite("test_bucket_shuffle_join") {
sql """
create table shuffle_join_t1 ( a varchar(10) not null )
ENGINE=OLAP
DISTRIBUTED BY HASH(a) BUCKETS 40
DISTRIBUTED BY HASH(a) BUCKETS 5
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
Expand All @@ -110,7 +111,7 @@ suite("test_bucket_shuffle_join") {
sql """
create table shuffle_join_t2 ( a varchar(5) not null, b string not null, c char(3) not null )
ENGINE=OLAP
DISTRIBUTED BY HASH(a) BUCKETS 40
DISTRIBUTED BY HASH(a) BUCKETS 5
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query13") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query19") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query44") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query45") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query54") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query56") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query6") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query61") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query68") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query8") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query91") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("query95") {
sql 'set exec_mem_limit=21G'
sql 'set be_number_for_test=3'
sql 'set parallel_fragment_exec_instance_num=8; '
sql 'set parallel_pipeline_task_num=8; '
sql 'set parallel_pipeline_task_num=1; '
sql 'set forbid_unknown_col_stats=true'
sql 'set enable_nereids_timeout = false'
sql 'set enable_runtime_filter_prune=false'
Expand Down

0 comments on commit c814ff0

Please sign in to comment.