diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index 52ba749cf600e01..012dec4c91cf49f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -77,7 +77,28 @@ * Base class for selecting materialized index rules. */ public abstract class AbstractSelectMaterializedIndexRule { - protected boolean shouldSelectIndex(LogicalOlapScan scan) { + protected boolean shouldSelectIndexWithAgg(LogicalOlapScan scan) { + switch (scan.getTable().getKeysType()) { + case AGG_KEYS: + case UNIQUE_KEYS: + case DUP_KEYS: + // SelectMaterializedIndexWithAggregate(R1) run before SelectMaterializedIndexWithoutAggregate(R2) + // if R1 selects baseIndex and preAggStatus is off + // we should give a chance to R2 to check if some prefix-index can be selected + // so if R1 selects baseIndex and preAggStatus is off, we keep scan's index unselected in order to + // let R2 to get a chance to do its work + // at last, after R1, the scan may be the 4 status + // 1. preAggStatus is ON and baseIndex is selected, it means select baseIndex is correct. + // 2. preAggStatus is ON and some other Index is selected, this is correct, too. + // 3. preAggStatus is OFF, no index is selected, it means R2 could get a chance to run + // so we check the preAggStatus and if some index is selected to make sure R1 can be run only once + return scan.getPreAggStatus().isOn() && !scan.isIndexSelected(); + default: + return false; + } + } + + protected boolean shouldSelectIndexWithoutAgg(LogicalOlapScan scan) { switch (scan.getTable().getKeysType()) { case AGG_KEYS: case UNIQUE_KEYS: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java index b4286472dfd6e02..468bb7d8ab0d533 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java @@ -105,7 +105,7 @@ public List buildRules() { return ImmutableList.of( // only agg above scan // Aggregate(Scan) - logicalAggregate(logicalOlapScan().when(this::shouldSelectIndex)).thenApply(ctx -> { + logicalAggregate(logicalOlapScan().when(this::shouldSelectIndexWithAgg)).thenApply(ctx -> { LogicalAggregate agg = ctx.root; LogicalOlapScan scan = agg.child(); SelectResult result = select( @@ -116,7 +116,7 @@ public List buildRules() { agg.getGroupByExpressions(), new HashSet<>(agg.getExpressions())); - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); if (result.exprRewriteMap.isEmpty()) { @@ -139,7 +139,7 @@ public List buildRules() { // filter could push down scan. // Aggregate(Filter(Scan)) - logicalAggregate(logicalFilter(logicalOlapScan().when(this::shouldSelectIndex))) + logicalAggregate(logicalFilter(logicalOlapScan().when(this::shouldSelectIndexWithAgg))) .thenApply(ctx -> { LogicalAggregate> agg = ctx.root; LogicalFilter filter = agg.child(); @@ -162,8 +162,7 @@ public List buildRules() { requiredExpr ); - LogicalOlapScan mvPlan = - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); if (result.exprRewriteMap.isEmpty()) { @@ -191,7 +190,7 @@ public List buildRules() { // column pruning or other projections such as alias, etc. // Aggregate(Project(Scan)) - logicalAggregate(logicalProject(logicalOlapScan().when(this::shouldSelectIndex))) + logicalAggregate(logicalProject(logicalOlapScan().when(this::shouldSelectIndexWithAgg))) .thenApply(ctx -> { LogicalAggregate> agg = ctx.root; LogicalProject project = agg.child(); @@ -207,8 +206,7 @@ public List buildRules() { collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects()) ); - LogicalOlapScan mvPlan = - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); if (result.exprRewriteMap.isEmpty()) { @@ -242,7 +240,7 @@ public List buildRules() { // filter could push down and project. // Aggregate(Project(Filter(Scan))) logicalAggregate(logicalProject(logicalFilter(logicalOlapScan() - .when(this::shouldSelectIndex)))).thenApply(ctx -> { + .when(this::shouldSelectIndexWithAgg)))).thenApply(ctx -> { LogicalAggregate>> agg = ctx.root; LogicalProject> project = agg.child(); LogicalFilter filter = project.child(); @@ -265,8 +263,7 @@ public List buildRules() { requiredExpr ); - LogicalOlapScan mvPlan = - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); if (result.exprRewriteMap.isEmpty()) { @@ -301,7 +298,7 @@ public List buildRules() { // filter can't push down // Aggregate(Filter(Project(Scan))) logicalAggregate(logicalFilter(logicalProject(logicalOlapScan() - .when(this::shouldSelectIndex)))).thenApply(ctx -> { + .when(this::shouldSelectIndexWithAgg)))).thenApply(ctx -> { LogicalAggregate>> agg = ctx.root; LogicalFilter> filter = agg.child(); LogicalProject project = filter.child(); @@ -322,8 +319,7 @@ public List buildRules() { requiredExpr ); - LogicalOlapScan mvPlan = - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); if (result.exprRewriteMap.isEmpty()) { @@ -357,48 +353,49 @@ public List buildRules() { // only agg above scan // Aggregate(Repeat(Scan)) - logicalAggregate(logicalRepeat(logicalOlapScan().when(this::shouldSelectIndex))).thenApply(ctx -> { - LogicalAggregate> agg = ctx.root; - LogicalRepeat repeat = agg.child(); - LogicalOlapScan scan = repeat.child(); - SelectResult result = select( - scan, - agg.getInputSlots(), - ImmutableSet.of(), - extractAggFunctionAndReplaceSlot(agg, Optional.empty()), - nonVirtualGroupByExprs(agg), - new HashSet<>(agg.getExpressions())); - - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); - SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - - if (result.exprRewriteMap.isEmpty()) { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - agg.withChildren( - repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan) - ), mvPlan)); - } else { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput( - agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), + logicalAggregate( + logicalRepeat(logicalOlapScan().when(this::shouldSelectIndexWithAgg))).thenApply(ctx -> { + LogicalAggregate> agg = ctx.root; + LogicalRepeat repeat = agg.child(); + LogicalOlapScan scan = repeat.child(); + SelectResult result = select( + scan, + agg.getInputSlots(), + ImmutableSet.of(), + extractAggFunctionAndReplaceSlot(agg, Optional.empty()), + nonVirtualGroupByExprs(agg), + new HashSet<>(agg.getExpressions())); + + LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + + if (result.exprRewriteMap.isEmpty()) { + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + agg.withChildren( repeat.withAggOutputAndChild( generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan) ), mvPlan)); - } - }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_SCAN), + } else { + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput( + agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + repeat.withAggOutputAndChild( + generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan) + ), mvPlan)); + } + }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_SCAN), // filter could push down scan. // Aggregate(Repeat(Filter(Scan))) - logicalAggregate(logicalRepeat(logicalFilter(logicalOlapScan().when(this::shouldSelectIndex)))) + logicalAggregate(logicalRepeat(logicalFilter(logicalOlapScan().when(this::shouldSelectIndexWithAgg)))) .thenApply(ctx -> { LogicalAggregate>> agg = ctx.root; LogicalRepeat> repeat = agg.child(); @@ -422,8 +419,7 @@ public List buildRules() { requiredExpr ); - LogicalOlapScan mvPlan = - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); if (result.exprRewriteMap.isEmpty()) { @@ -457,7 +453,7 @@ public List buildRules() { // column pruning or other projections such as alias, etc. // Aggregate(Repeat(Project(Scan))) - logicalAggregate(logicalRepeat(logicalProject(logicalOlapScan().when(this::shouldSelectIndex)))) + logicalAggregate(logicalRepeat(logicalProject(logicalOlapScan().when(this::shouldSelectIndexWithAgg)))) .thenApply(ctx -> { LogicalAggregate>> agg = ctx.root; LogicalRepeat> repeat = agg.child(); @@ -474,8 +470,7 @@ public List buildRules() { collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects()) ); - LogicalOlapScan mvPlan = - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); if (result.exprRewriteMap.isEmpty()) { @@ -514,7 +509,7 @@ public List buildRules() { // filter could push down and project. // Aggregate(Repeat(Project(Filter(Scan)))) logicalAggregate(logicalRepeat(logicalProject(logicalFilter(logicalOlapScan() - .when(this::shouldSelectIndex))))).thenApply(ctx -> { + .when(this::shouldSelectIndexWithAgg))))).thenApply(ctx -> { LogicalAggregate>>> agg = ctx.root; LogicalRepeat>> repeat = agg.child(); @@ -539,8 +534,7 @@ public List buildRules() { requiredExpr ); - LogicalOlapScan mvPlan = - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); if (result.exprRewriteMap.isEmpty()) { @@ -582,7 +576,7 @@ public List buildRules() { // filter can't push down // Aggregate(Repeat(Filter(Project(Scan)))) logicalAggregate(logicalRepeat(logicalFilter(logicalProject(logicalOlapScan() - .when(this::shouldSelectIndex))))).thenApply(ctx -> { + .when(this::shouldSelectIndexWithAgg))))).thenApply(ctx -> { LogicalAggregate>>> agg = ctx.root; LogicalRepeat>> repeat = agg.child(); @@ -605,8 +599,7 @@ public List buildRules() { requiredExpr ); - LogicalOlapScan mvPlan = - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); if (result.exprRewriteMap.isEmpty()) { @@ -750,6 +743,19 @@ public SelectResult(PreAggStatus preAggStatus, long indexId, ExprRewriteMap expr } } + private static LogicalOlapScan createLogicalOlapScan(LogicalOlapScan scan, SelectResult result) { + LogicalOlapScan mvPlan; + if (result.preAggStatus.isOff()) { + // we only set preAggStatus and make index unselected to let SelectMaterializedIndexWithoutAggregate + // have a chance to run and select proper index + mvPlan = scan.withPreAggStatus(result.preAggStatus); + } else { + mvPlan = + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + } + return mvPlan; + } + /** * Do aggregate function extraction and replace aggregate function's input slots by underlying project. *

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java index 9a752db6532272f..2ad22b003164f48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java @@ -24,6 +24,8 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; +import org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule.ReplaceExpressions; +import org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule.SlotContext; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.PreAggStatus; @@ -59,7 +61,7 @@ public List buildRules() { return ImmutableList.of( // project with pushdown filter. // Project(Filter(Scan)) - logicalProject(logicalFilter(logicalOlapScan().when(this::shouldSelectIndex))) + logicalProject(logicalFilter(logicalOlapScan().when(this::shouldSelectIndexWithoutAgg))) .thenApply(ctx -> { LogicalProject> project = ctx.root; LogicalFilter filter = project.child(); @@ -79,7 +81,7 @@ public List buildRules() { // project with filter that cannot be pushdown. // Filter(Project(Scan)) - logicalFilter(logicalProject(logicalOlapScan().when(this::shouldSelectIndex))) + logicalFilter(logicalProject(logicalOlapScan().when(this::shouldSelectIndexWithoutAgg))) .thenApply(ctx -> { LogicalFilter> filter = ctx.root; LogicalProject project = filter.child(); @@ -98,13 +100,14 @@ public List buildRules() { // scan with filters could be pushdown. // Filter(Scan) - logicalFilter(logicalOlapScan().when(this::shouldSelectIndex)) + logicalFilter(logicalOlapScan().when(this::shouldSelectIndexWithoutAgg)) .thenApply(ctx -> { LogicalFilter filter = ctx.root; LogicalOlapScan scan = filter.child(); LogicalOlapScan mvPlan = select( scan, filter::getOutputSet, filter::getConjuncts, - new HashSet<>(filter.getExpressions())); + Stream.concat(filter.getExpressions().stream(), + filter.getOutputSet().stream()).collect(ImmutableSet.toImmutableSet())); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); return new LogicalProject( @@ -116,7 +119,7 @@ public List buildRules() { // project and scan. // Project(Scan) - logicalProject(logicalOlapScan().when(this::shouldSelectIndex)) + logicalProject(logicalOlapScan().when(this::shouldSelectIndexWithoutAgg)) .thenApply(ctx -> { LogicalProject project = ctx.root; LogicalOlapScan scan = project.child(); @@ -135,7 +138,7 @@ public List buildRules() { // only scan. logicalOlapScan() - .when(this::shouldSelectIndex) + .when(this::shouldSelectIndexWithoutAgg) .thenApply(ctx -> { LogicalOlapScan scan = ctx.root; @@ -196,7 +199,9 @@ private LogicalOlapScan select( // PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys. preAggStatus = PreAggStatus.on(); } else { - preAggStatus = PreAggStatus.off("No aggregate on scan."); + // if PreAggStatus is OFF, we use the message from SelectMaterializedIndexWithAggregate + preAggStatus = scan.getPreAggStatus().isOff() ? scan.getPreAggStatus() + : PreAggStatus.off("No aggregate on scan."); } if (table.getIndexIdToMeta().size() == 1) { return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId); diff --git a/regression-test/suites/nereids_p0/test_mv_select.groovy b/regression-test/suites/nereids_p0/test_mv_select.groovy new file mode 100644 index 000000000000000..096cfd50fab16b0 --- /dev/null +++ b/regression-test/suites/nereids_p0/test_mv_select.groovy @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_mv_select") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + sql "DROP TABLE IF EXISTS mv_test_table_t" + sql """ + CREATE TABLE `mv_test_table_t` ( + `Uid` bigint(20) NOT NULL, + `DateCode` int(11) NOT NULL, + `ProductId` bigint(20) NOT NULL, + `LiveSales` int(11) REPLACE NULL + ) ENGINE=OLAP + AGGREGATE KEY(`Uid`, `DateCode`, `ProductId`) + DISTRIBUTED BY HASH(`Uid`, `ProductId`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql "ALTER TABLE mv_test_table_t ADD ROLLUP rollup_mv_test_table_t(ProductId,DateCode,Uid);" + + explain { + sql ("""select Uid + from mv_test_table_t + where ProductId = 3570093298674738221 and DateCode >=20230919 and DateCode <=20231018 + group by Uid;""") + contains "mv_test_table_t" + } +} \ No newline at end of file