Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix](mtmv)Fix high level materialized view not hit because group by eliminate fail #36888

Merged
merged 4 commits into from
Jul 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,10 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws Ana
return cache;
}

public MTMVCache getCache() {
return cache;
}

public Map<String, String> getMvProperties() {
readMvLock();
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
/**
* This is responsible for single table rewriting according to different pattern
* */
public abstract class MaterializedViewScanRule extends AbstractMaterializedViewRule {
public abstract class AbstractMaterializedViewScanRule extends AbstractMaterializedViewRule {

@Override
protected Plan rewriteQueryByView(MatchMode matchMode,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
/**
* MaterializedViewFilterProjectScanRule
*/
public class MaterializedViewFilterProjectScanRule extends MaterializedViewScanRule {
public class MaterializedViewFilterProjectScanRule extends AbstractMaterializedViewScanRule {

public static final MaterializedViewFilterProjectScanRule INSTANCE = new MaterializedViewFilterProjectScanRule();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
/**
* MaterializedViewFilterScanRule
*/
public class MaterializedViewFilterScanRule extends MaterializedViewScanRule {
public class MaterializedViewFilterScanRule extends AbstractMaterializedViewScanRule {

public static final MaterializedViewFilterScanRule INSTANCE = new MaterializedViewFilterScanRule();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
/**
* MaterializedViewOnlyScanRule
*/
public class MaterializedViewOnlyScanRule extends MaterializedViewScanRule {
public class MaterializedViewOnlyScanRule extends AbstractMaterializedViewScanRule {

public static final MaterializedViewOnlyScanRule INSTANCE = new MaterializedViewOnlyScanRule();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
/**
* MaterializedViewProjectFilterScanRule
*/
public class MaterializedViewProjectFilterScanRule extends MaterializedViewScanRule {
public class MaterializedViewProjectFilterScanRule extends AbstractMaterializedViewScanRule {

public static final MaterializedViewProjectFilterScanRule INSTANCE = new MaterializedViewProjectFilterScanRule();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
/**
* MaterializedViewProjectScanRule
*/
public class MaterializedViewProjectScanRule extends MaterializedViewScanRule {
public class MaterializedViewProjectScanRule extends AbstractMaterializedViewScanRule {

public static final MaterializedViewProjectScanRule INSTANCE = new MaterializedViewProjectScanRule();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@
package org.apache.doris.nereids.trees.plans.logical;

import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DataTrait;
import org.apache.doris.nereids.properties.LogicalProperties;
Expand Down Expand Up @@ -47,6 +49,7 @@
import org.json.JSONObject;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
Expand Down Expand Up @@ -495,7 +498,16 @@ AGGREGATE KEY (siteid,citycode,username)
return;
}
Set<Slot> outputSet = Utils.fastToImmutableSet(getOutputSet());
if (getTable().getKeysType().isAggregationFamily()) {
if (getTable() instanceof MTMV) {
MTMV mtmv = (MTMV) getTable();
MTMVCache cache = mtmv.getCache();
if (cache == null) {
return;
}
Plan originalPlan = cache.getOriginalPlan();
builder.addUniqueSlot(originalPlan.getLogicalProperties().getTrait());
builder.replaceUniqueBy(constructReplaceMap(mtmv));
} else if (getTable().getKeysType().isAggregationFamily()) {
ImmutableSet.Builder<Slot> uniqSlots = ImmutableSet.builderWithExpectedSize(outputSet.size());
for (Slot slot : outputSet) {
if (!(slot instanceof SlotReference)) {
Expand All @@ -509,4 +521,58 @@ AGGREGATE KEY (siteid,citycode,username)
builder.addUniqueSlot(uniqSlots.build());
}
}

@Override
public void computeUniform(DataTrait.Builder builder) {
if (getTable() instanceof MTMV) {
MTMV mtmv = (MTMV) getTable();
MTMVCache cache = mtmv.getCache();
if (cache == null) {
return;
}
Plan originalPlan = cache.getOriginalPlan();
builder.addUniformSlot(originalPlan.getLogicalProperties().getTrait());
builder.replaceUniformBy(constructReplaceMap(mtmv));
}
}

@Override
public void computeEqualSet(DataTrait.Builder builder) {
if (getTable() instanceof MTMV && getTable().getName().equals("mv1")) {
System.out.println();
}
if (getTable() instanceof MTMV) {
MTMV mtmv = (MTMV) getTable();
MTMVCache cache = mtmv.getCache();
if (cache == null) {
return;
}
Plan originalPlan = cache.getOriginalPlan();
builder.addEqualSet(originalPlan.getLogicalProperties().getTrait());
builder.replaceEqualSetBy(constructReplaceMap(mtmv));
}
}

@Override
public void computeFd(DataTrait.Builder builder) {
if (getTable() instanceof MTMV) {
MTMV mtmv = (MTMV) getTable();
MTMVCache cache = mtmv.getCache();
if (cache == null) {
return;
}
Plan originalPlan = cache.getOriginalPlan();
builder.addFuncDepsDG(originalPlan.getLogicalProperties().getTrait());
builder.replaceFuncDepsBy(constructReplaceMap(mtmv));
}
}

Map<Slot, Slot> constructReplaceMap(MTMV mtmv) {
Map<Slot, Slot> replaceMap = new HashMap<>();
List<Slot> originOutputs = mtmv.getCache().getOriginalPlan().getOutput();
for (int i = 0; i < getOutput().size(); i++) {
replaceMap.put(originOutputs.get(i), getOutput().get(i));
}
return replaceMap;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -839,23 +839,22 @@ suite("nested_mtmv") {
}
compare_res(sql_2 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")

// tmp and will fix soon
// explain {
// sql("${sql_3}")
// contains "${mv_3}(${mv_3})"
// }
// compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
//
// explain {
// sql("${sql_4}")
// contains "${mv_4}(${mv_4})"
// }
// compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
//
// explain {
// sql("${sql_5}")
// contains "${mv_5}(${mv_5})"
// }
// compare_res(sql_5 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
explain {
sql("${sql_3}")
contains "${mv_3}(${mv_3})"
}
compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")

explain {
sql("${sql_4}")
contains "${mv_4}(${mv_4})"
}
compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")

explain {
sql("${sql_5}")
contains "${mv_5}(${mv_5})"
}
compare_res(sql_5 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")

}
Loading