Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix](mtmv) Choose a valid partition column when there are both valid and invalid expressions #38367

Merged
merged 4 commits into from
Jul 31, 2024
Merged

[fix](mtmv) Choose a valid partition column when there are both valid and invalid expressions #38367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c34de35
[fix] (mtmv) Choose a valid partition column when there are both vali…
Jul 25, 2024
efe296e
add ut
Jul 29, 2024
ce8ccce
fix ut
Jul 29, 2024
3b2260f
small fix
Jul 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions ...re/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -589,7 +589,8 @@ private SlotReference getContextPartitionColumn(IncrementCheckerContext context)
private static boolean checkPartition(Collection<? extends Expression> expressionsToCheck, Plan plan,
IncrementCheckerContext context) {
NamedExpression partitionColumn = context.getMvPartitionColumn();
for (Expression projectSlot : expressionsToCheck) {

OUTER_CHECK: for (Expression projectSlot : expressionsToCheck) {
if (projectSlot.isColumnFromTable() && projectSlot.equals(partitionColumn.toSlot())) {
continue;
}
Expand Down Expand Up @@ -621,7 +622,7 @@ private static boolean checkPartition(Collection<? extends Expression> expressio
String.format("partition expression use more than one slot reference, invalid "
+ "expressionToCheckColumns is %s, partitionColumnDateColumns is %s",
expressionToCheckColumns, partitionColumns));
return false;
continue;
}
List<Expression> expressions = expressionToCheck.collectToList(Expression.class::isInstance);
for (Expression expression : expressions) {
Expand All @@ -630,7 +631,7 @@ private static boolean checkPartition(Collection<? extends Expression> expressio
context.addFailReason(
String.format("column to check use invalid implicit expression, invalid "
+ "expression is %s", expression));
return false;
continue OUTER_CHECK;
}
}
List<Expression> partitionExpressions = partitionExpression.collectToList(
Expand All @@ -641,7 +642,7 @@ private static boolean checkPartition(Collection<? extends Expression> expressio
context.addFailReason(
String.format("partition column use invalid implicit expression, invalid "
+ "expression is %s", expression));
return false;
continue OUTER_CHECK;
}
}
List<DateTrunc> expressionToCheckDataTruncList =
Expand All @@ -652,16 +653,17 @@ private static boolean checkPartition(Collection<? extends Expression> expressio
// mv time unit level is little then query
context.addFailReason("partition column time unit level should be "
+ "greater than sql select column");
return false;
continue;
}
if (!partitionColumn.isColumnFromTable()) {
context.setMvPartitionColumn(partitionColumns.iterator().next());
}
if (!context.getPartitionExpression().isPresent()) {
context.setPartitionExpression(partitionExpression);
}
return true;
}
return true;
return context.getMvPartitionColumn().isColumnFromTable();
}
}

Expand Down
49 changes: 49 additions & 0 deletions ...rc/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,19 @@ protected void runBeforeAll() throws Exception {
+ " \"replication_num\" = \"1\"\n"
+ ");\n"
);
createTable("CREATE TABLE `test3` (\n"
+ " `id` VARCHAR(36) NOT NULL COMMENT 'id',\n"
+ " `created_time` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT ''\n"
+ ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`id`)\n"
+ "COMMENT ''\n"
+ "PARTITION BY RANGE(`created_time`)\n"
+ "(PARTITION P_2024071713 VALUES [('2024-07-17 13:00:00'), ('2024-07-17 14:00:00')),\n"
+ "PARTITION P_2024071714 VALUES [('2024-07-17 14:00:00'), ('2024-07-17 15:00:00')))\n"
+ "DISTRIBUTED BY HASH(`id`) BUCKETS AUTO\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\"\n"
+ ");\n");
// Should not make scan to empty relation when the table used by materialized view has no data
connectContext.getSessionVariable().setDisableNereidsRules("OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION");
}
Expand Down Expand Up @@ -809,6 +822,42 @@ public void containTableQueryOperatorWithoutOperatorTest() {
});
}

@Test
public void getRelatedTableInfoWhenMultiPartitionExprs() {
PlanChecker.from(connectContext)
.checkExplain("select id, date_trunc(created_time, 'minute') as created_time_minute,"
+ " min(created_time) as start_time,"
+ " if(count(id) > 0, 1, 0) as status\n"
+ " from test3 \n"
+ " group by id, date_trunc(created_time, 'minute')",
nereidsPlanner -> {
Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan();
RelatedTableInfo relatedTableInfo =
MaterializedViewUtils.getRelatedTableInfo("created_time_minute",
"day", rewrittenPlan, nereidsPlanner.getCascadesContext());
checkRelatedTableInfo(relatedTableInfo,
"test3",
"created_time",
true);
});
PlanChecker.from(connectContext)
.checkExplain("select id, date_trunc(created_time, 'hour') as created_time_hour,"
+ " min(created_time) as start_time\n"
+ " from test3 \n"
+ " group by id, date_trunc(created_time, 'minute'),"
+ " date_trunc(created_time, 'hour');",
nereidsPlanner -> {
Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan();
RelatedTableInfo relatedTableInfo =
MaterializedViewUtils.getRelatedTableInfo("created_time_hour",
"day", rewrittenPlan, nereidsPlanner.getCascadesContext());
checkRelatedTableInfo(relatedTableInfo,
"test3",
"created_time",
true);
});
}

private void checkRelatedTableInfo(RelatedTableInfo relatedTableInfo,
String expectTableName,
String expectColumnName,
Expand Down
Loading