From 4d67c29ec5d0443bf01b2d8d1d9728eb3dc63be2 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:57:37 +0800 Subject: [PATCH] [opt](mtmv) Materialized view partition track supports date_trunc and optimize the fail reason (#35562) this depends on #34781 1. Materialized view partition track supports date_trunc and optimize the fail reason. 2. it supports create partition mv as following: this mv will be partition updated by day CREATE MATERIALIZED VIEW mv_6 BUILD IMMEDIATE REFRESH AUTO ON MANUAL partition by(date_trunc(date_alias, 'day')) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS SELECT date_trunc(t1.L_SHIPDATE, 'hour') as date_alias, t2.O_ORDERDATE, t1.L_QUANTITY, t2.O_ORDERSTATUS, count(distinct case when t1.L_SUPPKEY > 0 then t2.O_ORDERSTATUS else null end) as cnt_1 from (select * from lineitem where L_SHIPDATE in ('2017-01-30')) t1 left join (select * from orders where O_ORDERDATE in ('2017-01-30')) t2 on t1.L_ORDERKEY = t2.O_ORDERKEY group by t1.L_SHIPDATE, t2.O_ORDERDATE, t1.L_QUANTITY, t2.O_ORDERSTATUS; --- .../apache/doris/mtmv/MTMVPartitionInfo.java | 6 - .../mv/AsyncMaterializationContext.java | 6 +- .../exploration/mv/MaterializedViewUtils.java | 221 ++++++++++-- .../mv/rollup/AggFunctionRollUpHandler.java | 5 +- .../expression/ExpressionNormalization.java | 2 + .../expression/rules/MergeDateTrunc.java | 78 +++++ .../apache/doris/nereids/trees/TreeNode.java | 4 +- .../trees/expressions/literal/Interval.java | 37 +- .../info/MTMVPartitionDefinition.java | 107 +++--- .../mv/MaterializedViewUtilsTest.java | 172 ++++++--- .../ExpressionRewriteTestHelper.java | 6 + .../rules/expression/MergeDateTruncTest.java | 57 +++ .../mtmv_p0/test_rollup_partition_mtmv.out | 22 ++ .../mtmv_p0/test_rollup_partition_mtmv.groovy | 325 ++++++++++++++++++ 14 files changed, 900 insertions(+), 148 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MergeDateTrunc.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/MergeDateTruncTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java index 7ca1b7e3e63748..ff4060f334a952 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java @@ -21,8 +21,6 @@ import org.apache.doris.catalog.Column; import org.apache.doris.common.AnalysisException; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSortedSet; import com.google.gson.annotations.SerializedName; import java.util.List; @@ -38,10 +36,6 @@ public enum MTMVPartitionType { SELF_MANAGE } - public static final ImmutableSet MTMV_PARTITION_FUNCTIONS = new ImmutableSortedSet.Builder( - String.CASE_INSENSITIVE_ORDER).add("date_trunc") - .build(); - @SerializedName("pt") private MTMVPartitionType partitionType; @SerializedName("rt") diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java index 9776673de793d9..d830e9d41cbdee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java @@ -104,9 +104,9 @@ public Optional> getPlanStatistics(CascadesContext cascades return Optional.empty(); } RelationId relationId = null; - List logicalOlapScan = this.getScanPlan().collectFirst(LogicalOlapScan.class::isInstance); - if (!logicalOlapScan.isEmpty()) { - relationId = logicalOlapScan.get(0).getRelationId(); + Optional logicalOlapScan = this.getScanPlan().collectFirst(LogicalOlapScan.class::isInstance); + if (logicalOlapScan.isPresent()) { + relationId = logicalOlapScan.get().getRelationId(); } return Optional.of(Pair.of(relationId, normalizeStatisticsColumnExpression(mtmvCache.getStatistics()))); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 2e45fa44833276..3964dc00b7be3d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -27,12 +27,18 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.StructInfoMap; +import org.apache.doris.nereids.rules.expression.ExpressionNormalization; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PreAggStatus; @@ -48,10 +54,12 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; @@ -77,9 +85,11 @@ public class MaterializedViewUtils { * @param materializedViewPlan this should be rewritten or analyzed plan, should not be physical plan. * @param column ref column name. */ - public static Optional getRelatedTableInfo(String column, Plan materializedViewPlan) { + public static RelatedTableInfo getRelatedTableInfo(String column, String timeUnit, + Plan materializedViewPlan, CascadesContext cascadesContext) { + List outputExpressions = materializedViewPlan.getOutput(); - Slot columnExpr = null; + NamedExpression columnExpr = null; // get column slot for (Slot outputSlot : outputExpressions) { if (outputSlot.getName().equalsIgnoreCase(column)) { @@ -88,14 +98,12 @@ public static Optional getRelatedTableInfo(String column, Plan } } if (columnExpr == null) { - return Optional.empty(); - } - if (!(columnExpr instanceof SlotReference)) { - return Optional.empty(); + return RelatedTableInfo.failWith("partition column can not find from sql select column"); } - SlotReference columnSlot = (SlotReference) columnExpr; - if (!columnSlot.isColumnFromTable()) { - return Optional.empty(); + if (timeUnit != null) { + Expression dateTrunc = new DateTrunc(columnExpr, new VarcharLiteral(timeUnit)); + columnExpr = new Alias(dateTrunc); + materializedViewPlan = new LogicalProject<>(ImmutableList.of(columnExpr), materializedViewPlan); } // Collect table relation map which is used to identify self join List catalogRelationObjs = @@ -107,19 +115,20 @@ public static Optional getRelatedTableInfo(String column, Plan } // Check sql pattern IncrementCheckerContext checkContext = - new IncrementCheckerContext(columnSlot, tableCatalogRelationMultimapBuilder.build()); + new IncrementCheckerContext(columnExpr, tableCatalogRelationMultimapBuilder.build(), cascadesContext); materializedViewPlan.accept(MaterializedViewIncrementChecker.INSTANCE, checkContext); Multimap partitionRelatedTableAndColumnMap = checkContext.getPartitionRelatedTableAndColumnMap(); if (partitionRelatedTableAndColumnMap.isEmpty()) { - return Optional.empty(); + return RelatedTableInfo.failWith(String.format("can't not find valid partition track column, because %s", + String.join(",", checkContext.getFailReasons()))); } // TODO support to return only one related table info, support multi later for (Map.Entry entry : partitionRelatedTableAndColumnMap.entries()) { - return Optional.of(new RelatedTableInfo(new BaseTableInfo(entry.getKey()), true, - entry.getValue().getName())); + return RelatedTableInfo.successWith(new BaseTableInfo(entry.getKey()), true, + entry.getValue().getName(), checkContext.getPartitionExpression().orElseGet(() -> null)); } - return Optional.empty(); + return RelatedTableInfo.failWith("can't not find valid partition track column finally"); } /** @@ -288,10 +297,63 @@ private static final class MaterializedViewIncrementChecker extends DefaultPlanVisitor { public static final MaterializedViewIncrementChecker INSTANCE = new MaterializedViewIncrementChecker(); + public static final Set> SUPPORT_EXPRESSION_TYPES = + ImmutableSet.of(DateTrunc.class, SlotReference.class, Literal.class); @Override public Void visitLogicalProject(LogicalProject project, IncrementCheckerContext context) { - return visit(project, context); + NamedExpression mvPartitionColumn = context.getMvPartitionColumn(); + List output = project.getOutput(); + if (context.getMvPartitionColumn().isColumnFromTable()) { + return visit(project, context); + } + for (Slot projectSlot : output) { + if (!projectSlot.equals(mvPartitionColumn.toSlot())) { + continue; + } + if (projectSlot.isColumnFromTable()) { + context.setMvPartitionColumn(projectSlot); + } else { + // should be only use date_trunc + Expression shuttledExpression = + ExpressionUtils.shuttleExpressionWithLineage(projectSlot, project, new BitSet()); + // merge date_trunc + shuttledExpression = new ExpressionNormalization().rewrite(shuttledExpression, + new ExpressionRewriteContext(context.getCascadesContext())); + + List expressions = shuttledExpression.collectToList(Expression.class::isInstance); + for (Expression expression : expressions) { + if (SUPPORT_EXPRESSION_TYPES.stream().noneMatch( + supportExpression -> supportExpression.isAssignableFrom(expression.getClass()))) { + context.addFailReason( + String.format("partition column use invalid implicit expression, invalid " + + "expression is %s", expression)); + return null; + } + } + List dataTruncExpressions = + shuttledExpression.collectToList(DateTrunc.class::isInstance); + if (dataTruncExpressions.size() != 1) { + // mv time unit level is little then query + context.addFailReason("partition column time unit level should be " + + "greater than sql select column"); + return null; + } + Optional columnExpr = + shuttledExpression.getArgument(0).collectFirst(Slot.class::isInstance); + if (!columnExpr.isPresent() || !columnExpr.get().isColumnFromTable()) { + context.addFailReason(String.format("partition reference column should be direct column " + + "rather then expression except date_trunc, columnExpr is %s", columnExpr)); + return null; + } + context.setPartitionExpression(shuttledExpression); + context.setMvPartitionColumn(columnExpr.get()); + } + return visit(project, context); + } + context.addFailReason(String.format("partition reference column should be direct column " + + "rather then expression except date_trunc, current project is %s", project)); + return null; } @Override @@ -303,6 +365,7 @@ public Void visitLogicalFilter(LogicalFilter filter, IncrementCh public Void visitLogicalJoin(LogicalJoin join, IncrementCheckerContext context) { if (join.isMarkJoin()) { + context.addFailReason("partition track doesn't support mark join"); return null; } Plan left = join.child(0); @@ -311,7 +374,11 @@ public Void visitLogicalJoin(LogicalJoin join, && slot.isColumnFromTable()) .map(slot -> ((SlotReference) slot).getColumn().get()) .collect(Collectors.toSet()); - boolean useLeft = leftColumnSet.contains(context.getMvPartitionColumn().getColumn().get()); + SlotReference contextPartitionColumn = getContextPartitionColumn(context); + if (contextPartitionColumn == null) { + return null; + } + boolean useLeft = leftColumnSet.contains(contextPartitionColumn.getColumn().get()); JoinType joinType = join.getJoinType(); if (joinType.isInnerJoin() || joinType.isCrossJoin()) { return visit(join, context); @@ -324,12 +391,16 @@ public Void visitLogicalJoin(LogicalJoin join, || joinType.isRightSemiJoin()) && !useLeft) { return visit(join.right(), context); } + context.addFailReason(String.format("partition column is in un supported join null generate side, " + + "current join type is %s", joinType)); return null; } @Override public Void visitLogicalRelation(LogicalRelation relation, IncrementCheckerContext context) { if (!(relation instanceof LogicalCatalogRelation)) { + context.addFailReason(String.format("relation should be LogicalCatalogRelation, " + + "but now is %s", relation.getClass().getSimpleName())); return null; } LogicalCatalogRelation logicalCatalogRelation = (LogicalCatalogRelation) relation; @@ -337,25 +408,41 @@ public Void visitLogicalRelation(LogicalRelation relation, IncrementCheckerConte // if self join, self join can not partition track now, remove the partition column correspondingly if (context.getRelationByTable(table).size() > 1) { context.getPartitionRelatedTableAndColumnMap().removeAll(table); + context.addFailReason(String.format("self join doesn't support partition update, " + + "self join table name is %s", table.getName())); return null; } // TODO: 2024/1/31 support only one partition referenced column, support multi later if (!context.getPartitionRelatedTableAndColumnMap().isEmpty()) { + context.addFailReason(String.format("partition track already has an related base table column," + + "track info is %s", context.getPartitionRelatedTableAndColumnMap())); return null; } if (!(table instanceof MTMVRelatedTableIf)) { + context.addFailReason(String.format("relation base table is not MTMVRelatedTableIf, the table is %s", + table.getName())); return null; } MTMVRelatedTableIf relatedTable = (MTMVRelatedTableIf) table; PartitionType type = relatedTable.getPartitionType(); if (PartitionType.UNPARTITIONED.equals(type)) { + context.addFailReason(String.format("related base table is not partition table, the table is %s", + table.getName())); return null; } Set partitionColumnSet = new HashSet<>(relatedTable.getPartitionColumns()); - Column mvReferenceColumn = context.getMvPartitionColumn().getColumn().get(); + SlotReference contextPartitionColumn = getContextPartitionColumn(context); + if (contextPartitionColumn == null) { + return null; + } + Column mvReferenceColumn = contextPartitionColumn.getColumn().get(); if (partitionColumnSet.contains(mvReferenceColumn) && (!mvReferenceColumn.isAllowNull() || relatedTable.isPartitionColumnAllowNull())) { context.addTableColumn(table, mvReferenceColumn); + } else { + context.addFailReason(String.format("related base table partition column doesn't contain the mv" + + " partition or partition nullable check fail, the mvReferenceColumn is %s", + mvReferenceColumn)); } return visit(relation, context); } @@ -365,6 +452,7 @@ public Void visitLogicalAggregate(LogicalAggregate aggregate, IncrementCheckerContext context) { Set groupByExprSet = new HashSet<>(aggregate.getGroupByExpressions()); if (groupByExprSet.isEmpty()) { + context.addFailReason("group by sets is empty, doesn't contain the target partition"); return null; } Set originalGroupbyExprSet = new HashSet<>(); @@ -373,7 +461,12 @@ public Void visitLogicalAggregate(LogicalAggregate aggregate, originalGroupbyExprSet.add(((SlotReference) groupExpr).getColumn().get()); } }); - if (!originalGroupbyExprSet.contains(context.getMvPartitionColumn().getColumn().get())) { + SlotReference contextPartitionColumn = getContextPartitionColumn(context); + if (contextPartitionColumn == null) { + return null; + } + if (!originalGroupbyExprSet.contains(contextPartitionColumn.getColumn().get())) { + context.addFailReason("group by sets doesn't contain the target partition"); return null; } return visit(aggregate, context); @@ -387,6 +480,7 @@ public Void visitLogicalWindow(LogicalWindow window, IncrementCh } for (NamedExpression namedExpression : windowExpressions) { if (!checkWindowPartition(namedExpression, context)) { + context.addFailReason("window partition sets doesn't contain the target partition"); return null; } } @@ -419,29 +513,52 @@ private boolean checkWindowPartition(Expression expression, IncrementCheckerCont originalPartitionbyExprSet.add(((SlotReference) groupExpr).getColumn().get()); } }); - if (!originalPartitionbyExprSet.contains(context.getMvPartitionColumn().getColumn().get())) { + SlotReference contextPartitionColumn = getContextPartitionColumn(context); + if (contextPartitionColumn == null) { + return false; + } + if (!originalPartitionbyExprSet.contains(contextPartitionColumn.getColumn().get())) { return false; } } return true; } + + private SlotReference getContextPartitionColumn(IncrementCheckerContext context) { + if (!context.getMvPartitionColumn().isColumnFromTable()) { + context.addFailReason(String.format("context partition column should be slot from column, " + + "context column is %s", + context.getMvPartitionColumn())); + return null; + } + return (SlotReference) context.getMvPartitionColumn(); + } } private static final class IncrementCheckerContext { - private final SlotReference mvPartitionColumn; + private NamedExpression mvPartitionColumn; + private Optional partitionExpression = Optional.empty(); private final Multimap tableAndCatalogRelationMap; private final Multimap partitionRelatedTableAndColumnMap = HashMultimap.create(); + private final Set failReasons = new HashSet<>(); + private final CascadesContext cascadesContext; - public IncrementCheckerContext(SlotReference mvPartitionColumn, - Multimap tableAndCatalogRelationMap) { + public IncrementCheckerContext(NamedExpression mvPartitionColumn, + Multimap tableAndCatalogRelationMap, + CascadesContext cascadesContext) { this.mvPartitionColumn = mvPartitionColumn; this.tableAndCatalogRelationMap = tableAndCatalogRelationMap; + this.cascadesContext = cascadesContext; } - public SlotReference getMvPartitionColumn() { + public NamedExpression getMvPartitionColumn() { return mvPartitionColumn; } + public void setMvPartitionColumn(NamedExpression mvPartitionColumn) { + this.mvPartitionColumn = mvPartitionColumn; + } + public void addTableColumn(TableIf relatedTable, Column partitionColumn) { partitionRelatedTableAndColumnMap.put(relatedTable, partitionColumn); } @@ -457,20 +574,56 @@ public Collection getRelationByTable(TableIf tableIf) { public void addTableAndRelation(TableIf tableIf, CatalogRelation relation) { tableAndCatalogRelationMap.put(new TableIdentifier(tableIf), relation); } + + public Set getFailReasons() { + return failReasons; + } + + public void addFailReason(String failReason) { + this.failReasons.add(failReason); + } + + public CascadesContext getCascadesContext() { + return cascadesContext; + } + + public Optional getPartitionExpression() { + return partitionExpression; + } + + public void setPartitionExpression(Expression partitionExpression) { + this.partitionExpression = Optional.ofNullable(partitionExpression); + } } /** * The related table info that mv relate */ public static final class RelatedTableInfo { - private BaseTableInfo tableInfo; - private boolean pctPossible; - private String column; - - public RelatedTableInfo(BaseTableInfo tableInfo, boolean pctPossible, String column) { + private final BaseTableInfo tableInfo; + private final boolean pctPossible; + private final String column; + private final Set failReasons = new HashSet<>(); + // This records the partition expression if exist + private Optional partitionExpression; + + public RelatedTableInfo(BaseTableInfo tableInfo, boolean pctPossible, String column, String failReason, + Expression partitionExpression) { this.tableInfo = tableInfo; this.pctPossible = pctPossible; this.column = column; + this.failReasons.add(failReason); + this.partitionExpression = Optional.ofNullable(partitionExpression); + } + + public static RelatedTableInfo failWith(String failReason) { + return new RelatedTableInfo(null, false, null, failReason, + null); + } + + public static RelatedTableInfo successWith(BaseTableInfo tableInfo, boolean pctPossible, String column, + Expression partitionExpression) { + return new RelatedTableInfo(tableInfo, pctPossible, column, "", partitionExpression); } public BaseTableInfo getTableInfo() { @@ -484,5 +637,17 @@ public boolean isPctPossible() { public String getColumn() { return column; } + + public void addFailReason(String failReason) { + this.failReasons.add(failReason); + } + + public String getFailReason() { + return String.join(",", failReasons); + } + + public Optional getPartitionExpression() { + return partitionExpression; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java index 97229af62c3382..250d8a83c26621 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java @@ -24,6 +24,8 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.RollUpTrait; +import com.google.common.collect.ImmutableList; + import java.util.List; import java.util.Set; @@ -63,7 +65,8 @@ public abstract Function doRollup( protected static List extractArguments(Expression functionWithAny, Function actualFunction) { Set exprSetToRemove = functionWithAny.collectToSet(expr -> !(expr instanceof Any)); return actualFunction.collectFirst(expr -> - exprSetToRemove.stream().noneMatch(exprToRemove -> exprToRemove.equals(expr))); + exprSetToRemove.stream().noneMatch(exprToRemove -> exprToRemove.equals(expr))) + .map(expr -> ImmutableList.of((Expression) expr)).orElse(ImmutableList.of()); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java index f63ab2eee5f942..0e52f2aaaf6527 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule; import org.apache.doris.nereids.rules.expression.rules.InPredicateDedup; import org.apache.doris.nereids.rules.expression.rules.InPredicateToEqualToRule; +import org.apache.doris.nereids.rules.expression.rules.MergeDateTrunc; import org.apache.doris.nereids.rules.expression.rules.NormalizeBinaryPredicatesRule; import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticComparisonRule; import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticRule; @@ -53,6 +54,7 @@ public class ExpressionNormalization extends ExpressionRewrite { DigitalMaskingConvert.INSTANCE, SimplifyArithmeticComparisonRule.INSTANCE, ConvertAggStateCast.INSTANCE, + MergeDateTrunc.INSTANCE, CheckCast.INSTANCE ) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MergeDateTrunc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MergeDateTrunc.java new file mode 100644 index 00000000000000..892724cf8932fb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MergeDateTrunc.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression.rules; + +import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher; +import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.literal.Interval.TimeUnit; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Optional; + +/** + * Rewrite rule to convert + * For example: + * date_trunc(date_trunc(data_slot, 'hour'), 'day') -> date_trunc(data_slot, 'day') + */ +public class MergeDateTrunc implements ExpressionPatternRuleFactory { + + public static MergeDateTrunc INSTANCE = new MergeDateTrunc(); + public static ImmutableSet UN_SUPPORT_TIME_UNIT = ImmutableSet.of(TimeUnit.WEEK, TimeUnit.QUARTER); + + @Override + public List> buildRules() { + return ImmutableList.of( + matchesTopType(DateTrunc.class) + .when(dateTrunc -> dateTrunc.getArgument(0) instanceof DateTrunc) + .then(MergeDateTrunc::rewrite) + ); + } + + private static Expression rewrite(DateTrunc dateTrunc) { + + Expression parentTimeUnitArgument = dateTrunc.getArgument(1); + if (!(parentTimeUnitArgument instanceof Literal)) { + return dateTrunc; + } + Optional parentTimeUnit = TimeUnit.of(((Literal) parentTimeUnitArgument).getStringValue()); + DateTrunc childDateTrunc = (DateTrunc) dateTrunc.getArgument(0); + Expression childTimeUnitArgument = childDateTrunc.getArgument(1); + if (!(childTimeUnitArgument instanceof Literal)) { + return dateTrunc; + } + Optional childTimeUnit = TimeUnit.of(((Literal) childTimeUnitArgument).getStringValue()); + if (!parentTimeUnit.isPresent() || !childTimeUnit.isPresent()) { + return dateTrunc; + } + if (UN_SUPPORT_TIME_UNIT.contains(parentTimeUnit.get()) + || UN_SUPPORT_TIME_UNIT.contains(childTimeUnit.get())) { + return dateTrunc; + } + if (parentTimeUnit.get().getLevel() < childTimeUnit.get().getLevel()) { + return dateTrunc; + } + return new DateTrunc(childDateTrunc.getArgument(0), new VarcharLiteral(parentTimeUnit.get().toString())); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 6d1a298eb79fe2..24032d95fc6ba6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -271,7 +271,7 @@ default Set collectToSet(Predicate> predicate) { /** * Collect the nodes that satisfied the predicate firstly. */ - default List collectFirst(Predicate> predicate) { + default Optional collectFirst(Predicate> predicate) { List> result = new ArrayList<>(); foreach(node -> { if (result.isEmpty() && predicate.test(node)) { @@ -279,7 +279,7 @@ default List collectFirst(Predicate> predicate) { } return !result.isEmpty(); }); - return (List) ImmutableList.copyOf(result); + return result.isEmpty() ? Optional.empty() : Optional.of((T) result.get(0)); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java index 5cffc0c7fff7f5..454001fb3f1187 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java @@ -25,6 +25,9 @@ import org.apache.doris.nereids.types.DateType; import com.google.common.collect.ImmutableList; +import org.apache.commons.lang3.EnumUtils; + +import java.util.Optional; /** * Interval for timestamp calculation. @@ -61,30 +64,46 @@ public R accept(ExpressionVisitor visitor, C context) { * Supported time unit. */ public enum TimeUnit { - YEAR("YEAR", false), - MONTH("MONTH", false), - WEEK("WEEK", false), - DAY("DAY", false), - HOUR("HOUR", true), - MINUTE("MINUTE", true), - SECOND("SECOND", true); + YEAR("YEAR", false, 800), + MONTH("MONTH", false, 700), + QUARTER("QUARTER", false, 600), + WEEK("WEEK", false, 500), + DAY("DAY", false, 400), + HOUR("HOUR", true, 300), + MINUTE("MINUTE", true, 200), + SECOND("SECOND", true, 100); private final String description; - private final boolean isDateTimeUnit; + /** + * Time unit level, second level is low, year level is high + */ + private final int level; - TimeUnit(String description, boolean isDateTimeUnit) { + TimeUnit(String description, boolean isDateTimeUnit, int level) { this.description = description; this.isDateTimeUnit = isDateTimeUnit; + this.level = level; } public boolean isDateTimeUnit() { return isDateTimeUnit; } + public int getLevel() { + return level; + } + @Override public String toString() { return description; } + + /** + * Construct time unit by name + */ + public static Optional of(String name) { + return Optional.ofNullable(EnumUtils.getEnumIgnoreCase(TimeUnit.class, name)); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java index 09b30063b9de06..2c5cbb4649e561 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java @@ -40,7 +40,10 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; @@ -51,7 +54,6 @@ import com.google.common.collect.Sets; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -59,6 +61,7 @@ * MTMVPartitionDefinition */ public class MTMVPartitionDefinition { + public static final String PARTITION_BY_FUNCTION_NAME = "date_trunc"; private MTMVPartitionType partitionType; private String partitionCol; private Expression functionCallExpression; @@ -78,25 +81,39 @@ public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner pl return mtmvPartitionInfo; } String partitionColName; + String timeUnit; if (this.partitionType == MTMVPartitionType.EXPR) { - Expr expr; - if (functionCallExpression instanceof UnboundFunction) { - UnboundFunction function = (UnboundFunction) functionCallExpression; - expr = new FunctionCallExpr(function.getName(), - new FunctionParams(convertToLegacyArguments(function.children()))); + String functionName = ((UnboundFunction) functionCallExpression).getName(); + if (functionCallExpression instanceof UnboundFunction + && functionName.equalsIgnoreCase(PARTITION_BY_FUNCTION_NAME)) { + partitionColName = functionCallExpression.getArgument(0) instanceof UnboundSlot + ? ((UnboundSlot) functionCallExpression.getArgument(0)).getName() : null; + timeUnit = functionCallExpression.getArguments().get(1).isLiteral() + ? ((Literal) functionCallExpression.getArgument(1)).getStringValue() : null; } else { throw new AnalysisException( "unsupported auto partition expr " + functionCallExpression.toString()); } - partitionColName = getColNameFromExpr(expr); - mtmvPartitionInfo.setExpr(expr); } else { partitionColName = this.partitionCol; + timeUnit = null; } mtmvPartitionInfo.setPartitionCol(partitionColName); - RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, logicalQuery, partitionColName); + RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, logicalQuery, partitionColName, timeUnit); mtmvPartitionInfo.setRelatedCol(relatedTableInfo.getColumn()); mtmvPartitionInfo.setRelatedTable(relatedTableInfo.getTableInfo()); + if (relatedTableInfo.getPartitionExpression().isPresent()) { + // Set mv partition expr by relatedTableInfo, this is used for partition rollup and so on + if (relatedTableInfo.getPartitionExpression().get().getExpressionName() + .equalsIgnoreCase(PARTITION_BY_FUNCTION_NAME)) { + DateTrunc dateTrunc = (DateTrunc) relatedTableInfo.getPartitionExpression().get(); + // todo use new expression? + mtmvPartitionInfo.setExpr(new FunctionCallExpr(dateTrunc.getName(), + new FunctionParams(convertToLegacyArguments(dateTrunc.children())))); + mtmvPartitionInfo.setPartitionType(MTMVPartitionType.EXPR); + this.partitionType = MTMVPartitionType.EXPR; + } + } if (this.partitionType == MTMVPartitionType.EXPR) { try { MTMVPartitionExprFactory.getExprService(mtmvPartitionInfo.getExpr()).analyze(mtmvPartitionInfo); @@ -107,38 +124,10 @@ public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner pl return mtmvPartitionInfo; } - /** - * getColNameFromExpr - * - * @param expr expr - * @return String - */ - public static String getColNameFromExpr(Expr expr) { - if (!(expr instanceof FunctionCallExpr)) { - throw new AnalysisException( - "auto create partition only support function call expr is: " - + MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS); - } - FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr; - List paramsExpr = functionCallExpr.getParams().exprs(); - String name = functionCallExpr.getFnName().getFunction(); - if (MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS.contains(name)) { - for (Expr param : paramsExpr) { - if (param instanceof SlotRef) { - return ((SlotRef) param).getColumnName(); - } - } - throw new AnalysisException("can not find colName"); - } else { - throw new AnalysisException( - "auto create partition only support function call expr is: " - + MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS); - } - } - private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectContext ctx, LogicalPlan logicalQuery, - String partitionColName) { + String partitionColName, + String timeUnit) { CascadesContext cascadesContext = planner.getCascadesContext(); SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable(); Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); @@ -149,12 +138,13 @@ private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectCont try { Plan mvRewrittenPlan = planner.plan(logicalQuery, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); - Optional relatedTableInfo = MaterializedViewUtils - .getRelatedTableInfo(partitionColName, mvRewrittenPlan); - if (!relatedTableInfo.isPresent() || !relatedTableInfo.get().isPctPossible()) { - throw new AnalysisException("Unable to find a suitable base table for partitioning"); + RelatedTableInfo relatedTableInfo = MaterializedViewUtils + .getRelatedTableInfo(partitionColName, timeUnit, mvRewrittenPlan, cascadesContext); + if (!relatedTableInfo.isPctPossible()) { + throw new AnalysisException(String.format("Unable to find a suitable base table for partitioning," + + " the fail reason is %s", relatedTableInfo.getFailReason())); } - MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.get().getTableInfo()); + MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo()); Set partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); try { partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames()); @@ -162,14 +152,14 @@ private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectCont throw new AnalysisException(e.getMessage(), e); } - if (!partitionColumnNames.contains(relatedTableInfo.get().getColumn())) { - throw new AnalysisException("error related column: " + relatedTableInfo.get().getColumn()); + if (!partitionColumnNames.contains(relatedTableInfo.getColumn())) { + throw new AnalysisException("error related column: " + relatedTableInfo.getColumn()); } if (!(mtmvBaseRealtedTable instanceof HMSExternalTable) && partitionColumnNames.size() != 1) { throw new AnalysisException("only hms table support multi column partition."); } - return relatedTableInfo.get(); + return relatedTableInfo; } finally { // after operate, roll back the disable rules sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); @@ -178,15 +168,20 @@ private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectCont } private static List convertToLegacyArguments(List children) { - return children.stream().map(child -> { - if (child instanceof UnboundSlot) { - return new SlotRef(null, ((UnboundSlot) child).getName()); - } else if (child instanceof Literal) { - return new StringLiteral(((Literal) child).getStringValue()); - } else { - throw new AnalysisException("unsupported argument " + child.toString()); - } - }).collect(Collectors.toList()); + return children.stream().map(MTMVPartitionDefinition::convertToLegacyRecursion).collect(Collectors.toList()); + } + + private static Expr convertToLegacyRecursion(Expression expression) { + if (expression instanceof Slot) { + return new SlotRef(null, ((Slot) expression).getName()); + } else if (expression instanceof Literal) { + return new StringLiteral(((Literal) expression).getStringValue()); + } else if (expression instanceof Cast) { + // mv partition roll up only need the slot in cast + return convertToLegacyRecursion(((Cast) expression).child()); + } else { + throw new AnalysisException("unsupported argument " + expression.toString()); + } } public MTMVPartitionType getPartitionType() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 1ed6b92129afba..3878ab742ca019 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -28,8 +28,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.Optional; - /** * Test for materialized view util */ @@ -251,8 +249,9 @@ public void getRelatedTableInfoWhenAutoPartitionTest() { + "on t1.l_orderkey = o_orderkey;", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("l_shipdate", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); checkRelatedTableInfo(relatedTableInfo, "lineitem_no_data", "L_SHIPDATE", @@ -277,8 +276,9 @@ public void getRelatedTableInfoTestWithoutGroupTest() { + "ON l.L_PARTKEY = ps.PS_PARTKEY and l.L_SUPPKEY = ps.PS_SUPPKEY", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("l_shipdate", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); checkRelatedTableInfo(relatedTableInfo, "lineitem", "L_SHIPDATE", @@ -303,9 +303,10 @@ public void getRelatedTableInfoTestWithoutGroupNullTest() { + "ON l.L_PARTKEY = ps.PS_PARTKEY and l.L_SUPPKEY = ps.PS_SUPPKEY", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan); - Assertions.assertTrue(relatedTableInfo.isPresent()); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("l_shipdate", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); + Assertions.assertTrue(relatedTableInfo.isPctPossible()); }); } @@ -326,8 +327,9 @@ public void getRelatedTableInfoTestWithSubqueryTest() { + "GROUP BY l.L_SHIPDATE, o.O_ORDERDATE ", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("ship_data_alias", rewrittenPlan); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("ship_data_alias", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); checkRelatedTableInfo(relatedTableInfo, "lineitem", "l_shipdate", @@ -356,8 +358,9 @@ public void getRelatedTableInfoTestWithAliasAndGroupTest() { + "t2.O_ORDERSTATUS;", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("l_shipdate", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); checkRelatedTableInfo(relatedTableInfo, "lineitem", "L_SHIPDATE", @@ -375,8 +378,9 @@ public void getRelatedTableInfoLeftAntiJoinTest() { + " group by l_shipdate, l_orderkey", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("l_orderkey", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); checkRelatedTableInfo(relatedTableInfo, "lineitem_list_partition", "l_orderkey", @@ -394,9 +398,12 @@ public void getRelatedTableInfoSelfJoinTest() { + " group by t1.l_shipdate, t1.l_orderkey, t1.l_partkey, t1.l_suppkey", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan); - Assertions.assertFalse(relatedTableInfo.isPresent()); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("l_orderkey", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); + Assertions.assertTrue(relatedTableInfo.getFailReason().contains( + "self join doesn't support partition update")); + Assertions.assertFalse(relatedTableInfo.isPctPossible()); }); PlanChecker.from(connectContext) @@ -407,9 +414,12 @@ public void getRelatedTableInfoSelfJoinTest() { + " group by t1.l_shipdate, t1.l_orderkey, t1.l_partkey, t1.l_suppkey", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan); - Assertions.assertFalse(relatedTableInfo.isPresent()); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("l_orderkey", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); + Assertions.assertTrue(relatedTableInfo.getFailReason().contains( + "self join doesn't support partition update")); + Assertions.assertFalse(relatedTableInfo.isPctPossible()); }); PlanChecker.from(connectContext) @@ -420,9 +430,12 @@ public void getRelatedTableInfoSelfJoinTest() { + " group by t1.l_shipdate, t1.l_orderkey, t1.l_partkey, t1.l_suppkey", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan); - Assertions.assertFalse(relatedTableInfo.isPresent()); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("l_orderkey", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); + Assertions.assertTrue(relatedTableInfo.getFailReason().contains( + "partition column is in un supported join null generate side")); + Assertions.assertFalse(relatedTableInfo.isPctPossible()); }); } @@ -447,8 +460,9 @@ public void getRelatedTableInfoUseRightTest() { + "t2.O_ORDERSTATUS;", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("o_orderdate", rewrittenPlan); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("o_orderdate", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); checkRelatedTableInfo(relatedTableInfo, "orders", "O_ORDERDATE", @@ -477,9 +491,12 @@ public void getRelatedTableInfoUseNullGenerateSideTest() { + "t2.O_ORDERSTATUS;", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("o_orderdate", rewrittenPlan); - Assertions.assertFalse(relatedTableInfo.isPresent()); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("o_orderdate", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); + Assertions.assertTrue(relatedTableInfo.getFailReason().contains( + "partition column is in un supported join null generate side")); + Assertions.assertFalse(relatedTableInfo.isPctPossible()); }); } @@ -494,9 +511,12 @@ public void getRelatedTableInfoTestWithoutPartitionTest() { + "ON ps_1.PS_PARTKEY = ps_2.PS_SUPPKEY ", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("PS_SUPPLYCOST", rewrittenPlan); - Assertions.assertFalse(relatedTableInfo.isPresent()); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("PS_SUPPLYCOST", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); + Assertions.assertTrue(relatedTableInfo.getFailReason().contains( + "self join doesn't support partition update")); + Assertions.assertFalse(relatedTableInfo.isPctPossible()); }); } @@ -517,8 +537,9 @@ public void getRelatedTableInfoTestWithWindowTest() { + "ON l.L_PARTKEY = ps.PS_PARTKEY and l.L_SUPPKEY = ps.PS_SUPPKEY", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("l_shipdate", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); checkRelatedTableInfo(relatedTableInfo, "lineitem", "L_SHIPDATE", @@ -543,9 +564,73 @@ public void getRelatedTableInfoTestWithWindowButNotPartitionTest() { + "ON l.L_PARTKEY = ps.PS_PARTKEY and l.L_SUPPKEY = ps.PS_SUPPKEY", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("L_SHIPDATE", rewrittenPlan); - Assertions.assertFalse(relatedTableInfo.isPresent()); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("L_SHIPDATE", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); + Assertions.assertTrue(relatedTableInfo.getFailReason().contains( + "window partition sets doesn't contain the target partition")); + Assertions.assertFalse(relatedTableInfo.isPctPossible()); + }); + } + + @Test + public void testPartitionDateTrunc() { + PlanChecker.from(connectContext) + .checkExplain("SELECT date_trunc(t1.L_SHIPDATE, 'hour') as date_alias, t2.O_ORDERDATE, t1.L_QUANTITY, t2.O_ORDERSTATUS, " + + "count(distinct case when t1.L_SUPPKEY > 0 then t2.O_ORDERSTATUS else null end) as cnt_1 " + + "from " + + " (select * from " + + " lineitem " + + " where L_SHIPDATE in ('2017-01-30')) t1 " + + "left join " + + " (select * from " + + " orders " + + " where O_ORDERDATE in ('2017-01-30')) t2 " + + "on t1.L_ORDERKEY = t2.O_ORDERKEY " + + "group by " + + "t1.L_SHIPDATE, " + + "t2.O_ORDERDATE, " + + "t1.L_QUANTITY, " + + "t2.O_ORDERSTATUS;", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("date_alias", "day", + rewrittenPlan, nereidsPlanner.getCascadesContext()); + checkRelatedTableInfo(relatedTableInfo, + "lineitem", + "L_SHIPDATE", + true); + }); + } + + @Test + public void testPartitionDateTruncShouldNotTrack() { + PlanChecker.from(connectContext) + .checkExplain("SELECT date_trunc(t1.L_SHIPDATE, 'day') as date_alias, t2.O_ORDERDATE, t1.L_QUANTITY, t2.O_ORDERSTATUS, " + + "count(distinct case when t1.L_SUPPKEY > 0 then t2.O_ORDERSTATUS else null end) as cnt_1 " + + "from " + + " (select * from " + + " lineitem " + + " where L_SHIPDATE in ('2017-01-30')) t1 " + + "left join " + + " (select * from " + + " orders " + + " where O_ORDERDATE in ('2017-01-30')) t2 " + + "on t1.L_ORDERKEY = t2.O_ORDERKEY " + + "group by " + + "t1.L_SHIPDATE, " + + "t2.O_ORDERDATE, " + + "t1.L_QUANTITY, " + + "t2.O_ORDERSTATUS;", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("date_alias", "hour", + rewrittenPlan, nereidsPlanner.getCascadesContext()); + Assertions.assertTrue(relatedTableInfo.getFailReason().contains( + "partition column time unit level should be greater than sql select column")); + Assertions.assertFalse(relatedTableInfo.isPctPossible()); }); } @@ -581,8 +666,9 @@ public void getRelatedTableInfoWhenMultiBaseTablePartition() { + ")t2 on t1.vin_type1 = t2.vin_type2;", nereidsPlanner -> { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); - Optional relatedTableInfo = - MaterializedViewUtils.getRelatedTableInfo("upgrade_day", rewrittenPlan); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("upgrade_day", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); checkRelatedTableInfo(relatedTableInfo, "test1", "upgrade_day", @@ -630,12 +716,12 @@ public void containTableQueryOperatorWithoutOperatorTest() { }); } - private void checkRelatedTableInfo(Optional relatedTableInfo, + private void checkRelatedTableInfo(RelatedTableInfo relatedTableInfo, String expectTableName, String expectColumnName, boolean pctPossible) { - Assertions.assertTrue(relatedTableInfo.isPresent()); - BaseTableInfo relatedBaseTableInfo = relatedTableInfo.get().getTableInfo(); + Assertions.assertNotNull(relatedTableInfo); + BaseTableInfo relatedBaseTableInfo = relatedTableInfo.getTableInfo(); try { TableIf tableIf = Env.getCurrentEnv().getCatalogMgr() .getCatalogOrAnalysisException(relatedBaseTableInfo.getCtlId()) @@ -645,7 +731,7 @@ private void checkRelatedTableInfo(Optional relatedTableInfo, } catch (Exception exception) { Assertions.fail(); } - Assertions.assertEquals(relatedTableInfo.get().getColumn().toLowerCase(), expectColumnName.toLowerCase()); + Assertions.assertEquals(relatedTableInfo.getColumn().toLowerCase(), expectColumnName.toLowerCase()); Assertions.assertTrue(pctPossible); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java index b252b4650f7315..2c7ef166610eb3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java @@ -78,6 +78,12 @@ protected void assertRewrite(Expression expression, Expression expectedExpressio Assertions.assertEquals(expectedExpression, rewrittenExpression); } + protected void assertNotRewrite(Expression expression, Expression expectedExpression) { + expression = typeCoercion(expression); + Expression rewrittenExpression = executor.rewrite(expression, context); + Assertions.assertNotEquals(expectedExpression, rewrittenExpression); + } + protected void assertRewriteAfterTypeCoercion(String expression, String expected) { Map mem = Maps.newHashMap(); Expression needRewriteExpression = PARSER.parseExpression(expression); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/MergeDateTruncTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/MergeDateTruncTest.java new file mode 100644 index 00000000000000..945c647fcb5d38 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/MergeDateTruncTest.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression; + +import org.apache.doris.nereids.rules.expression.rules.MergeDateTrunc; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; +import org.apache.doris.nereids.types.DateV2Type; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Test; + +/** + * merge date_trunc test + */ +public class MergeDateTruncTest extends ExpressionRewriteTestHelper { + + @Test + public void testMergeDateTrunc() { + executor = new ExpressionRuleExecutor(ImmutableList.of( + bottomUp(MergeDateTrunc.INSTANCE) + )); + Slot dataSlot = new SlotReference("data_slot", DateV2Type.INSTANCE); + assertRewrite(new DateTrunc(new DateTrunc(dataSlot, new VarcharLiteral("HOUR")), + new VarcharLiteral("DAY")), + new DateTrunc(dataSlot, new VarcharLiteral("DAY"))); + + assertRewrite(new DateTrunc(new DateTrunc(dataSlot, new VarcharLiteral("DAY")), + new VarcharLiteral("YEAR")), + new DateTrunc(dataSlot, new VarcharLiteral("YEAR"))); + + assertNotRewrite(new DateTrunc(new DateTrunc(dataSlot, new VarcharLiteral("HOUR")), + new VarcharLiteral("WEEK")), + new DateTrunc(dataSlot, new VarcharLiteral("WEEK"))); + + assertNotRewrite(new DateTrunc(new DateTrunc(dataSlot, new VarcharLiteral("DAY")), + new VarcharLiteral("QUARTER")), + new DateTrunc(dataSlot, new VarcharLiteral("QUARTER"))); + } +} diff --git a/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out b/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out index 5552dac72c3005..38e59530d8c78d 100644 --- a/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out +++ b/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out @@ -4,6 +4,18 @@ 2 2020-01-02 3 2020-02-01 +-- !date_list_month_partition_by_column -- +2020-01-01 1 2020-01-01 +2020-01-01 2 2020-01-02 +2020-02-01 3 2020-02-01 + +-- !date_list_month_level -- +2020-01-01 1 2020-01-01 +2020-01-02 2 2020-01-02 +2020-02-01 3 2020-02-01 + +-- !date_list_year_partition_by_column -- + -- !string_list_month -- 1 2020==01==01 2 2020==01==02 @@ -14,3 +26,13 @@ 2 2020-01-02 3 2020-02-01 +-- !date_range_month_partition_by_column -- +2020-01-01 1 2020-01-01 +2020-01-01 2 2020-01-02 +2020-02-01 3 2020-02-01 + +-- !date_range_month_level -- +2020-01-01 +2020-01-02 +2020-02-01 + diff --git a/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy index 3a1bfe9f4fdd5a..d0a30e840c9816 100644 --- a/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy @@ -69,6 +69,82 @@ suite("test_rollup_partition_mtmv") { waitingMTMVTaskFinished(jobName) order_qt_date_list_month "SELECT * FROM ${mvName} order by k1,k2" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (month_alias) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; + """ + def date_list_month_partitions = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + date_list_month_partitions.toString()) + assertEquals(2, date_list_month_partitions.size()) + waitingMTMVTaskFinished(getJobName(dbName, mvName)) + order_qt_date_list_month_partition_by_column "SELECT * FROM ${mvName}" + + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (date_trunc(month_alias, 'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'day') as month_alias, * FROM ${tableName}; + """ + def date_list_month_partitions_level = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + date_list_month_partitions_level.toString()) + assertEquals(2, date_list_month_partitions_level.size()) + waitingMTMVTaskFinished(getJobName(dbName, mvName)) + order_qt_date_list_month_level "SELECT * FROM ${mvName}" + + // mv partition level should be higher or equal then query, should fail + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (date_trunc(month_alias, 'day')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("partition column time unit level should be greater than sql select column")) + } + + // mv partition use a column not in mv sql select, should fail + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (date_trunc(`k2`, 'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'day') as month_alias FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("partition column can not find from sql select column")) + } + sql """drop materialized view if exists ${mvName};""" // list date year sql """ @@ -86,6 +162,23 @@ suite("test_rollup_partition_mtmv") { logger.info("showPartitionsResult: " + showPartitionsResult.toString()) assertEquals(1, showPartitionsResult.size()) + sql """drop materialized view if exists ${mvName};""" + // list date year + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (year_alias) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'year') as year_alias, * FROM ${tableName}; + """ + def date_list_year_partitions = sql """show partitions from ${mvName}""" + assertEquals(1, date_list_year_partitions.size()) + order_qt_date_list_year_partition_by_column "SELECT * FROM ${mvName}" + // list string month sql """drop table if exists `${tableName}`""" sql """drop materialized view if exists ${mvName};""" @@ -134,6 +227,86 @@ suite("test_rollup_partition_mtmv") { order_qt_string_list_month "SELECT * FROM ${mvName} order by k1,k2" + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (date_trunc(month_alias, 'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'day') as month_alias, * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("partition column use invalid implicit expression")) + } + + // mv partition level should be higher or equal then query, should fail + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (date_trunc(month_alias, 'day')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("partition column use invalid implicit expression")) + } + + // mv partition use a column not in mv sql select, should fail + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (date_trunc(`k2`, 'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'day') as month_alias FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("partition column can not find from sql select column")) + } + + // mv partition column type is date, base table is string, partition mapping fail + // support later + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (month_alias) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'partition_date_format'='%Y==%m==%d' + ) + AS + SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + // range date month sql """drop table if exists `${tableName}`""" sql """drop materialized view if exists ${mvName};""" @@ -180,6 +353,86 @@ suite("test_rollup_partition_mtmv") { waitingMTMVTaskFinished(jobName) order_qt_date_range_month "SELECT * FROM ${mvName} order by k1,k2" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (month_alias) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; + """ + def date_range_month_partitions = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + date_range_month_partitions.toString()) + assertEquals(2, date_range_month_partitions.size()) + + jobName = getJobName(dbName, mvName); + log.info(jobName) + waitingMTMVTaskFinished(jobName) + order_qt_date_range_month_partition_by_column "SELECT * FROM ${mvName}" + + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (date_trunc(day_alias, 'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'day') as day_alias FROM ${tableName}; + """ + def date_range_month_partitions_level = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + date_range_month_partitions_level.toString()) + assertEquals(2, date_range_month_partitions_level.size()) + waitingMTMVTaskFinished(getJobName(dbName, mvName)) + order_qt_date_range_month_level "SELECT * FROM ${mvName}" + + // mv partition level should be higher or equal then query, should fail + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (date_trunc(month_alias, 'day')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("partition column time unit level should be greater than sql select column")) + } + + // mv partition use a column not in mv sql select, should fail + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (date_trunc(`k2`, 'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'day') as day_alias FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("partition column can not find from sql select column")) + } + + // not support MAXVALUE sql """drop table if exists `${tableName}`""" sql """drop materialized view if exists ${mvName};""" @@ -217,6 +470,24 @@ suite("test_rollup_partition_mtmv") { log.info(e.getMessage()) } + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (month_alias) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + // range not support other data type sql """drop table if exists `${tableName}`""" @@ -253,6 +524,24 @@ suite("test_rollup_partition_mtmv") { log.info(e.getMessage()) } + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (month_alias) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'month') as month_alias, * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + // not support trunc hour sql """drop table if exists `${tableName}`""" sql """drop materialized view if exists ${mvName};""" @@ -289,4 +578,40 @@ suite("test_rollup_partition_mtmv") { } catch (Exception e) { log.info(e.getMessage()) } + + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (hour_alias) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'hour') as hour_alias, * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + sql """drop materialized view if exists ${mvName};""" + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (date_trunc(miniute_alias, 'hour')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT date_trunc(`k2`,'miniute') as miniute_alias, * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } }