From 2ad8c661939598d2e61f1b2a7704aac17a939760 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Tue, 31 Oct 2023 16:31:45 +0800 Subject: [PATCH 1/7] [fix](compile) fe compile failed when generate doc error msg: [INFO] --- exec:3.1.0:java (doc) @ fe-core --- ... Failed to generate doc for ignoreRuntimeFilterIds --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index a454cd8e56871a..869b30bdb4e3ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1253,7 +1253,8 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) { public boolean fasterFloatConvert = false; @VariableMgr.VarAttr(name = IGNORE_RUNTIME_FILTER_IDS, - description = {"the runtime filter id in IGNORE_RUNTIME_FILTER_IDS list will not be generated"}) + description = {"在IGNORE_RUNTIME_FILTER_IDS列表中的runtime filter将不会被生成", + "the runtime filter id in IGNORE_RUNTIME_FILTER_IDS list will not be generated"}) public String ignoreRuntimeFilterIds = ""; public static final String IGNORE_RUNTIME_FILTER_IDS = "ignore_runtime_filter_ids"; From 947ca3fec777c9e09cce7d2f2a92465b72007835 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Tue, 31 Oct 2023 21:21:47 +0800 Subject: [PATCH 2/7] Revert "[enhancement](Nereids): optimize GroupExpressionMatching (#26130)" This reverts commit 19122b55cd95af097b4ef7b6eb809f37db29765f. --- .../pattern/GroupExpressionMatching.java | 61 +++++++++++-------- .../doris/nereids/pattern/GroupMatching.java | 6 ++ .../nereids/trees/expressions/Expression.java | 15 +---- .../plans/physical/PhysicalHashJoin.java | 15 ++--- .../apache/doris/nereids/util/JoinUtils.java | 12 ++-- .../apache/doris/nereids/util/PlanUtils.java | 2 +- 6 files changed, 56 insertions(+), 55 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java index e4e6888ec2cf55..02421e4041c69c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import com.google.common.collect.ImmutableList; @@ -69,19 +70,21 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre int childrenGroupArity = groupExpression.arity(); int patternArity = pattern.arity(); - // (logicalFilter(), multi()) match (logicalFilter()), - // but (logicalFilter(), logicalFilter(), multi()) not match (logicalFilter()) - boolean extraMulti = patternArity == childrenGroupArity + 1 - && (pattern.hasMultiChild() || pattern.hasMultiGroupChild()); - if (patternArity > childrenGroupArity && !extraMulti) { - return; - } + if (!(pattern instanceof SubTreePattern)) { + // (logicalFilter(), multi()) match (logicalFilter()), + // but (logicalFilter(), logicalFilter(), multi()) not match (logicalFilter()) + boolean extraMulti = patternArity == childrenGroupArity + 1 + && (pattern.hasMultiChild() || pattern.hasMultiGroupChild()); + if (patternArity > childrenGroupArity && !extraMulti) { + return; + } - // (multi()) match (logicalFilter(), logicalFilter()), - // but (logicalFilter()) not match (logicalFilter(), logicalFilter()) - if (!pattern.isAny() && patternArity < childrenGroupArity - && !pattern.hasMultiChild() && !pattern.hasMultiGroupChild()) { - return; + // (multi()) match (logicalFilter(), logicalFilter()), + // but (logicalFilter()) not match (logicalFilter(), logicalFilter()) + if (!pattern.isAny() && patternArity < childrenGroupArity + && !pattern.hasMultiChild() && !pattern.hasMultiGroupChild()) { + return; + } } // Pattern.GROUP / Pattern.MULTI / Pattern.MULTI_GROUP can not match GroupExpression @@ -91,7 +94,7 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre // getPlan return the plan with GroupPlan as children Plan root = groupExpression.getPlan(); - if (patternArity == 0) { + if (patternArity == 0 && !(pattern instanceof SubTreePattern)) { if (pattern.matchPredicates(root)) { // if no children pattern, we treat all children as GROUP. e.g. Pattern.ANY. // leaf plan will enter this branch too, e.g. logicalRelation(). @@ -107,8 +110,12 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre List childrenPlan = matchingChildGroup(pattern, childGroup, i); if (childrenPlan.isEmpty()) { - // current pattern is match but children patterns not match - return; + if (pattern instanceof SubTreePattern) { + childrenPlan = ImmutableList.of(new GroupPlan(childGroup)); + } else { + // current pattern is match but children patterns not match + return; + } } childrenPlans[i] = childrenPlan; } @@ -127,16 +134,20 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre private List matchingChildGroup(Pattern parentPattern, Group childGroup, int childIndex) { Pattern childPattern; - boolean isLastPattern = childIndex + 1 >= parentPattern.arity(); - int patternChildIndex = isLastPattern ? parentPattern.arity() - 1 : childIndex; - - childPattern = parentPattern.child(patternChildIndex); - // translate MULTI and MULTI_GROUP to ANY and GROUP - if (isLastPattern) { - if (childPattern.isMulti()) { - childPattern = Pattern.ANY; - } else if (childPattern.isMultiGroup()) { - childPattern = Pattern.GROUP; + if (parentPattern instanceof SubTreePattern) { + childPattern = parentPattern; + } else { + boolean isLastPattern = childIndex + 1 >= parentPattern.arity(); + int patternChildIndex = isLastPattern ? parentPattern.arity() - 1 : childIndex; + + childPattern = parentPattern.child(patternChildIndex); + // translate MULTI and MULTI_GROUP to ANY and GROUP + if (isLastPattern) { + if (childPattern.isMulti()) { + childPattern = Pattern.ANY; + } else if (childPattern.isMultiGroup()) { + childPattern = Pattern.GROUP; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java index a9521ca50e9106..d80fb40b3a5158 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java @@ -45,6 +45,12 @@ public static List getAllMatchingPlans(Pattern pattern, Group group) { matchingPlans.add(plan); } } + // Jackwener: We don't need to match physical expressions. + // for (GroupExpression groupExpression : group.getPhysicalExpressions()) { + // for (Plan plan : new GroupExpressionMatching(pattern, groupExpression)) { + // matchingPlans.add(plan); + // } + // } } return matchingPlans; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index 12a3a9768caf7d..3f0370d7c3bb9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -37,7 +37,6 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; @@ -45,6 +44,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; /** * Abstract class for all Expression in Nereids. @@ -247,19 +247,8 @@ public final Set getInputSlots() { return collect(Slot.class::isInstance); } - /** - * Get all the input slot ids of the expression. - *

- * Note that the input slots of subquery's inner plan is not included. - */ public final Set getInputSlotExprIds() { - ImmutableSet.Builder result = ImmutableSet.builder(); - foreach(node -> { - if (node instanceof Slot) { - result.add(((Slot) node).getExprId()); - } - }); - return result.build(); + return getInputSlots().stream().map(NamedExpression::getExprId).collect(Collectors.toSet()); } public boolean isLiteral() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java index 994b4d4f971261..b60afd673089ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java @@ -43,9 +43,9 @@ import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -113,25 +113,22 @@ private PhysicalHashJoin( * Return pair of left used slots and right used slots. */ public Pair, List> getHashConjunctsExprIds() { - int size = hashJoinConjuncts.size(); - - List exprIds1 = new ArrayList<>(size); - List exprIds2 = new ArrayList<>(size); + List exprIds1 = Lists.newArrayListWithCapacity(hashJoinConjuncts.size()); + List exprIds2 = Lists.newArrayListWithCapacity(hashJoinConjuncts.size()); Set leftExprIds = left().getOutputExprIdSet(); Set rightExprIds = right().getOutputExprIdSet(); for (Expression expr : hashJoinConjuncts) { - for (ExprId exprId : expr.getInputSlotExprIds()) { + expr.getInputSlotExprIds().forEach(exprId -> { if (leftExprIds.contains(exprId)) { exprIds1.add(exprId); } else if (rightExprIds.contains(exprId)) { exprIds2.add(exprId); } else { - throw new RuntimeException("Invalid ExprId found: " + exprId - + ". Cannot generate valid equal on clause slot pairs for join."); + throw new RuntimeException("Could not generate valid equal on clause slot pairs for join"); } - } + }); } return Pair.of(exprIds1, exprIds2); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index bcf53ce29f84eb..d1fb973dd61264 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -38,7 +38,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -67,10 +66,9 @@ public static boolean couldBroadcast(Join join) { * check if the row count of the left child in the broadcast join is less than a threshold value. */ public static boolean checkBroadcastJoinStats(PhysicalHashJoin join) { - SessionVariable sessionVariable = ConnectContext.get().getSessionVariable(); - double memLimit = sessionVariable.getMaxExecMemByte(); - double rowsLimit = sessionVariable.getBroadcastRowCountLimit(); - double brMemlimit = sessionVariable.getBroadcastHashtableMemLimitPercentage(); + double memLimit = ConnectContext.get().getSessionVariable().getMaxExecMemByte(); + double rowsLimit = ConnectContext.get().getSessionVariable().getBroadcastRowCountLimit(); + double brMemlimit = ConnectContext.get().getSessionVariable().getBroadcastHashtableMemLimitPercentage(); double datasize = join.getGroupExpression().get().child(1).getStatistics().computeSize(); double rowCount = join.getGroupExpression().get().child(1).getStatistics().getRowCount(); return rowCount <= rowsLimit && datasize <= memLimit * brMemlimit; @@ -116,12 +114,12 @@ boolean isCoveredByRightSlots(ExprId slot) { * @return true if the equal can be used as hash join condition */ public boolean isHashJoinCondition(EqualTo equalTo) { - Set equalLeft = equalTo.left().getInputSlots(); + Set equalLeft = equalTo.left().collect(Slot.class::isInstance); if (equalLeft.isEmpty()) { return false; } - Set equalRight = equalTo.right().getInputSlots(); + Set equalRight = equalTo.right().collect(Slot.class::isInstance); if (equalRight.isEmpty()) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java index 48eb452a74ca13..17034c15e6ee93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java @@ -55,7 +55,7 @@ public static Plan filterOrSelf(Set predicates, Plan plan) { * normalize comparison predicate on a binary plan to its two sides are corresponding to the child's output. */ public static ComparisonPredicate maybeCommuteComparisonPredicate(ComparisonPredicate expression, Plan left) { - Set slots = expression.left().getInputSlots(); + Set slots = expression.left().collect(Slot.class::isInstance); Set leftSlots = left.getOutputSet(); Set buffer = Sets.newHashSet(slots); buffer.removeAll(leftSlots); From b49ae4f037a86105d9ce3d48aa4b2b44ccd8d508 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Tue, 31 Oct 2023 21:21:51 +0800 Subject: [PATCH 3/7] Revert "Revert "[enhancement](Nereids): optimize GroupExpressionMatching (#26130)"" This reverts commit 947ca3fec777c9e09cce7d2f2a92465b72007835. --- .../pattern/GroupExpressionMatching.java | 61 ++++++++----------- .../doris/nereids/pattern/GroupMatching.java | 6 -- .../nereids/trees/expressions/Expression.java | 15 ++++- .../plans/physical/PhysicalHashJoin.java | 15 +++-- .../apache/doris/nereids/util/JoinUtils.java | 12 ++-- .../apache/doris/nereids/util/PlanUtils.java | 2 +- 6 files changed, 55 insertions(+), 56 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java index 02421e4041c69c..e4e6888ec2cf55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import com.google.common.collect.ImmutableList; @@ -70,21 +69,19 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre int childrenGroupArity = groupExpression.arity(); int patternArity = pattern.arity(); - if (!(pattern instanceof SubTreePattern)) { - // (logicalFilter(), multi()) match (logicalFilter()), - // but (logicalFilter(), logicalFilter(), multi()) not match (logicalFilter()) - boolean extraMulti = patternArity == childrenGroupArity + 1 - && (pattern.hasMultiChild() || pattern.hasMultiGroupChild()); - if (patternArity > childrenGroupArity && !extraMulti) { - return; - } + // (logicalFilter(), multi()) match (logicalFilter()), + // but (logicalFilter(), logicalFilter(), multi()) not match (logicalFilter()) + boolean extraMulti = patternArity == childrenGroupArity + 1 + && (pattern.hasMultiChild() || pattern.hasMultiGroupChild()); + if (patternArity > childrenGroupArity && !extraMulti) { + return; + } - // (multi()) match (logicalFilter(), logicalFilter()), - // but (logicalFilter()) not match (logicalFilter(), logicalFilter()) - if (!pattern.isAny() && patternArity < childrenGroupArity - && !pattern.hasMultiChild() && !pattern.hasMultiGroupChild()) { - return; - } + // (multi()) match (logicalFilter(), logicalFilter()), + // but (logicalFilter()) not match (logicalFilter(), logicalFilter()) + if (!pattern.isAny() && patternArity < childrenGroupArity + && !pattern.hasMultiChild() && !pattern.hasMultiGroupChild()) { + return; } // Pattern.GROUP / Pattern.MULTI / Pattern.MULTI_GROUP can not match GroupExpression @@ -94,7 +91,7 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre // getPlan return the plan with GroupPlan as children Plan root = groupExpression.getPlan(); - if (patternArity == 0 && !(pattern instanceof SubTreePattern)) { + if (patternArity == 0) { if (pattern.matchPredicates(root)) { // if no children pattern, we treat all children as GROUP. e.g. Pattern.ANY. // leaf plan will enter this branch too, e.g. logicalRelation(). @@ -110,12 +107,8 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre List childrenPlan = matchingChildGroup(pattern, childGroup, i); if (childrenPlan.isEmpty()) { - if (pattern instanceof SubTreePattern) { - childrenPlan = ImmutableList.of(new GroupPlan(childGroup)); - } else { - // current pattern is match but children patterns not match - return; - } + // current pattern is match but children patterns not match + return; } childrenPlans[i] = childrenPlan; } @@ -134,20 +127,16 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre private List matchingChildGroup(Pattern parentPattern, Group childGroup, int childIndex) { Pattern childPattern; - if (parentPattern instanceof SubTreePattern) { - childPattern = parentPattern; - } else { - boolean isLastPattern = childIndex + 1 >= parentPattern.arity(); - int patternChildIndex = isLastPattern ? parentPattern.arity() - 1 : childIndex; - - childPattern = parentPattern.child(patternChildIndex); - // translate MULTI and MULTI_GROUP to ANY and GROUP - if (isLastPattern) { - if (childPattern.isMulti()) { - childPattern = Pattern.ANY; - } else if (childPattern.isMultiGroup()) { - childPattern = Pattern.GROUP; - } + boolean isLastPattern = childIndex + 1 >= parentPattern.arity(); + int patternChildIndex = isLastPattern ? parentPattern.arity() - 1 : childIndex; + + childPattern = parentPattern.child(patternChildIndex); + // translate MULTI and MULTI_GROUP to ANY and GROUP + if (isLastPattern) { + if (childPattern.isMulti()) { + childPattern = Pattern.ANY; + } else if (childPattern.isMultiGroup()) { + childPattern = Pattern.GROUP; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java index d80fb40b3a5158..a9521ca50e9106 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java @@ -45,12 +45,6 @@ public static List getAllMatchingPlans(Pattern pattern, Group group) { matchingPlans.add(plan); } } - // Jackwener: We don't need to match physical expressions. - // for (GroupExpression groupExpression : group.getPhysicalExpressions()) { - // for (Plan plan : new GroupExpressionMatching(pattern, groupExpression)) { - // matchingPlans.add(plan); - // } - // } } return matchingPlans; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index 3f0370d7c3bb9d..12a3a9768caf7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -37,6 +37,7 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; @@ -44,7 +45,6 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; /** * Abstract class for all Expression in Nereids. @@ -247,8 +247,19 @@ public final Set getInputSlots() { return collect(Slot.class::isInstance); } + /** + * Get all the input slot ids of the expression. + *

+ * Note that the input slots of subquery's inner plan is not included. + */ public final Set getInputSlotExprIds() { - return getInputSlots().stream().map(NamedExpression::getExprId).collect(Collectors.toSet()); + ImmutableSet.Builder result = ImmutableSet.builder(); + foreach(node -> { + if (node instanceof Slot) { + result.add(((Slot) node).getExprId()); + } + }); + return result.build(); } public boolean isLiteral() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java index b60afd673089ca..994b4d4f971261 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java @@ -43,9 +43,9 @@ import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -113,22 +113,25 @@ private PhysicalHashJoin( * Return pair of left used slots and right used slots. */ public Pair, List> getHashConjunctsExprIds() { - List exprIds1 = Lists.newArrayListWithCapacity(hashJoinConjuncts.size()); - List exprIds2 = Lists.newArrayListWithCapacity(hashJoinConjuncts.size()); + int size = hashJoinConjuncts.size(); + + List exprIds1 = new ArrayList<>(size); + List exprIds2 = new ArrayList<>(size); Set leftExprIds = left().getOutputExprIdSet(); Set rightExprIds = right().getOutputExprIdSet(); for (Expression expr : hashJoinConjuncts) { - expr.getInputSlotExprIds().forEach(exprId -> { + for (ExprId exprId : expr.getInputSlotExprIds()) { if (leftExprIds.contains(exprId)) { exprIds1.add(exprId); } else if (rightExprIds.contains(exprId)) { exprIds2.add(exprId); } else { - throw new RuntimeException("Could not generate valid equal on clause slot pairs for join"); + throw new RuntimeException("Invalid ExprId found: " + exprId + + ". Cannot generate valid equal on clause slot pairs for join."); } - }); + } } return Pair.of(exprIds1, exprIds2); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index d1fb973dd61264..bcf53ce29f84eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -38,6 +38,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -66,9 +67,10 @@ public static boolean couldBroadcast(Join join) { * check if the row count of the left child in the broadcast join is less than a threshold value. */ public static boolean checkBroadcastJoinStats(PhysicalHashJoin join) { - double memLimit = ConnectContext.get().getSessionVariable().getMaxExecMemByte(); - double rowsLimit = ConnectContext.get().getSessionVariable().getBroadcastRowCountLimit(); - double brMemlimit = ConnectContext.get().getSessionVariable().getBroadcastHashtableMemLimitPercentage(); + SessionVariable sessionVariable = ConnectContext.get().getSessionVariable(); + double memLimit = sessionVariable.getMaxExecMemByte(); + double rowsLimit = sessionVariable.getBroadcastRowCountLimit(); + double brMemlimit = sessionVariable.getBroadcastHashtableMemLimitPercentage(); double datasize = join.getGroupExpression().get().child(1).getStatistics().computeSize(); double rowCount = join.getGroupExpression().get().child(1).getStatistics().getRowCount(); return rowCount <= rowsLimit && datasize <= memLimit * brMemlimit; @@ -114,12 +116,12 @@ boolean isCoveredByRightSlots(ExprId slot) { * @return true if the equal can be used as hash join condition */ public boolean isHashJoinCondition(EqualTo equalTo) { - Set equalLeft = equalTo.left().collect(Slot.class::isInstance); + Set equalLeft = equalTo.left().getInputSlots(); if (equalLeft.isEmpty()) { return false; } - Set equalRight = equalTo.right().collect(Slot.class::isInstance); + Set equalRight = equalTo.right().getInputSlots(); if (equalRight.isEmpty()) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java index 17034c15e6ee93..48eb452a74ca13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java @@ -55,7 +55,7 @@ public static Plan filterOrSelf(Set predicates, Plan plan) { * normalize comparison predicate on a binary plan to its two sides are corresponding to the child's output. */ public static ComparisonPredicate maybeCommuteComparisonPredicate(ComparisonPredicate expression, Plan left) { - Set slots = expression.left().collect(Slot.class::isInstance); + Set slots = expression.left().getInputSlots(); Set leftSlots = left.getOutputSet(); Set buffer = Sets.newHashSet(slots); buffer.removeAll(leftSlots); From fa163fc6047fad1680f6af577245a826161b8530 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Tue, 31 Oct 2023 21:22:16 +0800 Subject: [PATCH 4/7] Revert "[enhancement](Nereids): optimize GroupExpressionMatching (#26130)" This reverts commit 19122b55cd95af097b4ef7b6eb809f37db29765f. --- .../pattern/GroupExpressionMatching.java | 61 +++++++++++-------- .../doris/nereids/pattern/GroupMatching.java | 6 ++ .../nereids/trees/expressions/Expression.java | 15 +---- .../plans/physical/PhysicalHashJoin.java | 15 ++--- .../apache/doris/nereids/util/JoinUtils.java | 12 ++-- .../apache/doris/nereids/util/PlanUtils.java | 2 +- 6 files changed, 56 insertions(+), 55 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java index e4e6888ec2cf55..02421e4041c69c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import com.google.common.collect.ImmutableList; @@ -69,19 +70,21 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre int childrenGroupArity = groupExpression.arity(); int patternArity = pattern.arity(); - // (logicalFilter(), multi()) match (logicalFilter()), - // but (logicalFilter(), logicalFilter(), multi()) not match (logicalFilter()) - boolean extraMulti = patternArity == childrenGroupArity + 1 - && (pattern.hasMultiChild() || pattern.hasMultiGroupChild()); - if (patternArity > childrenGroupArity && !extraMulti) { - return; - } + if (!(pattern instanceof SubTreePattern)) { + // (logicalFilter(), multi()) match (logicalFilter()), + // but (logicalFilter(), logicalFilter(), multi()) not match (logicalFilter()) + boolean extraMulti = patternArity == childrenGroupArity + 1 + && (pattern.hasMultiChild() || pattern.hasMultiGroupChild()); + if (patternArity > childrenGroupArity && !extraMulti) { + return; + } - // (multi()) match (logicalFilter(), logicalFilter()), - // but (logicalFilter()) not match (logicalFilter(), logicalFilter()) - if (!pattern.isAny() && patternArity < childrenGroupArity - && !pattern.hasMultiChild() && !pattern.hasMultiGroupChild()) { - return; + // (multi()) match (logicalFilter(), logicalFilter()), + // but (logicalFilter()) not match (logicalFilter(), logicalFilter()) + if (!pattern.isAny() && patternArity < childrenGroupArity + && !pattern.hasMultiChild() && !pattern.hasMultiGroupChild()) { + return; + } } // Pattern.GROUP / Pattern.MULTI / Pattern.MULTI_GROUP can not match GroupExpression @@ -91,7 +94,7 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre // getPlan return the plan with GroupPlan as children Plan root = groupExpression.getPlan(); - if (patternArity == 0) { + if (patternArity == 0 && !(pattern instanceof SubTreePattern)) { if (pattern.matchPredicates(root)) { // if no children pattern, we treat all children as GROUP. e.g. Pattern.ANY. // leaf plan will enter this branch too, e.g. logicalRelation(). @@ -107,8 +110,12 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre List childrenPlan = matchingChildGroup(pattern, childGroup, i); if (childrenPlan.isEmpty()) { - // current pattern is match but children patterns not match - return; + if (pattern instanceof SubTreePattern) { + childrenPlan = ImmutableList.of(new GroupPlan(childGroup)); + } else { + // current pattern is match but children patterns not match + return; + } } childrenPlans[i] = childrenPlan; } @@ -127,16 +134,20 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre private List matchingChildGroup(Pattern parentPattern, Group childGroup, int childIndex) { Pattern childPattern; - boolean isLastPattern = childIndex + 1 >= parentPattern.arity(); - int patternChildIndex = isLastPattern ? parentPattern.arity() - 1 : childIndex; - - childPattern = parentPattern.child(patternChildIndex); - // translate MULTI and MULTI_GROUP to ANY and GROUP - if (isLastPattern) { - if (childPattern.isMulti()) { - childPattern = Pattern.ANY; - } else if (childPattern.isMultiGroup()) { - childPattern = Pattern.GROUP; + if (parentPattern instanceof SubTreePattern) { + childPattern = parentPattern; + } else { + boolean isLastPattern = childIndex + 1 >= parentPattern.arity(); + int patternChildIndex = isLastPattern ? parentPattern.arity() - 1 : childIndex; + + childPattern = parentPattern.child(patternChildIndex); + // translate MULTI and MULTI_GROUP to ANY and GROUP + if (isLastPattern) { + if (childPattern.isMulti()) { + childPattern = Pattern.ANY; + } else if (childPattern.isMultiGroup()) { + childPattern = Pattern.GROUP; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java index a9521ca50e9106..d80fb40b3a5158 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java @@ -45,6 +45,12 @@ public static List getAllMatchingPlans(Pattern pattern, Group group) { matchingPlans.add(plan); } } + // Jackwener: We don't need to match physical expressions. + // for (GroupExpression groupExpression : group.getPhysicalExpressions()) { + // for (Plan plan : new GroupExpressionMatching(pattern, groupExpression)) { + // matchingPlans.add(plan); + // } + // } } return matchingPlans; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index 12a3a9768caf7d..3f0370d7c3bb9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -37,7 +37,6 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; @@ -45,6 +44,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; /** * Abstract class for all Expression in Nereids. @@ -247,19 +247,8 @@ public final Set getInputSlots() { return collect(Slot.class::isInstance); } - /** - * Get all the input slot ids of the expression. - *

- * Note that the input slots of subquery's inner plan is not included. - */ public final Set getInputSlotExprIds() { - ImmutableSet.Builder result = ImmutableSet.builder(); - foreach(node -> { - if (node instanceof Slot) { - result.add(((Slot) node).getExprId()); - } - }); - return result.build(); + return getInputSlots().stream().map(NamedExpression::getExprId).collect(Collectors.toSet()); } public boolean isLiteral() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java index 994b4d4f971261..b60afd673089ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java @@ -43,9 +43,9 @@ import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -113,25 +113,22 @@ private PhysicalHashJoin( * Return pair of left used slots and right used slots. */ public Pair, List> getHashConjunctsExprIds() { - int size = hashJoinConjuncts.size(); - - List exprIds1 = new ArrayList<>(size); - List exprIds2 = new ArrayList<>(size); + List exprIds1 = Lists.newArrayListWithCapacity(hashJoinConjuncts.size()); + List exprIds2 = Lists.newArrayListWithCapacity(hashJoinConjuncts.size()); Set leftExprIds = left().getOutputExprIdSet(); Set rightExprIds = right().getOutputExprIdSet(); for (Expression expr : hashJoinConjuncts) { - for (ExprId exprId : expr.getInputSlotExprIds()) { + expr.getInputSlotExprIds().forEach(exprId -> { if (leftExprIds.contains(exprId)) { exprIds1.add(exprId); } else if (rightExprIds.contains(exprId)) { exprIds2.add(exprId); } else { - throw new RuntimeException("Invalid ExprId found: " + exprId - + ". Cannot generate valid equal on clause slot pairs for join."); + throw new RuntimeException("Could not generate valid equal on clause slot pairs for join"); } - } + }); } return Pair.of(exprIds1, exprIds2); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index bcf53ce29f84eb..d1fb973dd61264 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -38,7 +38,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -67,10 +66,9 @@ public static boolean couldBroadcast(Join join) { * check if the row count of the left child in the broadcast join is less than a threshold value. */ public static boolean checkBroadcastJoinStats(PhysicalHashJoin join) { - SessionVariable sessionVariable = ConnectContext.get().getSessionVariable(); - double memLimit = sessionVariable.getMaxExecMemByte(); - double rowsLimit = sessionVariable.getBroadcastRowCountLimit(); - double brMemlimit = sessionVariable.getBroadcastHashtableMemLimitPercentage(); + double memLimit = ConnectContext.get().getSessionVariable().getMaxExecMemByte(); + double rowsLimit = ConnectContext.get().getSessionVariable().getBroadcastRowCountLimit(); + double brMemlimit = ConnectContext.get().getSessionVariable().getBroadcastHashtableMemLimitPercentage(); double datasize = join.getGroupExpression().get().child(1).getStatistics().computeSize(); double rowCount = join.getGroupExpression().get().child(1).getStatistics().getRowCount(); return rowCount <= rowsLimit && datasize <= memLimit * brMemlimit; @@ -116,12 +114,12 @@ boolean isCoveredByRightSlots(ExprId slot) { * @return true if the equal can be used as hash join condition */ public boolean isHashJoinCondition(EqualTo equalTo) { - Set equalLeft = equalTo.left().getInputSlots(); + Set equalLeft = equalTo.left().collect(Slot.class::isInstance); if (equalLeft.isEmpty()) { return false; } - Set equalRight = equalTo.right().getInputSlots(); + Set equalRight = equalTo.right().collect(Slot.class::isInstance); if (equalRight.isEmpty()) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java index 48eb452a74ca13..17034c15e6ee93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java @@ -55,7 +55,7 @@ public static Plan filterOrSelf(Set predicates, Plan plan) { * normalize comparison predicate on a binary plan to its two sides are corresponding to the child's output. */ public static ComparisonPredicate maybeCommuteComparisonPredicate(ComparisonPredicate expression, Plan left) { - Set slots = expression.left().getInputSlots(); + Set slots = expression.left().collect(Slot.class::isInstance); Set leftSlots = left.getOutputSet(); Set buffer = Sets.newHashSet(slots); buffer.removeAll(leftSlots); From 1a4fa54a71faa14be023bf55bb07cceff65409ab Mon Sep 17 00:00:00 2001 From: morrySnow Date: Tue, 31 Oct 2023 21:22:19 +0800 Subject: [PATCH 5/7] Revert "[Performance](Nereids): optimize GroupExpressionMatching (#26084)" This reverts commit 0d956e90cf920039b8baa79c170a298be56a128d. --- .../nereids/jobs/cascades/ApplyRuleJob.java | 2 +- .../pattern/GroupExpressionMatching.java | 41 ++++++++++--------- .../doris/nereids/pattern/GroupMatching.java | 11 +++-- .../EnforceMissingPropertiesHelper.java | 2 + .../doris/nereids/trees/AbstractTreeNode.java | 13 ++++++ .../nereids/trees/plans/AbstractPlan.java | 11 ----- .../nereids_p0/expression/topn_to_max.groovy | 4 +- 7 files changed, 44 insertions(+), 40 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java index 5560c369dd6f74..3e73850b0158d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java @@ -61,7 +61,7 @@ public ApplyRuleJob(GroupExpression groupExpression, Rule rule, JobContext conte } @Override - public final void execute() throws AnalysisException { + public void execute() throws AnalysisException { if (groupExpression.hasApplied(rule) || groupExpression.isUnused()) { return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java index 02421e4041c69c..f73ddcc88688b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java @@ -55,7 +55,6 @@ public GroupExpressionIterator iterator() { public static class GroupExpressionIterator implements Iterator { private final List results = Lists.newArrayList(); private int resultIndex = 0; - private int resultsSize; /** * Constructor. @@ -104,7 +103,7 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre // matching children group, one List per child // first dimension is every child group's plan // second dimension is all matched plan in one group - List[] childrenPlans = new List[childrenGroupArity]; + List> childrenPlans = Lists.newArrayListWithCapacity(childrenGroupArity); for (int i = 0; i < childrenGroupArity; ++i) { Group childGroup = groupExpression.child(i); List childrenPlan = matchingChildGroup(pattern, childGroup, i); @@ -117,7 +116,7 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre return; } } - childrenPlans[i] = childrenPlan; + childrenPlans.add(childrenPlan); } assembleAllCombinationPlanTree(root, pattern, groupExpression, childrenPlans); } else if (patternArity == 1 && (pattern.hasMultiChild() || pattern.hasMultiGroupChild())) { @@ -128,7 +127,6 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre results.add(root); } } - this.resultsSize = results.size(); } private List matchingChildGroup(Pattern parentPattern, @@ -156,37 +154,40 @@ private List matchingChildGroup(Pattern parentPattern, } private void assembleAllCombinationPlanTree(Plan root, Pattern rootPattern, - GroupExpression groupExpression, List[] childrenPlans) { - int childrenPlansSize = childrenPlans.length; - int[] childrenPlanIndex = new int[childrenPlansSize]; + GroupExpression groupExpression, + List> childrenPlans) { + int[] childrenPlanIndex = new int[childrenPlans.size()]; int offset = 0; LogicalProperties logicalProperties = groupExpression.getOwnerGroup().getLogicalProperties(); // assemble all combination of plan tree by current root plan and children plan - Optional groupExprOption = Optional.of(groupExpression); - Optional logicalPropOption = Optional.of(logicalProperties); - while (offset < childrenPlansSize) { - ImmutableList.Builder childrenBuilder = ImmutableList.builderWithExpectedSize(childrenPlansSize); - for (int i = 0; i < childrenPlansSize; i++) { - childrenBuilder.add(childrenPlans[i].get(childrenPlanIndex[i])); + while (offset < childrenPlans.size()) { + ImmutableList.Builder childrenBuilder = + ImmutableList.builderWithExpectedSize(childrenPlans.size()); + for (int i = 0; i < childrenPlans.size(); i++) { + childrenBuilder.add(childrenPlans.get(i).get(childrenPlanIndex[i])); } List children = childrenBuilder.build(); // assemble children: replace GroupPlan to real plan, // withChildren will erase groupExpression, so we must // withGroupExpression too. - Plan rootWithChildren = root.withGroupExprLogicalPropChildren(groupExprOption, - logicalPropOption, children); + Plan rootWithChildren = root.withGroupExprLogicalPropChildren(Optional.of(groupExpression), + Optional.of(logicalProperties), children); if (rootPattern.matchPredicates(rootWithChildren)) { results.add(rootWithChildren); } - for (offset = 0; offset < childrenPlansSize; offset++) { + offset = 0; + while (true) { childrenPlanIndex[offset]++; - if (childrenPlanIndex[offset] == childrenPlans[offset].size()) { - // Reset the index when it reaches the size of the current child plan list + if (childrenPlanIndex[offset] == childrenPlans.get(offset).size()) { childrenPlanIndex[offset] = 0; + offset++; + if (offset == childrenPlans.size()) { + break; + } } else { - break; // Break the loop when the index is within the size of the current child plan list + break; } } } @@ -194,7 +195,7 @@ private void assembleAllCombinationPlanTree(Plan root, Pattern rootPattern @Override public boolean hasNext() { - return resultIndex < resultsSize; + return resultIndex < results.size(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java index d80fb40b3a5158..b3e59590d37a96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java @@ -45,12 +45,11 @@ public static List getAllMatchingPlans(Pattern pattern, Group group) { matchingPlans.add(plan); } } - // Jackwener: We don't need to match physical expressions. - // for (GroupExpression groupExpression : group.getPhysicalExpressions()) { - // for (Plan plan : new GroupExpressionMatching(pattern, groupExpression)) { - // matchingPlans.add(plan); - // } - // } + for (GroupExpression groupExpression : group.getPhysicalExpressions()) { + for (Plan plan : new GroupExpressionMatching(pattern, groupExpression)) { + matchingPlans.add(plan); + } + } } return matchingPlans; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java index bd9f103c7c704c..d548e3254c47d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java @@ -38,11 +38,13 @@ public class EnforceMissingPropertiesHelper { private static final EventProducer ENFORCER_TRACER = new EventProducer(EnforcerEvent.class, EventChannel.getDefaultChannel().addConsumers(new LogConsumer(EnforcerEvent.class, EventChannel.LOG))); + private final JobContext context; private final GroupExpression groupExpression; private Cost curTotalCost; public EnforceMissingPropertiesHelper(JobContext context, GroupExpression groupExpression, Cost curTotalCost) { + this.context = context; this.groupExpression = groupExpression; this.curTotalCost = curTotalCost; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java index 7a545ec17be8a2..0305ae2afad699 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java @@ -17,6 +17,10 @@ package org.apache.doris.nereids.trees; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.planner.PlanNodeId; + import com.google.common.collect.ImmutableList; import java.util.List; @@ -29,6 +33,7 @@ */ public abstract class AbstractTreeNode> implements TreeNode { + protected final ObjectId id = StatementScopeIdGenerator.newObjectId(); protected final List children; // TODO: Maybe we should use a GroupPlan to avoid TreeNode hold the GroupExpression. // https://github.com/apache/doris/pull/9807#discussion_r884829067 @@ -54,4 +59,12 @@ public List children() { public int arity() { return children.size(); } + + /** + * used for PhysicalPlanTranslator only + * @return PlanNodeId + */ + public PlanNodeId translatePlanNodeId() { + return id.toPlanNodeId(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index c223dd43b6ebac..38a209ff55f49b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -24,11 +24,9 @@ import org.apache.doris.nereids.trees.AbstractTreeNode; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.util.MutableState; import org.apache.doris.nereids.util.MutableState.EmptyMutableState; import org.apache.doris.nereids.util.TreeStringUtils; -import org.apache.doris.planner.PlanNodeId; import org.apache.doris.statistics.Statistics; import com.google.common.base.Supplier; @@ -47,7 +45,6 @@ */ public abstract class AbstractPlan extends AbstractTreeNode implements Plan { public static final String FRAGMENT_ID = "fragment"; - protected final ObjectId id = StatementScopeIdGenerator.newObjectId(); protected final Statistics statistics; protected final PlanType type; @@ -171,12 +168,4 @@ public Optional getMutableState(String key) { public void setMutableState(String key, Object state) { this.mutableState = this.mutableState.set(key, state); } - - /** - * used for PhysicalPlanTranslator only - * @return PlanNodeId - */ - public PlanNodeId translatePlanNodeId() { - return id.toPlanNodeId(); - } } diff --git a/regression-test/suites/nereids_p0/expression/topn_to_max.groovy b/regression-test/suites/nereids_p0/expression/topn_to_max.groovy index 4c05b42ccc3d06..ae848b5a244f3a 100644 --- a/regression-test/suites/nereids_p0/expression/topn_to_max.groovy +++ b/regression-test/suites/nereids_p0/expression/topn_to_max.groovy @@ -31,7 +31,7 @@ suite("test_topn_to_max") { group by k1; ''' res = sql ''' - explain rewritten plan select k1, topn(k2, 1) + explain rewritten plan select k1, max(k2) from test_topn_to_max group by k1; ''' @@ -42,7 +42,7 @@ suite("test_topn_to_max") { from test_topn_to_max; ''' res = sql ''' - explain rewritten plan select topn(k2, 1) + explain rewritten plan select max(k2) from test_topn_to_max; ''' assertTrue(res.toString().contains("max")) From 22152163232a2f4705861819a67aba4217cfdebb Mon Sep 17 00:00:00 2001 From: morrySnow Date: Tue, 31 Oct 2023 21:26:56 +0800 Subject: [PATCH 6/7] =?UTF-8?q?Revert=20"[Bug](materialized-view)=20Select?= =?UTF-8?q?MaterializedIndexWithAggregate=20do=20not=20change=20plan=20whe?= =?UTF-8?q?n=20match=20ba=E2=80=A6=20(#26145)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 8d7abf60f94d2d1208b71e96b9290ea02122b8d8. --- .../analysis/CreateMaterializedViewStmt.java | 3 -- .../SelectMaterializedIndexWithAggregate.java | 40 ------------------- .../testProjectionMV1/testProjectionMV1.out | 8 ---- .../testProjectionMV1.groovy | 17 -------- 4 files changed, 68 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index c42f3734f3d71d..5c3c117a71fd3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -298,9 +298,6 @@ private void analyzeFromClause() throws AnalysisException { if (tableRefList.size() != 1) { throw new AnalysisException("The materialized view only support one table in from clause."); } - if (!isReplay && tableRefList.get(0).hasExplicitAlias()) { - throw new AnalysisException("The materialized view not support table with alias."); - } TableName tableName = tableRefList.get(0).getName(); if (tableName == null) { throw new AnalysisException("table in from clause is invalid, please check if it's single table " diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java index 4c3179c29033c8..12fe7b75c9ca14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java @@ -116,10 +116,6 @@ public List buildRules() { agg.getGroupByExpressions(), new HashSet<>(agg.getExpressions())); - if (result.indexId == scan.getTable().getBaseIndexId()) { - return ctx.root; - } - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); @@ -166,10 +162,6 @@ public List buildRules() { requiredExpr ); - if (result.indexId == scan.getTable().getBaseIndexId()) { - return ctx.root; - } - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); @@ -215,10 +207,6 @@ public List buildRules() { collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects()) ); - if (result.indexId == scan.getTable().getBaseIndexId()) { - return ctx.root; - } - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); @@ -277,10 +265,6 @@ public List buildRules() { requiredExpr ); - if (result.indexId == scan.getTable().getBaseIndexId()) { - return ctx.root; - } - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); @@ -338,10 +322,6 @@ public List buildRules() { requiredExpr ); - if (result.indexId == scan.getTable().getBaseIndexId()) { - return ctx.root; - } - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); @@ -389,10 +369,6 @@ public List buildRules() { nonVirtualGroupByExprs(agg), new HashSet<>(agg.getExpressions())); - if (result.indexId == scan.getTable().getBaseIndexId()) { - return ctx.root; - } - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); @@ -446,10 +422,6 @@ public List buildRules() { requiredExpr ); - if (result.indexId == scan.getTable().getBaseIndexId()) { - return ctx.root; - } - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); @@ -502,10 +474,6 @@ public List buildRules() { collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects()) ); - if (result.indexId == scan.getTable().getBaseIndexId()) { - return ctx.root; - } - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); @@ -571,10 +539,6 @@ public List buildRules() { requiredExpr ); - if (result.indexId == scan.getTable().getBaseIndexId()) { - return ctx.root; - } - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); @@ -641,10 +605,6 @@ public List buildRules() { requiredExpr ); - if (result.indexId == scan.getTable().getBaseIndexId()) { - return ctx.root; - } - LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); diff --git a/regression-test/data/mv_p0/ut/testProjectionMV1/testProjectionMV1.out b/regression-test/data/mv_p0/ut/testProjectionMV1/testProjectionMV1.out index 523176bb0a5fdd..ba455b7e850d3c 100644 --- a/regression-test/data/mv_p0/ut/testProjectionMV1/testProjectionMV1.out +++ b/regression-test/data/mv_p0/ut/testProjectionMV1/testProjectionMV1.out @@ -9,11 +9,3 @@ 1 1 2 2 --- !select_mv -- -1 2 -2 2 - --- !select_mv -- -1 2 -2 2 - diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy index 329e54a1f2dcaf..c23b533832f2c0 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy @@ -34,11 +34,6 @@ suite ("testProjectionMV1") { sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" - test { - sql "create materialized view emps_mv as select deptno, empid from emps t order by deptno;" - exception "errCode = 2," - } - createMV("create materialized view emps_mv as select deptno, empid from emps order by deptno;") sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" @@ -55,16 +50,4 @@ suite ("testProjectionMV1") { contains "(emps_mv)" } qt_select_mv "select empid, deptno from emps order by empid;" - - explain { - sql("select empid, sum(deptno) from emps group by empid order by empid;") - contains "(emps_mv)" - } - qt_select_mv "select empid, sum(deptno) from emps group by empid order by empid;" - - explain { - sql("select deptno, sum(empid) from emps group by deptno order by deptno;") - contains "(emps_mv)" - } - qt_select_mv "select deptno, sum(empid) from emps group by deptno order by deptno;" } From 60ba4360eacefadd888cfd34f712f242300c58a6 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Tue, 31 Oct 2023 21:31:01 +0800 Subject: [PATCH 7/7] [fix](Nereids) ut failed intro by PR #25951 --- .../java/org/apache/doris/catalog/Column.java | 4 +- .../doris/analysis/AddColumnsClauseTest.java | 12 +-- .../apache/doris/analysis/ColumnDefTest.java | 10 +-- .../analysis/CreateTableAsSelectStmtTest.java | 78 +++++++++---------- .../doris/analysis/CreateTableStmtTest.java | 26 +++---- .../analysis/ShowCreateTableStmtTest.java | 2 +- .../apache/doris/catalog/ColumnTypeTest.java | 14 ++-- .../external/elasticsearch/EsUtilTest.java | 18 ++--- .../doris/load/loadv2/TokenManagerTest.java | 2 + .../apache/doris/planner/QueryPlanTest.java | 4 +- .../doris/planner/TableFunctionPlanTest.java | 38 ++++----- 11 files changed, 105 insertions(+), 103 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 58b1e1dc5dc9d9..3e741dd08edc00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -723,12 +723,12 @@ public String toSql(boolean isUniqueTable, boolean isCompatible) { // show change datetimeV2/dateV2 to datetime/date if (isCompatible) { if (type.isDatetimeV2()) { - sb.append("datetime"); + sb.append("DATETIME"); if (((ScalarType) type).getScalarScale() > 0) { sb.append("(").append(((ScalarType) type).getScalarScale()).append(")"); } } else if (type.isDateV2()) { - sb.append("date"); + sb.append("DATE"); } else if (type.isDecimalV3()) { sb.append("DECIMAL"); ScalarType sType = (ScalarType) type; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnsClauseTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnsClauseTest.java index 53ce6a81ccda75..06a4fe601381f6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnsClauseTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnsClauseTest.java @@ -48,21 +48,21 @@ public void testNormal() throws AnalysisException { columns.add(definition); AddColumnsClause clause = new AddColumnsClause(columns, null, null); clause.analyze(analyzer); - Assert.assertEquals("ADD COLUMN (`col1` int(11) NOT NULL DEFAULT \"0\" COMMENT \"\", " - + "`col2` int(11) NOT NULL DEFAULT \"0\" COMMENT \"\")", clause.toString()); + Assert.assertEquals("ADD COLUMN (`col1` INT NOT NULL DEFAULT \"0\" COMMENT \"\", " + + "`col2` INT NOT NULL DEFAULT \"0\" COMMENT \"\")", clause.toString()); clause = new AddColumnsClause(columns, "", null); clause.analyze(analyzer); - Assert.assertEquals("ADD COLUMN (`col1` int(11) NOT NULL DEFAULT \"0\" COMMENT \"\", " - + "`col2` int(11) NOT NULL DEFAULT \"0\" COMMENT \"\")", + Assert.assertEquals("ADD COLUMN (`col1` INT NOT NULL DEFAULT \"0\" COMMENT \"\", " + + "`col2` INT NOT NULL DEFAULT \"0\" COMMENT \"\")", clause.toString()); Assert.assertNull(clause.getRollupName()); clause = new AddColumnsClause(columns, "testTable", null); clause.analyze(analyzer); - Assert.assertEquals("ADD COLUMN (`col1` int(11) NOT NULL DEFAULT \"0\" COMMENT \"\", " - + "`col2` int(11) NOT NULL DEFAULT \"0\" COMMENT \"\") IN `testTable`", + Assert.assertEquals("ADD COLUMN (`col1` INT NOT NULL DEFAULT \"0\" COMMENT \"\", " + + "`col2` INT NOT NULL DEFAULT \"0\" COMMENT \"\") IN `testTable`", clause.toString()); Assert.assertNull(clause.getProperties()); Assert.assertEquals("testTable", clause.getRollupName()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java index d969be252032f2..fad5787a37bc79 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java @@ -61,7 +61,7 @@ public void testNormal() throws AnalysisException { ColumnDef column = new ColumnDef("col", intCol); column.analyze(true); - Assert.assertEquals("`col` int(11) NOT NULL COMMENT \"\"", column.toString()); + Assert.assertEquals("`col` INT NOT NULL COMMENT \"\"", column.toString()); Assert.assertEquals("col", column.getName()); Assert.assertEquals(PrimitiveType.INT, column.getType().getPrimitiveType()); Assert.assertNull(column.getAggregateType()); @@ -72,14 +72,14 @@ public void testNormal() throws AnalysisException { column.analyze(true); Assert.assertNull(column.getAggregateType()); Assert.assertEquals("10", column.getDefaultValue()); - Assert.assertEquals("`col` int(11) NOT NULL DEFAULT \"10\" COMMENT \"\"", column.toSql()); + Assert.assertEquals("`col` INT NOT NULL DEFAULT \"10\" COMMENT \"\"", column.toSql()); // agg column = new ColumnDef("col", floatCol, false, AggregateType.SUM, false, new DefaultValue(true, "10"), ""); column.analyze(true); Assert.assertEquals("10", column.getDefaultValue()); Assert.assertEquals(AggregateType.SUM, column.getAggregateType()); - Assert.assertEquals("`col` float SUM NOT NULL DEFAULT \"10\" COMMENT \"\"", column.toSql()); + Assert.assertEquals("`col` FLOAT SUM NOT NULL DEFAULT \"10\" COMMENT \"\"", column.toSql()); } @Test @@ -89,14 +89,14 @@ public void testReplaceIfNotNull() throws AnalysisException { ColumnDef column = new ColumnDef("col", intCol, false, AggregateType.REPLACE_IF_NOT_NULL, false, DefaultValue.NOT_SET, ""); column.analyze(true); Assert.assertEquals(AggregateType.REPLACE_IF_NOT_NULL, column.getAggregateType()); - Assert.assertEquals("`col` int(11) REPLACE_IF_NOT_NULL NULL DEFAULT \"null\" COMMENT \"\"", column.toSql()); + Assert.assertEquals("`col` INT REPLACE_IF_NOT_NULL NULL DEFAULT \"null\" COMMENT \"\"", column.toSql()); } // CHECKSTYLE IGNORE THIS LINE { // CHECKSTYLE IGNORE THIS LINE // not allow null ColumnDef column = new ColumnDef("col", intCol, false, AggregateType.REPLACE_IF_NOT_NULL, false, new DefaultValue(true, "10"), ""); column.analyze(true); Assert.assertEquals(AggregateType.REPLACE_IF_NOT_NULL, column.getAggregateType()); - Assert.assertEquals("`col` int(11) REPLACE_IF_NOT_NULL NULL DEFAULT \"10\" COMMENT \"\"", column.toSql()); + Assert.assertEquals("`col` INT REPLACE_IF_NOT_NULL NULL DEFAULT \"10\" COMMENT \"\"", column.toSql()); } // CHECKSTYLE IGNORE THIS LINE } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 664b5e55149258..86db19e1f8dd04 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -83,7 +83,7 @@ public void testDecimal() throws Exception { + "as select * from `test`.`decimal_table`"; createTableAsSelect(selectFromDecimal); Assertions.assertEquals("CREATE TABLE `select_decimal_table` (\n" - + " `userId` varchar(255) NOT NULL,\n" + + " `userId` VARCHAR(255) NOT NULL,\n" + " `amount_decimal` " + "DECIMAL" + "(10, 2) NOT NULL\n" + ") ENGINE=OLAP\n" @@ -160,8 +160,8 @@ public void testVarchar() throws Exception { createTableAsSelect(selectFromVarchar); ShowResultSet showResultSet = showCreateTableByName("select_varchar"); Assertions.assertEquals("CREATE TABLE `select_varchar` (\n" - + " `userId` varchar(255) NOT NULL,\n" - + " `username` varchar(255) NOT NULL\n" + + " `userId` VARCHAR(255) NOT NULL,\n" + + " `username` VARCHAR(255) NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" @@ -186,7 +186,7 @@ public void testFunction() throws Exception { ShowResultSet showResultSet1 = showCreateTableByName("select_function_1"); Assertions.assertEquals( "CREATE TABLE `select_function_1` (\n" - + " `__count_0` bigint(20) NULL\n" + + " `__count_0` BIGINT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`__count_0`)\n" + "COMMENT 'OLAP'\n" @@ -209,11 +209,11 @@ public void testFunction() throws Exception { ShowResultSet showResultSet2 = showCreateTableByName("select_function_2"); Assertions.assertEquals( "CREATE TABLE `select_function_2` (\n" - + " `__sum_0` bigint(20) NULL,\n" - + " `__sum_1` bigint(20) NULL,\n" - + " `__sum_2` bigint(20) NULL,\n" - + " `__count_3` bigint(20) NULL,\n" - + " `__count_4` bigint(20) NULL\n" + + " `__sum_0` BIGINT NULL,\n" + + " `__sum_1` BIGINT NULL,\n" + + " `__sum_2` BIGINT NULL,\n" + + " `__count_3` BIGINT NULL,\n" + + " `__count_4` BIGINT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`__sum_0`, `__sum_1`, `__sum_2`)\n" + "COMMENT 'OLAP'\n" @@ -237,7 +237,7 @@ public void testAlias() throws Exception { createTableAsSelect(selectAlias1); ShowResultSet showResultSet1 = showCreateTableByName("select_alias_1"); Assertions.assertEquals("CREATE TABLE `select_alias_1` (\n" - + " `amount` bigint(20) NULL\n" + + " `amount` BIGINT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`amount`)\n" + "COMMENT 'OLAP'\n" @@ -256,8 +256,8 @@ public void testAlias() throws Exception { createTableAsSelect(selectAlias2); ShowResultSet showResultSet2 = showCreateTableByName("select_alias_2"); Assertions.assertEquals("CREATE TABLE `select_alias_2` (\n" - + " `alias_name` varchar(255) NOT NULL,\n" - + " `username` varchar(255) NOT NULL\n" + + " `alias_name` VARCHAR(255) NOT NULL,\n" + + " `username` VARCHAR(255) NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`alias_name`)\n" + "COMMENT 'OLAP'\n" @@ -282,9 +282,9 @@ public void testJoin() throws Exception { createTableAsSelect(selectFromJoin); ShowResultSet showResultSet = showCreateTableByName("select_join"); Assertions.assertEquals("CREATE TABLE `select_join` (\n" - + " `userId` varchar(255) NOT NULL,\n" - + " `username` varchar(255) NOT NULL,\n" - + " `status` int(11) NOT NULL\n" + + " `userId` VARCHAR(255) NOT NULL,\n" + + " `username` VARCHAR(255) NOT NULL,\n" + + " `status` INT NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" @@ -305,10 +305,10 @@ public void testJoin() throws Exception { createTableAsSelect(selectFromJoin1); ShowResultSet showResultSet1 = showCreateTableByName("select_join1"); Assertions.assertEquals("CREATE TABLE `select_join1` (\n" - + " `userId1` varchar(255) NOT NULL,\n" - + " `userId2` varchar(255) NOT NULL,\n" - + " `username` varchar(255) NOT NULL,\n" - + " `status` int(11) NOT NULL\n" + + " `userId1` VARCHAR(255) NOT NULL,\n" + + " `userId2` VARCHAR(255) NOT NULL,\n" + + " `username` VARCHAR(255) NOT NULL,\n" + + " `status` INT NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId1`)\n" + "COMMENT 'OLAP'\n" @@ -334,9 +334,9 @@ public void testName() throws Exception { createTableAsSelect(selectFromName); ShowResultSet showResultSet = showCreateTableByName("select_name"); Assertions.assertEquals("CREATE TABLE `select_name` (\n" - + " `user` varchar(255) NOT NULL,\n" - + " `testname` varchar(255) NOT NULL,\n" - + " `userstatus` int(11) NOT NULL\n" + + " `user` VARCHAR(255) NOT NULL,\n" + + " `testname` VARCHAR(255) NOT NULL,\n" + + " `userstatus` INT NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`user`)\n" + "COMMENT 'OLAP'\n" @@ -361,7 +361,7 @@ public void testUnion() throws Exception { ShowResultSet showResultSet = showCreateTableByName("select_union"); Assertions.assertEquals( "CREATE TABLE `select_union` (\n" - + " `userId` varchar(255) NULL\n" + + " `userId` VARCHAR(255) NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" @@ -385,7 +385,7 @@ public void testCte() throws Exception { ShowResultSet showResultSet = showCreateTableByName("select_cte"); Assertions.assertEquals( "CREATE TABLE `select_cte` (\n" - + " `userId` varchar(255) NOT NULL\n" + + " `userId` VARCHAR(255) NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" @@ -405,7 +405,7 @@ public void testCte() throws Exception { createTableAsSelect(selectFromCteAndUnion); ShowResultSet showResultSet1 = showCreateTableByName("select_cte_union"); Assertions.assertEquals("CREATE TABLE `select_cte_union` (\n" - + " `id` tinyint(4) NULL\n" + + " `id` TINYINT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`id`)\n" + "COMMENT 'OLAP'\n" @@ -429,8 +429,8 @@ public void testPartition() throws Exception { createTableAsSelect(selectFromPartition); ShowResultSet showResultSet = showCreateTableByName("selectPartition"); Assertions.assertEquals("CREATE TABLE `selectPartition` (\n" - + " `userId` varchar(255) NOT NULL,\n" - + " `username` varchar(255) NOT NULL\n" + + " `userId` VARCHAR(255) NOT NULL,\n" + + " `username` VARCHAR(255) NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" @@ -456,8 +456,8 @@ public void testDefaultTimestamp() throws Exception { createTableAsSelect(createSql); ShowResultSet showResultSet = showCreateTableByName("test_default_timestamp"); Assertions.assertEquals("CREATE TABLE `test_default_timestamp` (\n" - + " `userId` varchar(255) NOT NULL,\n" - + " `date` datetime" + + " `userId` VARCHAR(255) NOT NULL,\n" + + " `date` DATETIME" + " NULL DEFAULT CURRENT_TIMESTAMP\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n" @@ -483,7 +483,7 @@ public void testAggValue() throws Exception { ShowResultSet showResultSet = showCreateTableByName("test_agg_value"); Assertions.assertEquals( "CREATE TABLE `test_agg_value` (\n" - + " `username` varchar(255) NOT NULL\n" + + " `username` VARCHAR(255) NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`username`)\n" + "COMMENT 'OLAP'\n" @@ -508,8 +508,8 @@ public void testUseKeyType() throws Exception { ShowResultSet showResultSet = showCreateTableByName("test_use_key_type"); Assertions.assertEquals( "CREATE TABLE `test_use_key_type` (\n" - + " `userId` varchar(255) NOT NULL,\n" - + " `username` varchar(255) NOT NULL\n" + + " `userId` VARCHAR(255) NOT NULL,\n" + + " `username` VARCHAR(255) NOT NULL\n" + ") ENGINE=OLAP\n" + "UNIQUE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" @@ -558,8 +558,8 @@ public void testQuerySchema() throws Exception { createStmts.add(createTableStmts.get(0)); if (tbl.getName().equals("qs1")) { Assert.assertEquals("CREATE TABLE `qs1` (\n" - + " `k1` int(11) NULL,\n" - + " `k2` int(11) NULL\n" + + " `k1` INT NULL,\n" + + " `k2` INT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`k1`, `k2`)\n" + "COMMENT 'OLAP'\n" @@ -576,8 +576,8 @@ public void testQuerySchema() throws Exception { createTableStmts.get(0)); } else { Assert.assertEquals("CREATE TABLE `qs2` (\n" - + " `k1` int(11) NULL,\n" - + " `k2` int(11) NULL\n" + + " `k1` INT NULL,\n" + + " `k2` INT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`k1`, `k2`)\n" + "COMMENT 'OLAP'\n" @@ -607,9 +607,9 @@ public void testVarcharLength() throws Exception { String showStr = showResultSet.getResultRows().get(0).get(1); Assertions.assertEquals( "CREATE TABLE `varchar_len1` (\n" - + " `__literal_0` varchar(*) NULL,\n" - + " `__concat_1` varchar(*) NULL,\n" - + " `userId` varchar(255) NOT NULL\n" + + " `__literal_0` VARCHAR(*) NULL,\n" + + " `__concat_1` VARCHAR(*) NULL,\n" + + " `userId` VARCHAR(255) NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`__literal_0`)\n" + "COMMENT 'OLAP'\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java index b7bdfed3ffd019..d856599a3d7b9a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java @@ -337,7 +337,7 @@ public void testHllNoAggTab() throws Exception { null, null, ""); expectedEx.expect(AnalysisException.class); expectedEx.expectMessage( - "Aggregate type `col3` hll NONE NOT NULL COMMENT \"\" is not compatible with primitive type hll"); + "Aggregate type `col3` HLL NONE NOT NULL COMMENT \"\" is not compatible with primitive type HLL"); stmt.analyze(analyzer); } @@ -419,8 +419,8 @@ public void testCreateHudiTableWithSchema() throws UserException { stmt.analyze(analyzer); Assert.assertEquals( - "CREATE EXTERNAL TABLE `testCluster:db1`.`table1` (\n" + " `id` int(11) NOT NULL COMMENT \"\",\n" - + " `name` int(11) NULL COMMENT \"\"\n" + ") ENGINE = hudi\n" + "CREATE EXTERNAL TABLE `testCluster:db1`.`table1` (\n" + " `id` INT NOT NULL COMMENT \"\",\n" + + " `name` INT NULL COMMENT \"\"\n" + ") ENGINE = hudi\n" + "PROPERTIES (\"hudi.database\" = \"doris\",\n" + "\"hudi.hive.metastore.uris\" = \"thrift://127.0.0.1:9087\",\n" + "\"hudi.table\" = \"test\")", stmt.toString()); @@ -455,8 +455,8 @@ public void testToSql() { properties, null, "", null); String createTableSql = "CREATE TABLE IF NOT EXISTS `demo`.`testTosql1` (\n" - + " `a` bigint(20) NOT NULL COMMENT \"\",\n" - + " `b` int(11) NOT NULL COMMENT \"\"\n" + + " `a` BIGINT NOT NULL COMMENT \"\",\n" + + " `b` INT NOT NULL COMMENT \"\"\n" + ") ENGINE = olap\n" + "AGGREGATE KEY(`a`)\n" + "PROPERTIES (\"replication_num\" = \"1\")"; @@ -484,14 +484,14 @@ public void testToSql() { tableName, columnDefs, engineName, keysDesc, null, null, properties, null, "", null); createTableSql = "CREATE TABLE `demo`.`testTosql2` (\n" - + " `a` bigint(20) NOT NULL COMMENT \"\",\n" - + " `b` int(11) NOT NULL COMMENT \"\",\n" - + " `c` text NULL COMMENT \"\",\n" - + " `d` double NULL COMMENT \"\",\n" - + " `e` decimalv3(38, 0) NOT NULL COMMENT \"\",\n" - + " `f` date NOT NULL COMMENT \"\",\n" - + " `g` smallint(6) NOT NULL COMMENT \"\",\n" - + " `h` boolean NOT NULL COMMENT \"\"\n" + + " `a` BIGINT NOT NULL COMMENT \"\",\n" + + " `b` INT NOT NULL COMMENT \"\",\n" + + " `c` TEXT NULL COMMENT \"\",\n" + + " `d` DOUBLE NULL COMMENT \"\",\n" + + " `e` DECIMALV3(38, 0) NOT NULL COMMENT \"\",\n" + + " `f` DATE NOT NULL COMMENT \"\",\n" + + " `g` SMALLINT NOT NULL COMMENT \"\",\n" + + " `h` BOOLEAN NOT NULL COMMENT \"\"\n" + ") ENGINE = olap\n" + "DUPLICATE KEY(`a`, `d`, `f`)\n" + "PROPERTIES (\"replication_num\" = \"10\")"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java index fe9adf9ccd0c31..0a5653b7f9ed34 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java @@ -48,7 +48,7 @@ public void testNormal() throws Exception { String sql = "show create table table1"; ShowResultSet showResultSet = showCreateTable(sql); String showSql = showResultSet.getResultRows().get(0).get(1); - Assertions.assertTrue(showSql.contains("`k1` int(11) NULL COMMENT 'test column k1'")); + Assertions.assertTrue(showSql.contains("`k1` INT NULL COMMENT 'test column k1'")); Assertions.assertTrue(showSql.contains("COMMENT 'test table1'")); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTypeTest.java index 87503b3a84044c..9cacbe7a5f6654 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTypeTest.java @@ -48,7 +48,7 @@ public void testPrimitiveType() throws AnalysisException { type.analyze(null); Assert.assertEquals(PrimitiveType.INT, type.getType().getPrimitiveType()); - Assert.assertEquals("int(11)", type.toSql()); + Assert.assertEquals("INT", type.toSql()); // equal type TypeDef type2 = TypeDef.create(PrimitiveType.INT); @@ -69,9 +69,9 @@ public void testInvalidType() throws AnalysisException { public void testCharType() throws AnalysisException { TypeDef type = TypeDef.createVarchar(10); type.analyze(null); - Assert.assertEquals("varchar(10)", type.toString()); + Assert.assertEquals("VARCHAR(10)", type.toString()); Assert.assertEquals(PrimitiveType.VARCHAR, type.getType().getPrimitiveType()); - Assert.assertEquals(10, ((ScalarType) type.getType()).getLength()); + Assert.assertEquals(10, type.getType().getLength()); // equal type TypeDef type2 = TypeDef.createVarchar(10); @@ -91,10 +91,10 @@ public void testDecimal() throws AnalysisException { TypeDef type = TypeDef.createDecimal(12, 5); type.analyze(null); if (Config.enable_decimal_conversion) { - Assert.assertEquals("decimalv3(12, 5)", type.toString()); + Assert.assertEquals("DECIMALV3(12, 5)", type.toString()); Assert.assertEquals(PrimitiveType.DECIMAL64, type.getType().getPrimitiveType()); } else { - Assert.assertEquals("decimal(12, 5)", type.toString()); + Assert.assertEquals("DECIMAL(12, 5)", type.toString()); Assert.assertEquals(PrimitiveType.DECIMALV2, type.getType().getPrimitiveType()); } Assert.assertEquals(12, ((ScalarType) type.getType()).getScalarPrecision()); @@ -119,7 +119,7 @@ public void testDecimal() throws AnalysisException { public void testDatetimeV2() throws AnalysisException { TypeDef type = TypeDef.createDatetimeV2(3); type.analyze(null); - Assert.assertEquals("datetimev2(3)", type.toString()); + Assert.assertEquals("DATETIMEV2(3)", type.toString()); Assert.assertEquals(PrimitiveType.DATETIMEV2, type.getType().getPrimitiveType()); Assert.assertEquals(ScalarType.DATETIME_PRECISION, ((ScalarType) type.getType()).getScalarPrecision()); Assert.assertEquals(3, ((ScalarType) type.getType()).getScalarScale()); @@ -160,7 +160,7 @@ public void testDateV2() throws AnalysisException { public void testTimeV2() throws AnalysisException { TypeDef type = TypeDef.createTimeV2(3); type.analyze(null); - Assert.assertEquals("time(3)", type.toString()); + Assert.assertEquals("TIME(3)", type.toString()); Assert.assertEquals(PrimitiveType.TIMEV2, type.getType().getPrimitiveType()); Assert.assertEquals(ScalarType.DATETIME_PRECISION, ((ScalarType) type.getType()).getScalarPrecision()); Assert.assertEquals(3, ((ScalarType) type.getType()).getScalarScale()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java index 1cdf1cb94df9d2..025876fd06effb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java @@ -227,25 +227,25 @@ public void testDateType() throws IOException, URISyntaxException { String name = column.getName(); String type = column.getType().toSql(); if ("test2".equals(name)) { - Assertions.assertEquals("datetimev2(0)", type); + Assertions.assertEquals("DATETIMEV2(0)", type); } if ("test3".equals(name)) { - Assertions.assertEquals("datetimev2(0)", type); + Assertions.assertEquals("DATETIMEV2(0)", type); } if ("test4".equals(name)) { - Assertions.assertEquals("datev2", type); + Assertions.assertEquals("DATEV2", type); } if ("test5".equals(name)) { - Assertions.assertEquals("datetimev2(0)", type); + Assertions.assertEquals("DATETIMEV2(0)", type); } if ("test6".equals(name)) { - Assertions.assertEquals("datev2", type); + Assertions.assertEquals("DATEV2", type); } if ("test7".equals(name)) { - Assertions.assertEquals("datetimev2(0)", type); + Assertions.assertEquals("DATETIMEV2(0)", type); } if ("test8".equals(name)) { - Assertions.assertEquals("bigint(20)", type); + Assertions.assertEquals("BIGINT", type); } } } @@ -255,8 +255,8 @@ public void testFieldAlias() throws IOException, URISyntaxException { ObjectNode testFieldAlias = EsUtil.getRootSchema( EsUtil.getMapping(loadJsonFromFile("data/es/test_field_alias.json")), null, new ArrayList<>()); List parseColumns = EsUtil.genColumnsFromEs("test_field_alias", null, testFieldAlias, true, new ArrayList<>()); - Assertions.assertEquals("datetimev2(0)", parseColumns.get(2).getType().toSql()); - Assertions.assertEquals("text", parseColumns.get(4).getType().toSql()); + Assertions.assertEquals("DATETIMEV2(0)", parseColumns.get(2).getType().toSql()); + Assertions.assertEquals("TEXT", parseColumns.get(4).getType().toSql()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/TokenManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/TokenManagerTest.java index b592b9df930f74..13ae9b6e44ea40 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/TokenManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/TokenManagerTest.java @@ -34,6 +34,7 @@ public void runBefore() { @Test public void testTokenCheck() throws UserException { TokenManager tokenManager = new TokenManager(); + tokenManager.start(); String token = tokenManager.acquireToken(); Assert.assertTrue(tokenManager.checkAuthToken(token)); } @@ -41,6 +42,7 @@ public void testTokenCheck() throws UserException { @Test public void testSameToken() throws UserException { TokenManager tokenManager = new TokenManager(); + tokenManager.start(); String token1 = tokenManager.acquireToken(); String token2 = tokenManager.acquireToken(); Assert.assertNotNull(token1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index 7bfa8a9227ed60..2a6367ef3c941b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -573,11 +573,11 @@ public void testTypeCast() throws Exception { // disable implicit cast hll/bitmap to string assertSQLPlanOrErrorMsgContains( "select length(id2) from test.hll_table;", - "No matching function with signature: length(hll)" + "No matching function with signature: length(HLL)" ); assertSQLPlanOrErrorMsgContains( "select length(id2) from test.bitmap_table;", - "No matching function with signature: length(bitmap)" + "No matching function with signature: length(BITMAP)" ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java index 52fa95486eabc6..9edfa976430895 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java @@ -79,7 +79,7 @@ public void normalTableFunction() throws Exception { explainString.contains("table function: explode_split(`default_cluster:db1`.`tbl1`.`k2`, ',')")); Assert.assertTrue(explainString.contains("tuple ids: 0 1")); Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=32}")); - Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=varchar")); + Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=VARCHAR")); } /* Case2 without output explode column @@ -95,7 +95,7 @@ public void withoutOutputExplodeColumn() throws Exception { explainString.contains("table function: explode_split(`default_cluster:db1`.`tbl1`.`k2`, ',')")); Assert.assertTrue(explainString.contains("tuple ids: 0 1")); Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=32}")); - Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=varchar")); + Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=VARCHAR")); } /* Case3 group by explode column @@ -116,7 +116,7 @@ public void groupByExplodeColumn() throws Exception { explainString.contains("table function: explode_split(`default_cluster:db1`.`tbl1`.`k2`, ',')")); Assert.assertTrue(explainString.contains("tuple ids: 0 1")); Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=32}")); - Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=varchar")); + Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=VARCHAR")); // group by tuple Assert.assertTrue(explainString.contains("TupleDescriptor{id=2, tbl=null, byteSize=32}")); } @@ -135,7 +135,7 @@ public void whereExplodeColumn() throws Exception { Assert.assertTrue(explainString.contains("PREDICATES: `e1` = '1'")); Assert.assertTrue(explainString.contains("tuple ids: 0 1")); Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=32}")); - Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=varchar")); + Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=VARCHAR")); } /* Case5 where normal column @@ -151,7 +151,7 @@ public void whereNormalColumn() throws Exception { explainString.contains("table function: explode_split(`default_cluster:db1`.`tbl1`.`k2`, ',')")); Assert.assertTrue(explainString.contains("tuple ids: 0 1")); Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=32}")); - Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=varchar")); + Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=VARCHAR")); Assert.assertTrue(UtFrameUtils.checkPlanResultContainsNode(explainString, 0, "OlapScanNode")); Assert.assertTrue(explainString.contains("PREDICATES: `k1` = 1")); } @@ -171,10 +171,10 @@ public void testMultiLateralView() throws Exception { Assert.assertTrue(explainString.contains("lateral view tuple id: 1 2")); // lateral view 2 tuple Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp2, byteSize=32}")); - Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e2, colUniqueId=-1, type=varchar")); + Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e2, colUniqueId=-1, type=VARCHAR")); // lateral view 1 tuple Assert.assertTrue(explainString.contains("TupleDescriptor{id=2, tbl=tmp1, byteSize=32}")); - Assert.assertTrue(explainString.contains("SlotDescriptor{id=2, col=e1, colUniqueId=-1, type=varchar")); + Assert.assertTrue(explainString.contains("SlotDescriptor{id=2, col=e1, colUniqueId=-1, type=VARCHAR")); } // test explode_split function @@ -188,11 +188,11 @@ public void testMultiLateralView() throws Exception { public void errorParam() throws Exception { String sql = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1, e1 from db1.tbl1 lateral view explode_split(k2) tmp as e1;"; String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql); - Assert.assertTrue(explainString.contains("No matching function with signature: explode_split(varchar(1))")); + Assert.assertTrue(explainString.contains("No matching function with signature: explode_split(VARCHAR(1))")); sql = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1, e1 from db1.tbl1 lateral view explode_split(k1) tmp as e1;"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql); - Assert.assertTrue(explainString.contains("No matching function with signature: explode_split(int(11))")); + Assert.assertTrue(explainString.contains("No matching function with signature: explode_split(INT)")); } /* Case2 table function in where stmt @@ -203,7 +203,7 @@ public void tableFunctionInWhere() throws Exception { String sql = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where explode_split(k2, \",\");"; String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql); Assert.assertTrue(explainString, - explainString.contains("No matching function with signature: explode_split(varchar(1), varchar(*)).")); + explainString.contains("No matching function with signature: explode_split(VARCHAR(1), VARCHAR(*)).")); } // test projection @@ -350,8 +350,8 @@ public void scalarFunctionInLateralView() throws Exception { explainString.contains("table function: explode_split(concat(`a`.`k2`, ',', `a`.`k3`), ',')")); Assert.assertTrue(explainString.contains("lateral view tuple id: 1")); Assert.assertTrue(explainString.contains("output slot id: 3")); - Assert.assertTrue(explainString.contains("SlotDescriptor{id=0, col=k2, colUniqueId=1, type=varchar(1)")); - Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=k3, colUniqueId=2, type=varchar(1)")); + Assert.assertTrue(explainString.contains("SlotDescriptor{id=0, col=k2, colUniqueId=1, type=VARCHAR(1)")); + Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=k3, colUniqueId=2, type=VARCHAR(1)")); } // lateral view of subquery @@ -368,7 +368,7 @@ public void lateralViewColumnOfReduceTuple() throws Exception { Assert.assertTrue(explainString.contains("lateral view tuple id: 2")); Assert.assertTrue(explainString.contains("output slot id: 2")); Assert.assertTrue(explainString.contains("tuple ids: 0 2")); - Assert.assertTrue(explainString.contains("SlotDescriptor{id=2, col=e1, colUniqueId=-1, type=varchar")); + Assert.assertTrue(explainString.contains("SlotDescriptor{id=2, col=e1, colUniqueId=-1, type=VARCHAR")); } /* @@ -384,7 +384,7 @@ public void aggInlineView() throws Exception { Assert.assertTrue(explainString.contains("lateral view tuple id: 3")); Assert.assertTrue(explainString.contains("output slot id: 3")); Assert.assertTrue(explainString.contains("tuple ids: 1 3")); - Assert.assertTrue(explainString.contains("SlotDescriptor{id=3, col=e1, colUniqueId=-1, type=varchar")); + Assert.assertTrue(explainString.contains("SlotDescriptor{id=3, col=e1, colUniqueId=-1, type=VARCHAR")); } /* @@ -403,19 +403,19 @@ public void aggColumnInlineViewInTB() throws Exception { Assert.assertTrue(explainString.contains("tuple ids: 1 3")); String formatString = explainString.replaceAll(" ", ""); Assert.assertTrue(formatString.contains( - "SlotDescriptor{id=0,col=k1,colUniqueId=0,type=int" + "SlotDescriptor{id=0,col=k1,colUniqueId=0,type=INT" )); Assert.assertTrue(formatString.contains( - "SlotDescriptor{id=1,col=k2,colUniqueId=1,type=varchar(1)" + "SlotDescriptor{id=1,col=k2,colUniqueId=1,type=VARCHAR(1)" )); Assert.assertTrue(formatString.contains( - "SlotDescriptor{id=2,col=k1,colUniqueId=0,type=int" + "SlotDescriptor{id=2,col=k1,colUniqueId=0,type=INT" )); Assert.assertTrue(formatString.contains( - "SlotDescriptor{id=3,col=null,colUniqueId=null,type=varchar" + "SlotDescriptor{id=3,col=null,colUniqueId=null,type=VARCHAR" )); Assert.assertTrue(formatString.contains( - "SlotDescriptor{id=6,col=e1,colUniqueId=-1,type=varchar" + "SlotDescriptor{id=6,col=e1,colUniqueId=-1,type=VARCHAR" )); }