diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java index 33701b85177358..4158f47254f6cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java @@ -844,7 +844,7 @@ public void updateMaterializedSlots() { // why output and intermediate may have different materialized slots? // because some slot is materialized by materializeSrcExpr method directly // in that case, only output slots is materialized - // assume output tuple has correct marterialized infomation + // assume output tuple has correct materialized information // we update intermediate tuple and materializedSlots based on output tuple materializedSlots.clear(); ArrayList outputSlots = outputTupleDesc.getSlots(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index b9548b0daa718b..bdc463662b1d7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -1671,7 +1671,11 @@ public PlanFragment visitPhysicalGenerate(PhysicalGenerate gener .map(e -> ExpressionTranslator.translate(e, context)) .collect(Collectors.toCollection(ArrayList::new)); TupleDescriptor tupleDescriptor = generateTupleDesc(generate.getGeneratorOutput(), null, context); - List outputSlotIds = Stream.concat(currentFragment.getPlanRoot().getTupleIds().stream(), + List childOutputTupleIds = currentFragment.getPlanRoot().getOutputTupleIds(); + if (childOutputTupleIds == null || childOutputTupleIds.isEmpty()) { + childOutputTupleIds = currentFragment.getPlanRoot().getTupleIds(); + } + List outputSlotIds = Stream.concat(childOutputTupleIds.stream(), Stream.of(tupleDescriptor.getId())) .map(id -> context.getTupleDesc(id).getSlots()) .flatMap(List::stream) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java index 155a1cffa1e5e9..d5939e61f3c2d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.RewriteJob; +import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; @@ -56,6 +57,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; import org.apache.doris.nereids.rules.rewrite.logical.MergeSetOperations; import org.apache.doris.nereids.rules.rewrite.logical.NormalizeAggregate; +import org.apache.doris.nereids.rules.rewrite.logical.NormalizeSort; import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanPartition; import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanTablet; import org.apache.doris.nereids.rules.rewrite.logical.PushFilterInsideJoin; @@ -63,6 +65,8 @@ import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin; import org.apache.doris.nereids.rules.rewrite.logical.SplitLimit; +import com.google.common.collect.ImmutableList; + import java.util.List; /** @@ -89,7 +93,7 @@ public class NereidsRewriter extends BatchRewriteJob { // ExtractSingleTableExpressionFromDisjunction conflict to InPredicateToEqualToRule // in the ExpressionNormalization, so must invoke in another job, or else run into - // deep loop + // dead loop topDown( new ExtractSingleTableExpressionFromDisjunction() ) @@ -117,10 +121,11 @@ public class NereidsRewriter extends BatchRewriteJob { // The rule modification needs to be done after the subquery is unnested, // because for scalarSubQuery, the connection condition is stored in apply in the analyzer phase, - // but when normalizeAggregate is performed, the members in apply cannot be obtained, + // but when normalizeAggregate/normalizeSort is performed, the members in apply cannot be obtained, // resulting in inconsistent output results and results in apply topDown( - new NormalizeAggregate() + new NormalizeAggregate(), + new NormalizeSort() ), topDown( @@ -138,50 +143,50 @@ public class NereidsRewriter extends BatchRewriteJob { ), topic("Rewrite join", - // infer not null filter, then push down filter, and then reorder join(cross join to inner join) - topDown( - new InferFilterNotNull(), - new InferJoinNotNull() - ), - // ReorderJoin depends PUSH_DOWN_FILTERS - // the PUSH_DOWN_FILTERS depends on lots of rules, e.g. merge project, eliminate outer, - // sometimes transform the bottom plan make some rules usable which can apply to the top plan, - // but top-down traverse can not cover this case in one iteration, so bottom-up is more - // efficient because it can find the new plans and apply transform wherever it is - bottomUp(RuleSet.PUSH_DOWN_FILTERS), - - topDown( - new MergeFilters(), - new ReorderJoin(), - new PushFilterInsideJoin(), - new FindHashConditionForJoin(), - new ConvertInnerOrCrossJoin(), - new EliminateNullAwareLeftAntiJoin() - ), - topDown( - new EliminateDedupJoinCondition() - ) + // infer not null filter, then push down filter, and then reorder join(cross join to inner join) + topDown( + new InferFilterNotNull(), + new InferJoinNotNull() + ), + // ReorderJoin depends PUSH_DOWN_FILTERS + // the PUSH_DOWN_FILTERS depends on lots of rules, e.g. merge project, eliminate outer, + // sometimes transform the bottom plan make some rules usable which can apply to the top plan, + // but top-down traverse can not cover this case in one iteration, so bottom-up is more + // efficient because it can find the new plans and apply transform wherever it is + bottomUp(RuleSet.PUSH_DOWN_FILTERS), + + topDown( + new MergeFilters(), + new ReorderJoin(), + new PushFilterInsideJoin(), + new FindHashConditionForJoin(), + new ConvertInnerOrCrossJoin(), + new EliminateNullAwareLeftAntiJoin() + ), + topDown( + new EliminateDedupJoinCondition() + ) ), topic("Column pruning and infer predicate", - topDown(new ColumnPruning()), + custom(RuleType.COLUMN_PRUNING, () -> new ColumnPruning()), - custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()), + custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()), - // column pruning create new project, so we should use PUSH_DOWN_FILTERS - // to change filter-project to project-filter - bottomUp(RuleSet.PUSH_DOWN_FILTERS), + // column pruning create new project, so we should use PUSH_DOWN_FILTERS + // to change filter-project to project-filter + bottomUp(RuleSet.PUSH_DOWN_FILTERS), - // after eliminate outer join in the PUSH_DOWN_FILTERS, we can infer more predicate and push down - custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()), + // after eliminate outer join in the PUSH_DOWN_FILTERS, we can infer more predicate and push down + custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()), - bottomUp(RuleSet.PUSH_DOWN_FILTERS), + bottomUp(RuleSet.PUSH_DOWN_FILTERS), - // after eliminate outer join, we can move some filters to join.otherJoinConjuncts, - // this can help to translate plan to backend - topDown( - new PushFilterInsideJoin() - ) + // after eliminate outer join, we can move some filters to join.otherJoinConjuncts, + // this can help to translate plan to backend + topDown( + new PushFilterInsideJoin() + ) ), // this rule should invoke after ColumnPruning @@ -189,20 +194,35 @@ public class NereidsRewriter extends BatchRewriteJob { // we need to execute this rule at the end of rewrite // to avoid two consecutive same project appear when we do optimization. - topic("Others optimization", topDown( + topic("Others optimization", + bottomUp(ImmutableList.builder().addAll(ImmutableList.of( new EliminateNotNull(), new EliminateLimit(), new EliminateFilter(), - new PruneOlapScanPartition(), - new SelectMaterializedIndexWithAggregate(), - new SelectMaterializedIndexWithoutAggregate(), - new PruneOlapScanTablet(), new EliminateAggregate(), new MergeSetOperations(), new PushdownLimit(), - new SplitLimit(), new BuildAggForUnion() - )), + // after eliminate filter, the project maybe can push down again, + // so we add push down rules + )).addAll(RuleSet.PUSH_DOWN_FILTERS).build()) + ), + + // TODO: I think these rules should be implementation rules, and generate alternative physical plans. + topic("Table/MV/Physical optimization", + topDown( + // TODO: the logical plan should not contains any phase information, + // we should refactor like AggregateStrategies, e.g. LimitStrategies, + // generate one PhysicalLimit if current distribution is gather or two + // PhysicalLimits with gather exchange + new SplitLimit(), + + new SelectMaterializedIndexWithAggregate(), + new SelectMaterializedIndexWithoutAggregate(), + new PruneOlapScanTablet(), + new PruneOlapScanPartition() + ) + ), // this rule batch must keep at the end of rewrite to do some plan check topic("Final rewrite and check", bottomUp( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 1053393f0c2b46..cf04bc302426f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -88,6 +88,7 @@ public enum RuleType { // rewrite rules NORMALIZE_AGGREGATE(RuleTypeClass.REWRITE), + NORMALIZE_SORT(RuleTypeClass.REWRITE), NORMALIZE_REPEAT(RuleTypeClass.REWRITE), EXTRACT_AND_NORMALIZE_WINDOW_EXPRESSIONS(RuleTypeClass.REWRITE), CHECK_AND_STANDARDIZE_WINDOW_FUNCTION_AND_FRAME(RuleTypeClass.REWRITE), @@ -128,14 +129,7 @@ public enum RuleType { PUSHDOWN_PROJECT_THROUGH_LIMIT(RuleTypeClass.REWRITE), PUSHDOWN_ALIAS_THROUGH_JOIN(RuleTypeClass.REWRITE), PUSHDOWN_FILTER_THROUGH_SET_OPERATION(RuleTypeClass.REWRITE), - // column prune rules, - COLUMN_PRUNE_AGGREGATION_CHILD(RuleTypeClass.REWRITE), - COLUMN_PRUNE_FILTER_CHILD(RuleTypeClass.REWRITE), - PRUNE_ONE_ROW_RELATION_COLUMN(RuleTypeClass.REWRITE), - COLUMN_PRUNE_SORT_CHILD(RuleTypeClass.REWRITE), - COLUMN_PRUNE_SORT(RuleTypeClass.REWRITE), - COLUMN_PRUNE_JOIN_CHILD(RuleTypeClass.REWRITE), - COLUMN_PRUNE_REPEAT_CHILD(RuleTypeClass.REWRITE), + COLUMN_PRUNING(RuleTypeClass.REWRITE), // expression of plan rewrite REWRITE_ONE_ROW_RELATION_EXPRESSION(RuleTypeClass.REWRITE), REWRITE_PROJECT_EXPRESSION(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruning.java index 96a4d8cbc24486..8b79e734709266 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruning.java @@ -17,33 +17,271 @@ package org.apache.doris.nereids.rules.rewrite.logical; -import org.apache.doris.nereids.rules.PlanRuleFactory; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RulePromise; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.rules.rewrite.logical.ColumnPruning.PruneContext; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.logical.OutputPrunable; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; /** - * column prune rule set. + * ColumnPruning. + * + * you should implement OutputPrunable for your plan to provide the ability of column pruning + * + * functions: + * + * 1. prune/shrink output field for OutputPrunable, e.g. + * + * project(projects=[k1, sum(v1)]) project(projects=[k1, sum(v1)]) + * | -> | + * agg(groupBy=[k1], output=[k1, sum(v1), sum(v2)] agg(groupBy=[k1], output=[k1, sum(v1)]) + * + * 2. add project for the plan which prune children's output failed, e.g. the filter not record + * the output, and we can not prune/shrink output field for the filter, so we should add project on filter. + * + * agg(groupBy=[a]) agg(groupBy=[a]) + * | | + * filter(b > 10) -> project(a) + * | | + * plan filter(b > 10) + * | + * plan */ -public class ColumnPruning implements PlanRuleFactory { +public class ColumnPruning extends DefaultPlanRewriter implements CustomRewriter { @Override - public List buildRules() { - return ImmutableList.of( - new PruneFilterChildColumns().build(), - new PruneAggChildColumns().build(), - new PruneJoinChildrenColumns().build(), - new PruneSortColumns().build(), - new PruneSortChildColumns().build(), - new MergeProjects().build(), - new PruneRepeatChildColumns().build() - ); + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + return plan.accept(this, new PruneContext(plan.getOutputSet(), null)); } @Override - public RulePromise defaultPromise() { - return RulePromise.REWRITE; + public Plan visit(Plan plan, PruneContext context) { + if (plan instanceof OutputPrunable) { + // the case 1 in the class comment + // two steps: prune current output and prune children + OutputPrunable outputPrunable = (OutputPrunable) plan; + plan = pruneOutput(plan, outputPrunable.getOutputs(), outputPrunable::pruneOutputs, context); + return pruneChildren(plan); + } else { + // e.g. + // + // project(a) + // | + // | require: [a] + // v + // filter(b > 1) <- process currently + // | + // | require: [a, b] + // v + // child plan + // + // the filter is not OutputPrunable, we should pass through the parent required slots + // (slot a, which in the context.requiredSlots) and the used slots currently(slot b) to child plan. + return pruneChildren(plan, context.requiredSlots); + } + } + + // union can not prune children by the common logic, we must override visit method to write special code. + @Override + public Plan visitLogicalUnion(LogicalUnion union, PruneContext context) { + LogicalUnion prunedOutputUnion = pruneOutput(union, union.getOutputs(), union::pruneOutputs, context); + + // start prune children of union + List originOutput = union.getOutput(); + Set prunedOutput = prunedOutputUnion.getOutputSet(); + Set prunedOutputIndexes = IntStream.range(0, originOutput.size()) + .filter(index -> prunedOutput.contains(originOutput.get(index))) + .boxed() + .collect(ImmutableSet.toImmutableSet()); + + AtomicBoolean changed = new AtomicBoolean(false); + List prunedChildren = prunedOutputUnion.children().stream() + .map(child -> { + List childOutput = child.getOutput(); + Set prunedChildOutput = prunedOutputIndexes.stream() + .map(childOutput::get) + .collect(ImmutableSet.toImmutableSet()); + + Plan prunedChild = doPruneChild(prunedOutputUnion, child, prunedChildOutput); + if (prunedChild != child) { + changed.set(true); + } + return prunedChild; + }) + .collect(ImmutableList.toImmutableList()); + + if (!changed.get()) { + return prunedOutputUnion; + } + + return prunedOutputUnion.withChildren(prunedChildren); + } + + // we should keep the output of LogicalSetOperation and all the children + @Override + public Plan visitLogicalExcept(LogicalExcept except, PruneContext context) { + return skipPruneThisAndFirstLevelChildren(except); + } + + @Override + public Plan visitLogicalIntersect(LogicalIntersect intersect, PruneContext context) { + return skipPruneThisAndFirstLevelChildren(intersect); + } + + // the backend not support filter(project(agg)), so we can not prune the key set in the agg, + // only prune the agg functions here + @Override + public Plan visitLogicalAggregate(LogicalAggregate aggregate, PruneContext context) { + return pruneAggregate(aggregate, context); + } + + // same as aggregate + @Override + public Plan visitLogicalRepeat(LogicalRepeat repeat, PruneContext context) { + return pruneAggregate(repeat, context); + } + + private Plan pruneAggregate(Aggregate agg, PruneContext context) { + // first try to prune group by and aggregate functions + Aggregate prunedOutputAgg = pruneOutput(agg, agg.getOutputs(), agg::pruneOutputs, context); + + List groupByExpressions = prunedOutputAgg.getGroupByExpressions(); + List outputExpressions = prunedOutputAgg.getOutputExpressions(); + + // then fill up group by + Aggregate fillUpOutputRepeat = fillUpGroupByToOutput(groupByExpressions, outputExpressions) + .map(fullOutput -> prunedOutputAgg.withAggOutput(fullOutput)) + .orElse(prunedOutputAgg); + + return pruneChildren(fillUpOutputRepeat); + } + + private Plan skipPruneThisAndFirstLevelChildren(Plan plan) { + Set requireAllOutputOfChildren = plan.children() + .stream() + .flatMap(child -> child.getOutputSet().stream()) + .collect(Collectors.toSet()); + return pruneChildren(plan, requireAllOutputOfChildren); + } + + private static Optional> fillUpGroupByToOutput( + List groupBy, List output) { + + if (output.containsAll(groupBy)) { + return Optional.empty(); + } + + List aggFunctions = Lists.newArrayList(output); + aggFunctions.removeAll(groupBy); + + return Optional.of(ImmutableList.builder() + .addAll((List) groupBy) + .addAll(aggFunctions) + .build()); + } + + public static final

P pruneOutput(P plan, List originOutput, + Function, P> withPrunedOutput, PruneContext context) { + Optional> prunedOutputs = pruneOutput(originOutput, context); + return prunedOutputs.map(withPrunedOutput).orElse(plan); + } + + /** prune output */ + public static Optional> pruneOutput( + List originOutput, PruneContext context) { + List prunedOutputs = originOutput.stream() + .filter(output -> context.requiredSlots.contains(output.toSlot())) + .collect(ImmutableList.toImmutableList()); + + if (prunedOutputs.isEmpty()) { + NamedExpression minimumColumn = ExpressionUtils.selectMinimumColumn(originOutput); + prunedOutputs = ImmutableList.of(minimumColumn); + } + + return prunedOutputs.equals(originOutput) + ? Optional.empty() + : Optional.of(prunedOutputs); + } + + private final

P pruneChildren(P plan) { + return pruneChildren(plan, ImmutableSet.of()); + } + + private final

P pruneChildren(P plan, Set parentRequiredSlots) { + if (plan.arity() == 0) { + // leaf + return plan; + } + + Set currentUsedSlots = plan.getInputSlots(); + Set childrenRequiredSlots = parentRequiredSlots.isEmpty() + ? currentUsedSlots + : ImmutableSet.builder() + .addAll(parentRequiredSlots) + .addAll(currentUsedSlots) + .build(); + + List newChildren = new ArrayList<>(); + boolean hasNewChildren = false; + for (Plan child : plan.children()) { + Set childOutputSet = child.getOutputSet(); + Set childRequiredSlots = Sets.intersection(childrenRequiredSlots, childOutputSet); + if (childRequiredSlots.isEmpty()) { + childRequiredSlots = ImmutableSet.of(ExpressionUtils.selectMinimumColumn(childOutputSet)); + } + Plan prunedChild = doPruneChild(plan, child, childRequiredSlots); + if (prunedChild != child) { + hasNewChildren = true; + } + newChildren.add(prunedChild); + } + return hasNewChildren ? (P) plan.withChildren(newChildren) : plan; + } + + private Plan doPruneChild(Plan plan, Plan child, Set childRequiredSlots) { + boolean isProject = plan instanceof LogicalProject; + Plan prunedChild = child.accept(this, new PruneContext(childRequiredSlots, plan)); + + // the case 2 in the class comment, prune child's output failed + if (!isProject && !Sets.difference(prunedChild.getOutputSet(), childRequiredSlots).isEmpty()) { + prunedChild = new LogicalProject<>(ImmutableList.copyOf(childRequiredSlots), prunedChild); + } + return prunedChild; + } + + /** PruneContext */ + public static class PruneContext { + public Set requiredSlots; + public Optional parent; + + public PruneContext(Set requiredSlots, Plan parent) { + this.requiredSlots = requiredSlots; + this.parent = Optional.ofNullable(parent); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeSort.java similarity index 63% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortColumns.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeSort.java index f77a16b6624bf9..7b74cb43856323 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortColumns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeSort.java @@ -27,19 +27,24 @@ import java.util.stream.Collectors; /** - the sort node will create new slots for order by keys if the order by keys is not in the output - so need create a project above sort node to prune the unnecessary order by keys + * the sort node will create new slots for order by keys if the order by keys is not in the output + * so need create a project above sort node to prune the unnecessary order by keys. This means the + * Tuple slots size is difference to PhysicalSort.output.size. If not prune and hide the order key, + * the upper plan node will see the temporary slots and treat as output, and then translate failed. + * This is trick, we should add sort output tuple to ensure the tuple slot size is equals, but it + * has large workload. I think we should refactor the PhysicalPlanTranslator in the future, and + * process PhysicalProject(output)/PhysicalDistribute more general. */ -public class PruneSortColumns extends OneRewriteRuleFactory { +public class NormalizeSort extends OneRewriteRuleFactory { @Override public Rule build() { return logicalSort() - .when(sort -> !sort.isOrderKeysPruned() && !sort.getOutputSet() + .when(sort -> !sort.isNormalized() && !sort.getOutputSet() .containsAll(sort.getOrderKeys().stream() .map(orderKey -> orderKey.getExpr()).collect(Collectors.toSet()))) .then(sort -> { return new LogicalProject(sort.getOutput(), ImmutableList.of(), false, - sort.withOrderKeysPruned(true)); - }).toRule(RuleType.COLUMN_PRUNE_SORT); + sort.withNormalize(true)); + }).toRule(RuleType.NORMALIZE_SORT); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java deleted file mode 100644 index 47df6ad4f632b4..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java +++ /dev/null @@ -1,89 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite.logical; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.util.ExpressionUtils; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Set; - -/** - * prune its child output according to agg. - * pattern: agg() - * table a: k1,k2,k3,v1 - * select k1,sum(v1) from a group by k1 - * plan tree: - * agg - * | - * scan(k1,k2,k3,v1) - * transformed: - * agg - * | - * project(k1,v1) - * | - * scan(k1,k2,k3,v1) - */ -public class PruneAggChildColumns extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return RuleType.COLUMN_PRUNE_AGGREGATION_CHILD.build(logicalAggregate().then(agg -> { - List childOutput = agg.child().getOutput(); - if (isAggregateWithConstant(agg) && agg.getGroupByExpressions().isEmpty()) { - Slot slot = ExpressionUtils.selectMinimumColumn(childOutput); - if (childOutput.size() == 1 && childOutput.get(0).equals(slot)) { - return agg; - } - return agg.withChildren(ImmutableList.of(new LogicalProject<>(ImmutableList.of(slot), agg.child()))); - } - Set aggInputSlots = agg.getInputSlots(); - List prunedOutputs = childOutput.stream().filter(aggInputSlots::contains) - .collect(ImmutableList.toImmutableList()); - if (prunedOutputs.size() == agg.child().getOutput().size()) { - return agg; - } - return agg.withChildren(ImmutableList.of(new LogicalProject<>(prunedOutputs, agg.child()))); - })); - } - - /** - * For these aggregate function with constant param. Such as: - * count(*), count(1), sum(1)..etc. - * @return null, if there exists an aggregation function that its parameters contains non-constant expr. - * else return a slot with min data type. - */ - private boolean isAggregateWithConstant(LogicalAggregate agg) { - for (NamedExpression output : agg.getOutputExpressions()) { - if (output.anyMatch(SlotReference.class::isInstance)) { - return false; - } - } - return true; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java deleted file mode 100644 index b1f28def1b11a1..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite.logical; - -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; - -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * prune filter output. - * pattern: project(filter()) - * table a: k1,k2,k3,v1 - * select k1 from a where k2 > 3 - * plan tree: - * project(k1) - * | - * filter(k2 > 3) - * | - * scan(k1,k2,k3,v1) - * transformed: - * 　project(k1) - * | - * filter(k2 > 3) - * | - * project(k1,k2) - * | - * scan(k1,k2,k3,v1) - */ -public class PruneFilterChildColumns extends AbstractPushDownProjectRule> { - - public PruneFilterChildColumns() { - setRuleType(RuleType.COLUMN_PRUNE_FILTER_CHILD); - setTarget(logicalFilter()); - } - - @Override - protected Plan pushDownProject(LogicalFilter filter, Set references) { - Set filterInputSlots = filter.getInputSlots(); - Set required = Stream.concat(references.stream(), filterInputSlots.stream()).collect(Collectors.toSet()); - if (required.containsAll(filter.child().getOutput())) { - return filter; - } - return filter.withChildren( - ImmutableList.of(new LogicalProject<>(Lists.newArrayList(required), filter.child())) - ); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java deleted file mode 100644 index daf4ad3a3f2c29..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite.logical; - -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.util.ExpressionUtils; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * prune join children output. - * pattern: project(join()) - * table a: k1,k2,k3,v1 - * table b: k1,k2,v1,v2 - * select a.k1,b.k2 from a join b on a.k1 = b.k1 where a.k3 > 1 - * plan tree: - * project(a.k1,b.k2) - * | - * join(k1,k2,k3,v1,k1,k2,v1,v2) - * / \ - * scan(a) scan(b) - * transformed: - * project(a.k1,b.k2) - * | - * join(k1,k2,k3,v1,k1,k2,v1,v2) - * / \ - * project(a.k1,a.k3) project(b.k2,b.k1) - * | | - * scan scan - */ -public class PruneJoinChildrenColumns - extends AbstractPushDownProjectRule> { - - public PruneJoinChildrenColumns() { - setRuleType(RuleType.COLUMN_PRUNE_JOIN_CHILD); - setTarget(logicalJoin()); - } - - @Override - protected Plan pushDownProject(LogicalJoin joinPlan, - Set references) { - - Set exprIds = Stream.of(references, joinPlan.getInputSlots()) - .flatMap(Set::stream) - .map(NamedExpression::getExprId) - .collect(Collectors.toSet()); - - List leftInputs = joinPlan.left().getOutput().stream() - .filter(r -> exprIds.contains(r.getExprId())).collect(ImmutableList.toImmutableList()); - List rightInputs = joinPlan.right().getOutput().stream() - .filter(r -> exprIds.contains(r.getExprId())).collect(ImmutableList.toImmutableList()); - - if (leftInputs.isEmpty()) { - leftInputs = ImmutableList.of(ExpressionUtils.selectMinimumColumn(joinPlan.left().getOutput())); - } - if (rightInputs.isEmpty()) { - rightInputs = ImmutableList.of(ExpressionUtils.selectMinimumColumn(joinPlan.right().getOutput())); - } - - Plan leftPlan = joinPlan.left(); - Plan rightPlan = joinPlan.right(); - - if (leftInputs.size() != leftPlan.getOutput().size()) { - leftPlan = new LogicalProject<>(leftInputs, leftPlan); - } - - if (rightInputs.size() != rightPlan.getOutput().size()) { - rightPlan = new LogicalProject<>(rightInputs, rightPlan); - } - return joinPlan.withChildren(ImmutableList.of(leftPlan, rightPlan)); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneRepeatChildColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneRepeatChildColumns.java deleted file mode 100644 index 02d3ef12af4dd2..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneRepeatChildColumns.java +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite.logical; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Set; - -/** - * prune its child output according to repeat. - * pattern: repeat() - * table a: k1,k2,k3,v1 - * select k1,sum(v1) from a group by grouping sets ((k1)) - * plan tree: - * repeat - * | - * scan(k1,k2,k3,v1) - * transformed: - * repeat - * | - * project(k1,v1) - * | - * scan(k1,k2,k3,v1) - */ -public class PruneRepeatChildColumns extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return RuleType.COLUMN_PRUNE_REPEAT_CHILD.build(logicalRepeat().then(repeat -> { - List childOutput = repeat.child().getOutput(); - Set groupByInputSlots = repeat.getInputSlots(); - List prunedOutputs = childOutput.stream().filter(groupByInputSlots::contains) - .collect(ImmutableList.toImmutableList()); - if (prunedOutputs.size() == repeat.child().getOutput().size()) { - return repeat; - } - return repeat.withChildren(ImmutableList.of(new LogicalProject<>(prunedOutputs, repeat.child()))); - })); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java deleted file mode 100644 index 90adbd067a973b..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite.logical; - -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.trees.plans.logical.LogicalSort; - -import com.google.common.collect.ImmutableList; - -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * prune join children output. - * pattern: project(sort()) - */ -public class PruneSortChildColumns extends AbstractPushDownProjectRule> { - - public PruneSortChildColumns() { - setRuleType(RuleType.COLUMN_PRUNE_SORT_CHILD); - setTarget(logicalSort()); - } - - @Override - protected Plan pushDownProject(LogicalSort sortPlan, Set references) { - Set sortSlots = sortPlan.getOutputSet(); - Set required = Stream.concat(references.stream(), sortSlots.stream()).collect(Collectors.toSet()); - if (required.containsAll(sortPlan.child().getOutput())) { - return sortPlan; - } - return sortPlan.withChildren( - ImmutableList.of(new LogicalProject<>(ImmutableList.copyOf(required), sortPlan.child())) - ); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java index c8818310ab23ea..c76577941308f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java @@ -48,7 +48,7 @@ public class PushdownProjectThroughLimit extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalProject(logicalLimit(any())).thenApply(ctx -> { + return logicalProject(logicalLimit()).thenApply(ctx -> { LogicalProject> logicalProject = ctx.root; LogicalLimit logicalLimit = logicalProject.child(); return new LogicalLimit<>(logicalLimit.getLimit(), logicalLimit.getOffset(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Aggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Aggregate.java index 5ac7f37dfc2cbc..6731bde58a3ea2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Aggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Aggregate.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.UnaryPlan; +import org.apache.doris.nereids.trees.plans.logical.OutputPrunable; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; @@ -32,7 +33,7 @@ /** * Common interface for logical/physical Aggregate. */ -public interface Aggregate extends UnaryPlan { +public interface Aggregate extends UnaryPlan, OutputPrunable { List getGroupByExpressions(); @@ -43,6 +44,11 @@ public interface Aggregate extends UnaryPlan withChildren(List children); + @Override + default Aggregate pruneOutputs(List prunedOutputs) { + return withAggOutput(prunedOutputs); + } + default Set getAggregateFunctions() { return ExpressionUtils.collect(getOutputExpressions(), AggregateFunction.class::isInstance); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index ad62daca454645..59fec03fbebadd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -52,7 +52,7 @@ */ public class LogicalAggregate extends LogicalUnary - implements Aggregate, OutputSavePoint { + implements Aggregate { private final boolean normalized; private final List groupByExpressions; @@ -225,6 +225,11 @@ public LogicalAggregate withGroupByAndOutput(List groupByExprL sourceRepeat, Optional.empty(), Optional.empty(), child()); } + @Override + public List getOutputs() { + return outputExpressions; + } + @Override public LogicalAggregate withAggOutput(List newOutput) { return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java index e686cbe475933a..d60fd002bcb952 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java @@ -39,9 +39,9 @@ * e.g. * select * from tbl limit 0 */ -public class LogicalEmptyRelation extends LogicalLeaf implements EmptyRelation { +public class LogicalEmptyRelation extends LogicalLeaf implements EmptyRelation, OutputPrunable { - private final List projects; + private final List projects; public LogicalEmptyRelation(List projects) { this(projects, Optional.empty(), Optional.empty()); @@ -59,7 +59,7 @@ public R accept(PlanVisitor visitor, C context) { } @Override - public List getProjects() { + public List getProjects() { return projects; } @@ -68,6 +68,10 @@ public List getExpressions() { return ImmutableList.of(); } + public LogicalEmptyRelation withProjects(List projects) { + return new LogicalEmptyRelation(projects, Optional.empty(), Optional.empty()); + } + @Override public Plan withGroupExpression(Optional groupExpression) { return new LogicalEmptyRelation(projects, groupExpression, Optional.of(logicalPropertiesSupplier.get())); @@ -111,4 +115,14 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(projects); } + + @Override + public List getOutputs() { + return projects; + } + + @Override + public Plan pruneOutputs(List prunedOutputs) { + return withProjects(prunedOutputs); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index 3380288e98f0a3..a2e8bee7bc9209 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -39,7 +39,7 @@ * A relation that contains only one row consist of some constant expressions. * e.g. select 100, 'value' */ -public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation { +public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation, OutputPrunable { private final List projects; private final boolean buildUnionNode; @@ -125,7 +125,21 @@ public boolean buildUnionNode() { return buildUnionNode; } + public LogicalOneRowRelation withProjects(List namedExpressions) { + return new LogicalOneRowRelation(namedExpressions, buildUnionNode, Optional.empty(), Optional.empty()); + } + public Plan withBuildUnionNode(boolean buildUnionNode) { return new LogicalOneRowRelation(projects, buildUnionNode, Optional.empty(), Optional.empty()); } + + @Override + public List getOutputs() { + return projects; + } + + @Override + public Plan pruneOutputs(List prunedOutputs) { + return withProjects(prunedOutputs); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 8961f3211a9aac..8debace27910eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -41,7 +41,7 @@ * Logical project plan. */ public class LogicalProject extends LogicalUnary - implements Project, OutputSavePoint { + implements Project, OutputPrunable { private final List projects; private final List excepts; @@ -199,4 +199,14 @@ public boolean canEliminate() { public boolean isDistinct() { return isDistinct; } + + @Override + public List getOutputs() { + return projects; + } + + @Override + public Plan pruneOutputs(List prunedOutputs) { + return withProjects(prunedOutputs); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 8cc322fc53c203..165d4685ae4926 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -41,7 +41,7 @@ * LogicalRepeat. */ public class LogicalRepeat extends LogicalUnary - implements Repeat, OutputSavePoint { + implements Repeat { // max num of distinct sets in grouping sets clause public static final int MAX_GROUPING_SETS_NUM = 64; @@ -84,6 +84,11 @@ public List getOutputExpressions() { return outputExpressions; } + @Override + public List getOutputs() { + return outputExpressions; + } + @Override public String toString() { return Utils.toSqlString("LogicalRepeat", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java index 1fb19b722cba4a..b6f7b5a50a28de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java @@ -46,14 +46,14 @@ public class LogicalSort extends LogicalUnary orderKeys; - private final boolean orderKeysPruned; + private final boolean normalized; public LogicalSort(List orderKeys, CHILD_TYPE child) { this(orderKeys, Optional.empty(), Optional.empty(), child); } - public LogicalSort(List orderKeys, CHILD_TYPE child, boolean orderKeysPruned) { - this(orderKeys, Optional.empty(), Optional.empty(), child, orderKeysPruned); + public LogicalSort(List orderKeys, CHILD_TYPE child, boolean normalized) { + this(orderKeys, Optional.empty(), Optional.empty(), child, normalized); } /** @@ -65,10 +65,10 @@ public LogicalSort(List orderKeys, Optional groupExpr } public LogicalSort(List orderKeys, Optional groupExpression, - Optional logicalProperties, CHILD_TYPE child, boolean orderKeysPruned) { + Optional logicalProperties, CHILD_TYPE child, boolean normalized) { super(PlanType.LOGICAL_SORT, groupExpression, logicalProperties, child); this.orderKeys = ImmutableList.copyOf(Objects.requireNonNull(orderKeys, "orderKeys can not be null")); - this.orderKeysPruned = orderKeysPruned; + this.normalized = normalized; } @Override @@ -80,8 +80,8 @@ public List getOrderKeys() { return orderKeys; } - public boolean isOrderKeysPruned() { - return orderKeysPruned; + public boolean isNormalized() { + return normalized; } @Override @@ -122,13 +122,13 @@ public List getExpressions() { @Override public LogicalSort withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalSort<>(orderKeys, children.get(0), orderKeysPruned); + return new LogicalSort<>(orderKeys, children.get(0), normalized); } @Override public LogicalSort withGroupExpression(Optional groupExpression) { return new LogicalSort<>(orderKeys, groupExpression, Optional.of(getLogicalProperties()), child(), - orderKeysPruned); + normalized); } @Override @@ -141,7 +141,7 @@ public LogicalSort withOrderKeys(List orderKeys) { Optional.of(getLogicalProperties()), child(), false); } - public LogicalSort withOrderKeysPruned(boolean orderKeysPruned) { + public LogicalSort withNormalize(boolean orderKeysPruned) { return new LogicalSort<>(orderKeys, groupExpression, Optional.of(getLogicalProperties()), child(), orderKeysPruned); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java index aa9a9db21e7312..1a0ff5364524bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -32,7 +32,7 @@ /** * Logical Union. */ -public class LogicalUnion extends LogicalSetOperation { +public class LogicalUnion extends LogicalSetOperation implements OutputPrunable { // When the union is DISTINCT, an additional LogicalAggregation needs to be created, // so add this flag to judge whether agg has been created to avoid repeated creation @@ -143,4 +143,9 @@ public LogicalUnion withHasPushedFilter() { public LogicalUnion withNewChildren(List children) { return withChildren(children); } + + @Override + public LogicalUnion pruneOutputs(List prunedOutputs) { + return withNewOutputs(prunedOutputs); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/OutputPrunable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/OutputPrunable.java new file mode 100644 index 00000000000000..03129ab4e5964a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/OutputPrunable.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; + +/** OutputPrunable */ +public interface OutputPrunable extends OutputSavePoint { + List getOutputs(); + + Plan pruneOutputs(List prunedOutputs); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java index 1d8cdbf71f3480..e713f6dd87c87b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java @@ -134,6 +134,11 @@ public List getOutputExpressions() { return outputExpressions; } + @Override + public List getOutputs() { + return outputExpressions; + } + public Optional> getPartitionExpressions() { return partitionExpressions; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java index 7d4448e518f966..67b525214c935d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java @@ -90,6 +90,11 @@ public List getOutputExpressions() { return outputExpressions; } + @Override + public List getOutputs() { + return outputExpressions; + } + @Override public String toString() { return Utils.toSqlString("PhysicalRepeat[" + id.asInt() + "]" + getGroupIdAsString(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/DefaultPlanRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/DefaultPlanRewriter.java index 5339b13d42afc0..1dd90772f11ea9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/DefaultPlanRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/DefaultPlanRewriter.java @@ -32,16 +32,7 @@ public abstract class DefaultPlanRewriter extends PlanVisitor { @Override public Plan visit(Plan plan, C context) { - List newChildren = new ArrayList<>(); - boolean hasNewChildren = false; - for (Plan child : plan.children()) { - Plan newChild = child.accept(this, context); - if (newChild != child) { - hasNewChildren = true; - } - newChildren.add(newChild); - } - return hasNewChildren ? plan.withChildren(newChildren) : plan; + return visitChildren(this, plan, context); } @Override @@ -52,4 +43,18 @@ public Plan visitPhysicalStorageLayerAggregate(PhysicalStorageLayerAggregate sto } return storageLayerAggregate; } + + /** visitChildren */ + public static final

P visitChildren(DefaultPlanRewriter rewriter, P plan, C context) { + List newChildren = new ArrayList<>(); + boolean hasNewChildren = false; + for (Plan child : plan.children()) { + Plan newChild = child.accept(rewriter, context); + if (newChild != child) { + hasNewChildren = true; + } + newChildren.add(newChild); + } + return hasNewChildren ? (P) plan.withChildren(newChildren) : plan; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 7cd8b91979eaf1..25ba7076240886 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.Slot; @@ -203,10 +204,10 @@ public static Expression combine(Class type, List slots) { + public static S selectMinimumColumn(Collection slots) { Preconditions.checkArgument(!slots.isEmpty()); - Slot minSlot = null; - for (Slot slot : slots) { + S minSlot = null; + for (S slot : slots) { if (minSlot == null) { minSlot = slot; } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java index 55fce8edbffcd2..92e1d7cdafa104 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java @@ -53,7 +53,12 @@ public class TableFunctionNode extends PlanNode { public TableFunctionNode(PlanNodeId id, PlanNode inputNode, TupleId lateralViewTupleId, ArrayList fnCallExprList, List outputSlotIds) { super(id, "TABLE FUNCTION NODE", StatisticalType.TABLE_FUNCTION_NODE); - tupleIds.addAll(inputNode.getTupleIds()); + List childOutputTupleIds = inputNode.getOutputTupleIds(); + if (childOutputTupleIds != null && !childOutputTupleIds.isEmpty()) { + tupleIds.addAll(childOutputTupleIds); + } else { + tupleIds.addAll(inputNode.getTupleIds()); + } tupleIds.add(lateralViewTupleId); this.lateralViewTupleIds = Lists.newArrayList(lateralViewTupleId); this.fnCallExprList = fnCallExprList; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java index 8806c878d6f9c2..99e9f3842f0a51 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java @@ -233,7 +233,8 @@ public void testCastAtOnExpression() { } private Optional> getRuntimeFilters(String sql) { - PlanChecker checker = PlanChecker.from(connectContext).analyze(sql) + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(sql) .rewrite() .implement(); PhysicalPlan plan = checker.getPhysicalPlan(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java index 7379df6bea4c13..c89d3aebdb3f79 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java @@ -61,7 +61,7 @@ public void testPruneColumns1() { PlanChecker.from(connectContext) .analyze("select id,name,grade from student left join score on student.id = score.sid" + " where score.grade > 60") - .applyTopDown(new ColumnPruning()) + .customRewrite(new ColumnPruning()) .matchesFromRoot( logicalProject( logicalFilter( @@ -93,7 +93,7 @@ public void testPruneColumns2() { .analyze("select name,sex,cid,grade " + "from student left join score on student.id = score.sid " + "where score.grade > 60") - .applyTopDown(new ColumnPruning()) + .customRewrite(new ColumnPruning()) .matchesFromRoot( logicalProject( logicalFilter( @@ -123,7 +123,7 @@ public void testPruneColumns2() { public void testPruneColumns3() { PlanChecker.from(connectContext) .analyze("select id,name from student where age > 18") - .applyTopDown(new ColumnPruning()) + .customRewrite(new ColumnPruning()) .matchesFromRoot( logicalProject( logicalFilter( @@ -145,7 +145,7 @@ public void testPruneColumns4() { + "on student.id = score.sid left join course " + "on score.cid = course.cid " + "where score.grade > 60") - .applyTopDown(new ColumnPruning()) + .customRewrite(new ColumnPruning()) .matchesFromRoot( logicalProject( logicalFilter( @@ -183,7 +183,7 @@ public void testPruneColumns4() { public void pruneCountStarStmt() { PlanChecker.from(connectContext) .analyze("SELECT COUNT(*) FROM test.course") - .applyTopDown(new ColumnPruning()) + .customRewrite(new ColumnPruning()) .matchesFromRoot( logicalAggregate( logicalProject( @@ -198,7 +198,7 @@ public void pruneCountStarStmt() { public void pruneCountConstantStmt() { PlanChecker.from(connectContext) .analyze("SELECT COUNT(1) FROM test.course") - .applyTopDown(new ColumnPruning()) + .customRewrite(new ColumnPruning()) .matchesFromRoot( logicalAggregate( logicalProject( @@ -213,7 +213,7 @@ public void pruneCountConstantStmt() { public void pruneCountConstantAndSumConstantStmt() { PlanChecker.from(connectContext) .analyze("SELECT COUNT(1), SUM(2) FROM test.course") - .applyTopDown(new ColumnPruning()) + .customRewrite(new ColumnPruning()) .matchesFromRoot( logicalAggregate( logicalProject( @@ -228,7 +228,7 @@ public void pruneCountConstantAndSumConstantStmt() { public void pruneCountStarAndSumConstantStmt() { PlanChecker.from(connectContext) .analyze("SELECT COUNT(*), SUM(2) FROM test.course") - .applyTopDown(new ColumnPruning()) + .customRewrite(new ColumnPruning()) .matchesFromRoot( logicalAggregate( logicalProject( @@ -243,7 +243,7 @@ public void pruneCountStarAndSumConstantStmt() { public void pruneCountStarAndSumColumnStmt() { PlanChecker.from(connectContext) .analyze("SELECT COUNT(*), SUM(grade) FROM test.score") - .applyTopDown(new ColumnPruning()) + .customRewrite(new ColumnPruning()) .matchesFromRoot( logicalAggregate( logicalProject( @@ -258,7 +258,7 @@ public void pruneCountStarAndSumColumnStmt() { public void pruneCountStarAndSumColumnAndSumConstantStmt() { PlanChecker.from(connectContext) .analyze("SELECT COUNT(*), SUM(grade) + SUM(2) FROM test.score") - .applyTopDown(new ColumnPruning()) + .customRewrite(new ColumnPruning()) .matchesFromRoot( logicalAggregate( logicalProject( @@ -273,7 +273,7 @@ public void pruneCountStarAndSumColumnAndSumConstantStmt() { public void pruneColumnForOneSideOnCrossJoin() { PlanChecker.from(connectContext) .analyze("select id,name from student cross join score") - .applyTopDown(new ColumnPruning()) + .customRewrite(new ColumnPruning()) .matchesFromRoot( logicalProject( logicalJoin( @@ -291,7 +291,33 @@ public void pruneColumnForOneSideOnCrossJoin() { ); } + @Test + public void pruneAggregateOutput() { + PlanChecker.from(connectContext) + .analyze("select id from (select id, sum(age) from student group by id)a") + .customRewrite(new ColumnPruning()) + .matchesFromRoot( + logicalProject( + logicalSubQueryAlias( + logicalAggregate( + logicalProject( + logicalOlapScan() + ).when(p -> getOutputQualifiedNames(p).equals( + ImmutableList.of("default_cluster:test.student.id") + )) + ).when(agg -> getOutputQualifiedNames(agg.getOutputs()).equals( + ImmutableList.of("default_cluster:test.student.id") + )) + ) + ) + ); + } + private List getOutputQualifiedNames(LogicalProject p) { - return p.getProjects().stream().map(NamedExpression::getQualifiedName).collect(Collectors.toList()); + return getOutputQualifiedNames(p.getOutputs()); + } + + private List getOutputQualifiedNames(List output) { + return output.stream().map(NamedExpression::getQualifiedName).collect(Collectors.toList()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java index bf37e5666f0672..0fca4b1f0a4d81 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java @@ -83,15 +83,25 @@ public void testLeftSemiJoin() { .join(scan2, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0)) .joinEmptyOn(scan3, JoinType.CROSS_JOIN) .filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0))) - .build(), - new LogicalPlanBuilder(scan1) - .joinEmptyOn(scan3, JoinType.CROSS_JOIN) - .join(scan2, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0)) - .filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0))) .build() ); - check(plans); + + LogicalPlan plan2 = new LogicalPlanBuilder(scan1) + .joinEmptyOn(scan3, JoinType.CROSS_JOIN) + .join(scan2, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0)) + .filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0))) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan2) + .rewrite() + .printlnTree() + .matchesFromRoot( + logicalJoin( + logicalJoin().whenNot(join -> join.getJoinType().isCrossJoin()), + logicalProject(logicalOlapScan()) + ).whenNot(join -> join.getJoinType().isCrossJoin()) + ); } @Test @@ -116,7 +126,7 @@ public void testRightSemiJoin() { .rewrite() .matchesFromRoot( rightSemiLogicalJoin( - leafPlan(), + logicalProject(logicalOlapScan()), innerLogicalJoin() ) ); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 1714f5083d0933..c994ace23beaab 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.jobs.batch.NereidsRewriter; import org.apache.doris.nereids.jobs.cascades.DeriveStatsJob; import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob; +import org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob; import org.apache.doris.nereids.memo.CopyInResult; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -123,6 +124,12 @@ public PlanChecker analyze(Plan plan) { return this; } + public PlanChecker customRewrite(CustomRewriter customRewriter) { + new CustomRewriteJob(() -> customRewriter, RuleType.TEST_REWRITE).execute(cascadesContext.getCurrentJobContext()); + cascadesContext.toMemo(); + return this; + } + public PlanChecker applyTopDown(RuleFactory ruleFactory) { return applyTopDown(ruleFactory.buildRules()); } diff --git a/regression-test/data/nereids_syntax_p0/set_operation.out b/regression-test/data/nereids_syntax_p0/set_operation.out index 09b3124475117f..8f8db3a5405e1a 100644 --- a/regression-test/data/nereids_syntax_p0/set_operation.out +++ b/regression-test/data/nereids_syntax_p0/set_operation.out @@ -406,9 +406,9 @@ d d 3 3 9.0 3 9 -- !union30 -- +0.0001 1E-7 1.0000 2.0000000 1.0100 2.0000000 -0.0001 1E-7 -- !union31 -- 1 2 diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/CreateMVAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/CreateMVAction.groovy index b4b37df01619ad..2b77a8815e5016 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/CreateMVAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/CreateMVAction.groovy @@ -17,14 +17,13 @@ package org.apache.doris.regression.action -import groovy.transform.stc.ClosureParams -import groovy.transform.stc.FromString + +import groovy.util.logging.Slf4j import org.apache.doris.regression.suite.SuiteContext import org.apache.doris.regression.util.JdbcUtils -import groovy.util.logging.Slf4j -import java.sql.ResultSetMetaData -import java.util.stream.Collectors + import java.sql.Connection +import java.sql.ResultSetMetaData @Slf4j class CreateMVAction implements SuiteAction { @@ -63,7 +62,7 @@ class CreateMVAction implements SuiteAction { while (!sqlResult.contains("FINISHED")) { def tmp = doRun("SHOW ALTER TABLE MATERIALIZED VIEW ORDER BY CreateTime DESC LIMIT 1;") sqlResult = tmp.result[0] - log.info("result: ${sqlResult}") + log.info("result: ${sqlResult}".toString()) if (tryTimes == 60 || sqlResult.contains("CANCELLED")) { throw new IllegalStateException("MV create check times over limit"); } diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv.groovy index fa22595a1df83b..ab45a4804802fc 100644 --- a/regression-test/suites/mtmv_p0/test_create_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_create_mtmv.groovy @@ -56,7 +56,7 @@ suite("test_create_mtmv") { INSERT INTO ${tableNamePv} VALUES("2022-10-26",1,200),("2022-10-28",2,200),("2022-10-28",3,300); """ - sql """drop materialized view if exists ${mvName}""" + sql """drop materialized view if exists ${mvName}""" sql """ CREATE MATERIALIZED VIEW ${mvName} diff --git a/regression-test/suites/mv_p0/multi_slot_multi_mv/multi_slot_multi_mv.groovy b/regression-test/suites/mv_p0/multi_slot_multi_mv/multi_slot_multi_mv.groovy index 4e6f89055ad9ff..70caf98b16500c 100644 --- a/regression-test/suites/mv_p0/multi_slot_multi_mv/multi_slot_multi_mv.groovy +++ b/regression-test/suites/mv_p0/multi_slot_multi_mv/multi_slot_multi_mv.groovy @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -import org.codehaus.groovy.runtime.IOGroovyMethods - suite ("multi_slot_multi_mv") { sql """ DROP TABLE IF EXISTS d_table; """ @@ -38,24 +36,40 @@ suite ("multi_slot_multi_mv") { createMV ("create materialized view k1a2p2ap3p as select abs(k1)+k2+1,abs(k2+2)+k3+3 from d_table;") - sql "create materialized view k1a2p2ap3ps as select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1;" - while (!result.contains("FINISHED")){ - result = sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='d_table' ORDER BY CreateTime DESC LIMIT 1;" - result = result.toString() - logger.info("result: ${result}") - if(result.contains("CANCELLED")){ - return - } - Thread.sleep(1000) - } + createMV("create materialized view k1a2p2ap3ps as select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1;") sql "insert into d_table select -4,-4,-4,'d';" qt_select_star "select * from d_table order by k1;" - explain { - sql("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by abs(k1)+k2+1") - contains "(k1a2p2ap3p)" + def retry_times = 60 + for (def i = 0; i < retry_times; ++i) { + boolean is_k1a2p2ap3p = false + boolean is_k1a2p2ap3ps = false + boolean is_d_table = false + explain { + sql("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by abs(k1)+k2+1") + check { explainStr, ex, startTime, endTime -> + if (ex != null) { + throw ex; + } + logger.info("explain result: ${explainStr}".toString()) + is_k1a2p2ap3p = explainStr.contains"(k1a2p2ap3p)" + is_k1a2p2ap3ps = explainStr.contains("(k1a2p2ap3ps)") + is_d_table = explainStr.contains("(d_table)") + assert is_k1a2p2ap3p || is_k1a2p2ap3ps || is_d_table + } + } + // FIXME: the mv selector maybe select base table forever when exist multi mv, + // so this pr just treat as success if select base table. + // we should remove is_d_table in the future + if (is_d_table || is_k1a2p2ap3p || is_k1a2p2ap3ps) { + break + } + if (i + 1 == retry_times) { + throw new IllegalStateException("retry and failed too much") + } + sleep(1000) } qt_select_mv "select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by abs(k1)+k2+1;" diff --git a/regression-test/suites/mv_p0/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy b/regression-test/suites/mv_p0/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy index fdd71d17ab14de..fa9daf5ff0bac3 100644 --- a/regression-test/suites/mv_p0/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy +++ b/regression-test/suites/mv_p0/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -import org.codehaus.groovy.runtime.IOGroovyMethods - suite ("testAggQueryOnAggMV2") { sql """ DROP TABLE IF EXISTS emps; """ sql """ @@ -51,11 +49,11 @@ suite ("testAggQueryOnAggMV2") { } qt_select_star "select * from emps order by empid, salary;" - explain { + explain { sql("select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where (sum_salary * 2) > 3 order by deptno ;") contains "(emps_mv)" } qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where (sum_salary * 2) > 3 order by deptno ;" -} \ No newline at end of file +} diff --git a/regression-test/suites/mysql_ssl_p0/test_mysql_connection.groovy b/regression-test/suites/mysql_ssl_p0/test_mysql_connection.groovy index 25e00d1b476717..f95ef88058421a 100644 --- a/regression-test/suites/mysql_ssl_p0/test_mysql_connection.groovy +++ b/regression-test/suites/mysql_ssl_p0/test_mysql_connection.groovy @@ -15,31 +15,21 @@ // specific language governing permissions and limitations // under the License. -suite("test_mysql_connection") { +suite("test_mysql_connection") { suite -> + // NOTE: this suite need you install mysql client 5.7 + to support --ssl-mode parameter def executeMySQLCommand = { String command -> - try { - String line; - StringBuilder errMsg = new StringBuilder(); - StringBuilder msg = new StringBuilder(); - Process p = Runtime.getRuntime().exec(new String[]{"/bin/bash", "-c", command}); + def cmds = ["/bin/bash", "-c", command] + logger.info("Execute: ${cmds}".toString()) + Process p = cmds.execute() - BufferedReader errInput = new BufferedReader(new InputStreamReader(p.getErrorStream())); - while ((line = errInput.readLine()) != null) { - errMsg.append(line); - } - assert errMsg.length() == 0: "error occurred!" + errMsg.toString(); - errInput.close(); + def errMsg = new StringBuilder() + def msg = new StringBuilder() + p.waitForProcessOutput(msg, errMsg) - BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream())); - while ((line = input.readLine()) != null) { - msg.append(line); - } - assert msg.toString().contains("version"): "error occurred!" + errMsg.toString(); - input.close(); - } catch (IOException e) { - e.printStackTrace(); - } + assert errMsg.length() == 0: "error occurred!" + errMsg + assert msg.toString().contains("version"): "error occurred!" + errMsg + assert p.exitValue() == 0 } String jdbcUrlConfig = context.config.jdbcUrl; diff --git a/regression-test/suites/nereids_p0/aggregate/aggregate_output_null.groovy b/regression-test/suites/nereids_p0/aggregate/aggregate_output_null.groovy index 39f028bd90dd14..efb905e342d08a 100644 --- a/regression-test/suites/nereids_p0/aggregate/aggregate_output_null.groovy +++ b/regression-test/suites/nereids_p0/aggregate/aggregate_output_null.groovy @@ -74,7 +74,7 @@ suite("aggregate_output_null") { ('z178NhOZ','b'); """ - qt_select """ + order_qt_select """ SELECT t2.a, t1.c, @@ -89,4 +89,4 @@ suite("aggregate_output_null") { sql "DROP TABLE t1" sql "DROP TABLE t2" -} \ No newline at end of file +} diff --git a/regression-test/suites/nereids_p0/sql_functions/aggregate_functions/test_aggregate_collect.groovy b/regression-test/suites/nereids_p0/sql_functions/aggregate_functions/test_aggregate_collect.groovy index 5701e7ddd318be..4c002532c4b381 100644 --- a/regression-test/suites/nereids_p0/sql_functions/aggregate_functions/test_aggregate_collect.groovy +++ b/regression-test/suites/nereids_p0/sql_functions/aggregate_functions/test_aggregate_collect.groovy @@ -67,12 +67,12 @@ suite("test_aggregate_collect") { sql "INSERT INTO ${tableName_12} values(1,10,'2022-11-1',6.8754576), (2,8,'2022-11-3',0.576), (2,10,'2022-11-2',1.234) ,(3,10,'2022-11-2',0.576) ,(5,29,'2022-11-2',6.8754576) ,(6,8,'2022-11-1',6.8754576)" // Nereids does't support array function - // qt_select43 "select topn_array(level,2) from ${tableName_12}" + // order_qt_select43 "select topn_array(level,2) from ${tableName_12}" // Nereids does't support array function - // qt_select44 "select topn_array(level,2,100) from ${tableName_12}" + // order_qt_select44 "select topn_array(level,2,100) from ${tableName_12}" // Nereids does't support array function - // qt_select45 "select topn_array(dt,2,100) from ${tableName_12}" + // order_qt_select45 "select topn_array(dt,2,100) from ${tableName_12}" // Nereids does't support array function - // qt_select46 "select topn_array(num,2,100) from ${tableName_12}" + // order_qt_select46 "select topn_array(num,2,100) from ${tableName_12}" sql "DROP TABLE IF EXISTS ${tableName_12}" } diff --git a/regression-test/suites/nereids_p0/sql_functions/array_functions/test_array_functions_of_array_difference.groovy b/regression-test/suites/nereids_p0/sql_functions/array_functions/test_array_functions_of_array_difference.groovy index da700c2d278dda..c424fe34b72640 100644 --- a/regression-test/suites/nereids_p0/sql_functions/array_functions/test_array_functions_of_array_difference.groovy +++ b/regression-test/suites/nereids_p0/sql_functions/array_functions/test_array_functions_of_array_difference.groovy @@ -43,6 +43,6 @@ suite("test_array_functions_of_array_difference") { // Nereids does't support array function - // qt_select "SELECT *, array_difference(k2) FROM ${tableName}" + // qt_select "SELECT *, array_difference(k2) FROM ${tableName} order by k1" } diff --git a/regression-test/suites/nereids_p0/sql_functions/json_function/test_query_json_array.groovy b/regression-test/suites/nereids_p0/sql_functions/json_function/test_query_json_array.groovy index f95305bb0cf4bb..d452a4c231e19e 100644 --- a/regression-test/suites/nereids_p0/sql_functions/json_function/test_query_json_array.groovy +++ b/regression-test/suites/nereids_p0/sql_functions/json_function/test_query_json_array.groovy @@ -43,6 +43,6 @@ suite("test_query_json_array", "query") { sql "insert into ${tableName} values(4,null,null,'test','2022-01-01 11:11:11');" sql "insert into ${tableName} values(5,1,true,'test','2022-01-01 11:11:11');" // Nereids does't support array function - // qt_sql2 "select json_array('k0',k0,'k1',k1,'k2',k2,'k3',k3,'k4',k4,'k5', null,'k6','k6') from ${tableName};" + // order_qt_sql2 "select json_array('k0',k0,'k1',k1,'k2',k2,'k3',k3,'k4',k4,'k5', null,'k6','k6') from ${tableName};" sql "DROP TABLE ${tableName};" } diff --git a/regression-test/suites/nereids_syntax_p0/lateral_view.groovy b/regression-test/suites/nereids_syntax_p0/lateral_view.groovy index 8bf32cf4ba4468..f7a32d8d246ced 100644 --- a/regression-test/suites/nereids_syntax_p0/lateral_view.groovy +++ b/regression-test/suites/nereids_syntax_p0/lateral_view.groovy @@ -45,6 +45,7 @@ suite("nereids_lateral_view") { LATERAL VIEW explode_json_array_string(c2) lv2 AS clv2 LATERAL VIEW explode_json_array_int(c3) lv3 AS clv3 LATERAL VIEW explode_json_array_double(c4) lv4 AS clv4 + order by c1, c2, c3, c4, clv1, clv2, clv3, clv4 """ order_qt_all_function_outer """ @@ -53,6 +54,7 @@ suite("nereids_lateral_view") { LATERAL VIEW explode_json_array_string_outer(c2) lv2 AS clv2 LATERAL VIEW explode_json_array_int_outer(c3) lv3 AS clv3 LATERAL VIEW explode_json_array_double_outer(c4) lv4 AS clv4 + order by c1, c2, c3, c4, clv1, clv2, clv3, clv4 """ order_qt_column_prune """ @@ -61,6 +63,7 @@ suite("nereids_lateral_view") { LATERAL VIEW explode_json_array_string_outer(c2) lv2 AS clv2 LATERAL VIEW explode_json_array_int(c3) lv3 AS clv3 LATERAL VIEW explode_json_array_double_outer(c4) lv4 AS clv4 + order by c1, c2, c3, c4, clv1, clv2, clv3, clv4 """ order_qt_alias_query """ @@ -69,6 +72,7 @@ suite("nereids_lateral_view") { LATERAL VIEW explode_json_array_string_outer(c2) lv2 AS clv2 LATERAL VIEW explode_json_array_int(c3) lv3 AS clv3 LATERAL VIEW explode_json_array_double_outer(c4) lv4 AS clv4 + order by c1, c2, c3, c4, clv1, clv2, clv3, clv4 """ order_qt_function_nested """ diff --git a/regression-test/suites/nereids_syntax_p0/set_operation.groovy b/regression-test/suites/nereids_syntax_p0/set_operation.groovy index c6838f99867f2f..94baeb47f31acb 100644 --- a/regression-test/suites/nereids_syntax_p0/set_operation.groovy +++ b/regression-test/suites/nereids_syntax_p0/set_operation.groovy @@ -193,14 +193,14 @@ suite("test_nereids_set_operation") { """ // test_union_basic - qt_union30 """select 1, 2 union select 1.01, 2.0 union (select 0.0001, 0.0000001)""" - qt_union31 """select 1, 2 union (select "hell0", "")""" - qt_union32 """select 1, 2 union select 1.0, 2.0 union (select 1.00000000, 2.00000)""" - qt_union33 """select 1, 2 union all select 1.0, 2.0 union (select 1.00000000, 2.00000) """ - qt_union34 """select 1, 2 union all select 1.0, 2.0 union all (select 1.00000000, 2.00000) """ - qt_union35 """select 1, 2 union select 1.0, 2.0 union all (select 1.00000000, 2.00000) """ - qt_union36 """select 1, 2 union distinct select 1.0, 2.0 union distinct (select 1.00000000, 2.00000) """ - qt_union38 """select "2016-07-01" union (select "2016-07-02")""" + qt_union30 """select 1 c1, 2 union select 1.01, 2.0 union (select 0.0001, 0.0000001) order by c1""" + qt_union31 """select 1 c1, 2 union (select "hell0", "") order by c1""" + qt_union32 """select 1 c1, 2 union select 1.0, 2.0 union (select 1.00000000, 2.00000) order by c1""" + qt_union33 """select 1 c1, 2 union all select 1.0, 2.0 union (select 1.00000000, 2.00000) order by c1""" + qt_union34 """select 1 c1, 2 union all select 1.0, 2.0 union all (select 1.00000000, 2.00000) order by c1""" + qt_union35 """select 1 c1, 2 union select 1.0, 2.0 union all (select 1.00000000, 2.00000) order by c1""" + qt_union36 """select 1 c1, 2 union distinct select 1.0, 2.0 union distinct (select 1.00000000, 2.00000) order by c1""" + qt_union38 """select "2016-07-01" c1 union (select "2016-07-02") order by c1""" // test_union_bug // PALO-3617 @@ -275,7 +275,7 @@ suite("test_nereids_set_operation") { (select k1, k5 from setOperationTable) """ - qt_union43 """select '2020-05-25' day from test_table union all select day from test_table;""" + order_qt_union43 """select '2020-05-25' day from test_table union all select day from test_table;""" qt_union44 """ select * from diff --git a/regression-test/suites/query_p0/aggregate/aggregate_output_null.groovy b/regression-test/suites/query_p0/aggregate/aggregate_output_null.groovy index 870bdecd7b9dd4..c563d9f5a3bdc2 100644 --- a/regression-test/suites/query_p0/aggregate/aggregate_output_null.groovy +++ b/regression-test/suites/query_p0/aggregate/aggregate_output_null.groovy @@ -72,7 +72,7 @@ suite("aggregate_output_null") { ('z178NhOZ','b'); """ - qt_select """ + order_qt_select """ SELECT t2.a, t1.c, @@ -87,4 +87,4 @@ suite("aggregate_output_null") { sql "DROP TABLE t1" sql "DROP TABLE t2" -} \ No newline at end of file +} diff --git a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_collect.groovy b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_collect.groovy index 3beea000432d1e..8f600c24cd8654 100644 --- a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_collect.groovy +++ b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_collect.groovy @@ -177,7 +177,7 @@ suite("test_aggregate_collect") { ${tableName} """ - qt_select """ + order_qt_select """ SELECT size(collect_list(c_bool,1)), size(collect_list(c_tinyint,1)), @@ -449,7 +449,7 @@ suite("test_aggregate_collect") { ${tableName_11} """ - qt_select """ + order_qt_select """ SELECT size(group_uniq_array(c_bool,1)), size(group_uniq_array(c_tinyint,1)), @@ -472,7 +472,7 @@ suite("test_aggregate_collect") { ${tableName_11} """ - qt_select """ + order_qt_select """ SELECT size(group_array(c_bool,1)), size(group_array(c_tinyint,1)), @@ -613,7 +613,7 @@ suite("test_aggregate_collect") { sql """ CREATE TABLE IF NOT EXISTS ${tableName_12} ( id int, - level int, + level int, dt datev2, num decimal(27,9) ) @@ -624,9 +624,9 @@ suite("test_aggregate_collect") { """ sql "INSERT INTO ${tableName_12} values(1,10,'2022-11-1',6.8754576), (2,8,'2022-11-3',0.576), (2,10,'2022-11-2',1.234) ,(3,10,'2022-11-2',0.576) ,(5,29,'2022-11-2',6.8754576) ,(6,8,'2022-11-1',6.8754576)" - qt_select43 "select topn_array(level,2) from ${tableName_12}" - qt_select44 "select topn_array(level,2,100) from ${tableName_12}" - qt_select45 "select topn_array(dt,2,100) from ${tableName_12}" - qt_select46 "select topn_array(num,2,100) from ${tableName_12}" + order_qt_select43 "select topn_array(level,2) from ${tableName_12}" + order_qt_select44 "select topn_array(level,2,100) from ${tableName_12}" + order_qt_select45 "select topn_array(dt,2,100) from ${tableName_12}" + order_qt_select46 "select topn_array(num,2,100) from ${tableName_12}" sql "DROP TABLE IF EXISTS ${tableName_12}" }