Skip to content

Commit

Permalink
(Refactor)[Nereids] Combine operator and plan (#10786)
Browse files Browse the repository at this point in the history
in #9755, we split plan into plan & operator, but in subsequent development, we found the rule became complex and counter intuition: 
1. we must create an operator instance, then wrap a plan by the operator type.
2. relational algebra(operator) not contains children 

e.g.
```java
logicalProject().then(project -> {
    List<NamedExpression> boundSlots =
        bind(project.operator.getProjects(), project.children(), project);
    LogicalProject op = new LogicalProject(flatBoundStar(boundSlots));
    // wrap a plan
    return new LogicalUnaryPlan(op, project.child());
})
```

after combine operator and plan, the code become to:
```java
logicalProject().then(project -> {
    List<NamedExpression> boundSlots =
        bind(project.getProjects(), project.children(), project);
    return new LogicalProject(flatBoundStar(boundSlots), project.child());
})
```

Originally, we thought it would be convenient for `Memo.copyIn()` after split plan & operator, because Memo don't known how to re-new the plan(assembling child plan in the children groups) by the plan type. So plan must provide the `withChildren()` abstract method to assembling children. The less plan type, the lower code cost we have(logical/physical with leaf/unary/binary plan, about 6 plans, no concrete plan e.g. LogicalAggregatePlan). 

But the convenient make negative effect that difficult to understand, and people must known the concept then can develop some new rules, and rule become ugly. So we combine the plan & operator, make the rule as simple as possible, the negative effect is we must overwrite some withXxx for all concrete plan, e.g. LogicalAggregate, PhysicalHashJoin.
  • Loading branch information
924060929 authored Jul 13, 2022
1 parent 56b5556 commit e78cca1
Show file tree
Hide file tree
Showing 161 changed files with 2,068 additions and 2,533 deletions.
2 changes: 1 addition & 1 deletion fe/fe-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -859,7 +859,7 @@ under the License.
<configuration>
<proc>only</proc>
<compilerArgs>
<arg>-AoperatorPath=${basedir}/src/main/java/org/apache/doris/nereids</arg>
<arg>-AplanPath=${basedir}/src/main/java/org/apache/doris/nereids</arg>
</compilerArgs>
<includes>
<include>org/apache/doris/nereids/pattern/generator/PatternDescribableProcessPoint.java</include>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ public enum Operator {
// The intermediate type for this function if it is constant regardless of
// input type. Set to null if it can only be determined during analysis.
private final org.apache.doris.catalog.Type intermediateType;

Operator(String description, TAggregationOp thriftOp,
org.apache.doris.catalog.Type intermediateType) {
this.description = description;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,8 @@ private enum DateLiteralType {
DATEV2(3);

private final int value;
private DateLiteralType(int value) {

DateLiteralType(int value) {
this.value = value;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ public enum InternalErrorCode {
TASKS_ABORT_ERR(104);

private long errCode;
private InternalErrorCode(long code) {

InternalErrorCode(long code) {
this.errCode = code;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,7 @@ private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physical
planChildren.add(chooseBestPlan(groupExpression.child(i), inputPropertiesList.get(i)));
}

Plan plan = ((PhysicalPlan) groupExpression.getOperator().toTreeNode(groupExpression)).withChildren(
planChildren);
Plan plan = groupExpression.getPlan().withChildren(planChildren);
if (!(plan instanceof PhysicalPlan)) {
throw new AnalysisException("generate logical plan");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
package org.apache.doris.nereids.analyzer;

import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.ExpressionType;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
Expand All @@ -35,7 +35,7 @@
public class UnboundAlias extends NamedExpression implements UnaryExpression, Unbound {

public UnboundAlias(Expression child) {
super(NodeType.UNBOUND_ALIAS, child);
super(ExpressionType.UNBOUND_ALIAS, child);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
package org.apache.doris.nereids.analyzer;

import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.ExpressionType;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;

import com.google.common.base.Joiner;
Expand All @@ -37,7 +37,7 @@ public class UnboundFunction extends Expression implements Unbound {
private final boolean isDistinct;

public UnboundFunction(String name, boolean isDistinct, List<Expression> arguments) {
super(NodeType.UNBOUND_FUNCTION, arguments.toArray(new Expression[0]));
super(ExpressionType.UNBOUND_FUNCTION, arguments.toArray(new Expression[0]));
this.name = Objects.requireNonNull(name, "name can not be null");
this.isDistinct = isDistinct;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,38 +19,51 @@

import org.apache.doris.nereids.analyzer.identifier.TableIdentifier;
import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.operators.OperatorType;
import org.apache.doris.nereids.operators.plans.logical.LogicalLeafOperator;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.UnboundLogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;

import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;

import java.util.List;
import java.util.Optional;

/**
* Represent a relation plan node that has not been bound.
*/
public class UnboundRelation extends LogicalLeafOperator implements Unbound {
public class UnboundRelation extends LogicalLeaf implements Unbound {
private final List<String> nameParts;

public UnboundRelation(List<String> nameParts) {
super(OperatorType.LOGICAL_UNBOUND_RELATION);
this(nameParts, Optional.empty(), Optional.empty());
}

public UnboundRelation(List<String> nameParts, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties) {
super(PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties);
this.nameParts = nameParts;
}

public UnboundRelation(TableIdentifier identifier) {
this(identifier, Optional.empty(), Optional.empty());
}

/**
* Constructor for UnboundRelation.
*
* @param identifier relation identifier
*/
public UnboundRelation(TableIdentifier identifier) {
super(OperatorType.LOGICAL_UNBOUND_RELATION);
public UnboundRelation(TableIdentifier identifier, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties) {
super(PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties);
this.nameParts = Lists.newArrayList();
if (identifier.getDatabaseName().isPresent()) {
nameParts.add(identifier.getDatabaseName().get());
Expand All @@ -72,6 +85,16 @@ public LogicalProperties computeLogicalProperties(Plan... inputs) {
return new UnboundLogicalProperties();
}

@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new UnboundRelation(nameParts, groupExpression, Optional.of(logicalProperties));
}

@Override
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new UnboundRelation(nameParts, Optional.empty(), logicalProperties);
}

@Override
public List<Slot> computeOutput() {
throw new UnboundException("output");
Expand All @@ -82,6 +105,11 @@ public String toString() {
return "UnresolvedRelation" + "(" + StringUtils.join(nameParts, ".") + ")";
}

@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitUnboundRelation(this, context);
}

@Override
public List<Expression> getExpressions() {
throw new UnsupportedOperationException(this.getClass().getSimpleName() + " don't support getExpression()");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.doris.nereids.analyzer;

import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.nereids.trees.expressions.ExpressionType;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.util.Utils;
Expand All @@ -33,7 +33,7 @@ public class UnboundSlot extends Slot implements Unbound {
private final List<String> nameParts;

public UnboundSlot(List<String> nameParts) {
super(NodeType.UNBOUND_SLOT);
super(ExpressionType.UNBOUND_SLOT);
this.nameParts = nameParts;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.apache.doris.nereids.analyzer;

import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.nereids.trees.expressions.ExpressionType;
import org.apache.doris.nereids.trees.expressions.LeafExpression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
Expand All @@ -35,7 +35,7 @@ public class UnboundStar extends NamedExpression implements LeafExpression, Unbo
private final List<String> qualifier;

public UnboundStar(List<String> qualifier) {
super(NodeType.UNBOUND_STAR);
super(ExpressionType.UNBOUND_STAR);
this.qualifier = qualifier;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@
import org.apache.doris.common.Id;
import org.apache.doris.nereids.PlanContext;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.operators.Operator;
import org.apache.doris.nereids.operators.OperatorVisitor;
import org.apache.doris.nereids.operators.plans.physical.PhysicalAggregate;
import org.apache.doris.nereids.operators.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.operators.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.operators.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.statistics.StatsDeriveResult;

import com.google.common.base.Preconditions;
Expand All @@ -43,7 +43,7 @@ public class CostCalculator {
public static double calculateCost(GroupExpression groupExpression) {
PlanContext planContext = new PlanContext(groupExpression);
CostEstimator costCalculator = new CostEstimator();
CostEstimate costEstimate = groupExpression.getOperator().accept(costCalculator, planContext);
CostEstimate costEstimate = groupExpression.getPlan().accept(costCalculator, planContext);
return costFormula(costEstimate);
}

Expand All @@ -55,14 +55,14 @@ private static double costFormula(CostEstimate costEstimate) {
+ costEstimate.getNetworkCost() * networkCostWeight;
}

private static class CostEstimator extends OperatorVisitor<CostEstimate, PlanContext> {
private static class CostEstimator extends PlanVisitor<CostEstimate, PlanContext> {
@Override
public CostEstimate visitOperator(Operator operator, PlanContext context) {
public CostEstimate visit(Plan plan, PlanContext context) {
return CostEstimate.zero();
}

@Override
public CostEstimate visitPhysicalAggregation(PhysicalAggregate physicalAggregate, PlanContext context) {
public CostEstimate visitPhysicalAggregate(PhysicalAggregate<Plan> aggregate, PlanContext context) {
StatsDeriveResult statistics = context.getStatisticsWithCheck();
return CostEstimate.ofCpu(statistics.computeSize());
}
Expand All @@ -74,7 +74,7 @@ public CostEstimate visitPhysicalOlapScan(PhysicalOlapScan physicalOlapScan, Pla
}

@Override
public CostEstimate visitPhysicalHashJoin(PhysicalHashJoin physicalHashJoin, PlanContext context) {
public CostEstimate visitPhysicalHashJoin(PhysicalHashJoin<Plan, Plan> physicalHashJoin, PlanContext context) {
Preconditions.checkState(context.getGroupExpression().arity() == 2);
Preconditions.checkState(context.getChildrenStats().size() == 2);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Arithmetic;
import org.apache.doris.nereids.trees.expressions.Between;
import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.ExpressionType;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.LessThan;
Expand Down Expand Up @@ -170,7 +170,7 @@ public Expr visitBetween(Between between, PlanTranslatorContext context) {

@Override
public Expr visitCompoundPredicate(CompoundPredicate compoundPredicate, PlanTranslatorContext context) {
NodeType nodeType = compoundPredicate.getType();
ExpressionType nodeType = compoundPredicate.getType();
org.apache.doris.analysis.CompoundPredicate.Operator staleOp;
switch (nodeType) {
case OR:
Expand Down
Loading

0 comments on commit e78cca1

Please sign in to comment.