diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index 82c7af684459f..6ee479939d25c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -65,8 +65,7 @@ abstract class BinaryPredicate extends BinaryExpression with Predicate { def nullable = left.nullable || right.nullable } -case class Not(child: Expression) extends Predicate with trees.UnaryNode[Expression] { - def references = child.references +case class Not(child: Expression) extends UnaryExpression with Predicate { override def foldable = child.foldable def nullable = child.nullable override def toString = s"NOT $child" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index c0a09a16ac98d..e11155539fd75 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -94,11 +94,59 @@ object ConstantFolding extends Rule[LogicalPlan] { case q: LogicalPlan => q transformExpressionsDown { // Skip redundant folding of literals. case l: Literal => l + case e @ If(Literal(v, _), trueValue, falseValue) => if(v == true) trueValue else falseValue + case e @ In(Literal(v, _), list) if(list.exists(c => c match { + case Literal(candidate, _) if(candidate == v) => true + case _ => false + })) => Literal(true, BooleanType) case e if e.foldable => Literal(e.eval(null), e.dataType) } } } +/** + * The expression may be constant value, due to one or more of its children expressions is null or + * not null constantly, replaces [[catalyst.expressions.Expression Expressions]] with equivalent + * [[catalyst.expressions.Literal Literal]] values if possible caused by that. + */ +object NullPropagation extends Rule[LogicalPlan] { + def apply(plan: LogicalPlan): LogicalPlan = plan transform { + case q: LogicalPlan => q transformExpressionsUp { + case l: Literal => l + case e @ IsNull(Literal(null, _)) => Literal(true, BooleanType) + case e @ IsNull(Literal(_, _)) => Literal(false, BooleanType) + case e @ IsNull(c @ Rand) => Literal(false, BooleanType) + case e @ IsNotNull(Literal(null, _)) => Literal(false, BooleanType) + case e @ IsNotNull(Literal(_, _)) => Literal(true, BooleanType) + case e @ IsNotNull(c @ Rand) => Literal(true, BooleanType) + case e @ GetItem(Literal(null, _), _) => Literal(null, e.dataType) + case e @ GetItem(_, Literal(null, _)) => Literal(null, e.dataType) + case e @ GetField(Literal(null, _), _) => Literal(null, e.dataType) + case e @ Coalesce(children) => { + val newChildren = children.filter(c => c match { + case Literal(null, _) => false + case _ => true + }) + if(newChildren.length == null) { + Literal(null, e.dataType) + } else if(newChildren.length == children.length){ + e + } else { + Coalesce(newChildren) + } + } + // TODO put exceptional cases(Unary & Binary Expression) before here. + case e: UnaryExpression => e.child match { + case Literal(null, _) => Literal(null, e.dataType) + } + case e: BinaryExpression => e.children match { + case Literal(null, _) :: right :: Nil => Literal(null, e.dataType) + case left :: Literal(null, _) :: Nil => Literal(null, e.dataType) + } + } + } +} + /** * Simplifies boolean expressions where the answer can be determined without evaluating both sides. * Note that this rule can eliminate expressions that might otherwise have been evaluated and thus