Skip to content

Commit

Permalink
improvement and test fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@scwf
scwf committed Dec 27, 2014
1 parent e99a26c commit 32a595b
Show file tree
Hide file tree
Showing 3 changed files with 120 additions and 62 deletions.
14 changes: 7 additions & 7 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@
<id>central</id>
<!-- This should be at top, it makes maven try the central repo first and then others and hence faster dep resolution -->
<name>Maven Repository</name>
<url>https://repo1.maven.org/maven2</url>
<url>http://repo1.maven.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>
Expand All @@ -167,7 +167,7 @@
<repository>
<id>apache-repo</id>
<name>Apache Repository</name>
<url>https://repository.apache.org/content/repositories/releases</url>
<url>http://repository.apache.org/content/repositories/releases</url>
<releases>
<enabled>true</enabled>
</releases>
Expand All @@ -178,7 +178,7 @@
<repository>
<id>jboss-repo</id>
<name>JBoss Repository</name>
<url>https://repository.jboss.org/nexus/content/repositories/releases</url>
<url>http://repository.jboss.org/nexus/content/repositories/releases</url>
<releases>
<enabled>true</enabled>
</releases>
Expand All @@ -189,7 +189,7 @@
<repository>
<id>mqtt-repo</id>
<name>MQTT Repository</name>
<url>https://repo.eclipse.org/content/repositories/paho-releases</url>
<url>http://repo.eclipse.org/content/repositories/paho-releases</url>
<releases>
<enabled>true</enabled>
</releases>
Expand All @@ -200,7 +200,7 @@
<repository>
<id>cloudera-repo</id>
<name>Cloudera Repository</name>
<url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
<url>http://repository.cloudera.com/artifactory/cloudera-repos</url>
<releases>
<enabled>true</enabled>
</releases>
Expand All @@ -222,7 +222,7 @@
<repository>
<id>spring-releases</id>
<name>Spring Release Repository</name>
<url>https://repo.spring.io/libs-release</url>
<url>http://repo.spring.io/libs-release</url>
<releases>
<enabled>true</enabled>
</releases>
Expand All @@ -234,7 +234,7 @@
<pluginRepositories>
<pluginRepository>
<id>central</id>
<url>https://repo1.maven.org/maven2</url>
<url>http://repo1.maven.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>
Expand Down
127 changes: 92 additions & 35 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ object DefaultOptimizer extends Optimizer {
NullPropagation,
ConstantFolding,
LikeSimplification,
ConditionSimplification,
BooleanSimplification,
SimplifyFilters,
SimplifyCasts,
Expand Down Expand Up @@ -302,7 +303,8 @@ object OptimizeIn extends Rule[LogicalPlan] {
object ConditionSimplification extends Rule[LogicalPlan] with PredicateHelper {

def apply(plan: LogicalPlan): LogicalPlan = plan transform {
case q: LogicalPlan => q transformExpressionsDown {
case q: LogicalPlan => q transformExpressionsUp {
/** 1. one And/Or with same condition. */
// a && a => a
case And(left, right) if left.fastEquals(right) =>
left
Expand All @@ -311,85 +313,139 @@ object ConditionSimplification extends Rule[LogicalPlan] with PredicateHelper {
case Or(left, right) if left.fastEquals(right) =>
left

/** 2. one And/Or with literal conditions that can be merged. */
// a < 2 && a > 2 => false, a > 3 && a > 5 => a > 5
case and @ And(
e1 @ NumericLiteralBinaryComparison(n1, i1),
e2 @ NumericLiteralBinaryComparison(n2, i2)) if n1 == n2 =>
e1 @ NumLitBinComparison(n1, i1),
e2 @ NumLitBinComparison(n2, i2)) if n1 == n2 =>
if (!i1.intersects(i2)) Literal(false)
else if (i1.isSubsetOf(i2)) e1
else if (i1.isSupersetOf(i2)) e2
else if (i1.intersect(i2).isPoint)
EqualTo(n1, Literal(i1.intersect(i2).asInstanceOf[Point[Double]].value, n1.dataType))
else and

// a < 2 || a >= 2 => true, a > 3 || a > 5 => a > 3
case or @ Or(
e1 @ NumericLiteralBinaryComparison(n1, i1),
e2 @ NumericLiteralBinaryComparison(n2, i2)) if n1 == n2 =>
if (i1.intersects(i2)) Literal(true)
e1 @ NumLitBinComparison(n1, i1),
e2 @ NumLitBinComparison(n2, i2)) if n1 == n2 =>
// a hack to avoid bug of spire
val op = Interval.all[Double] -- i1
if (i1.intersects(i2) && i1.union(i2) == Interval.all[Double]) Literal(true)
else if (op(op.size - 1) == i2) Literal(true)
else if (i1.isSubsetOf(i2)) e2
else if (i1.isSupersetOf(i2)) e1
else or

// (a < 3 && b > 5) || a > 2 => b > 5 || a > 2
case Or(left1 @ And(left2, right2), right1) =>
And(Or(left2, right1), Or(right2, right1))

/** 3. Two And/Or with literal condition that can be merged, do a transformation to reuse 2. */
// (a < 3 || b > 5) || a > 2 => true, (b > 5 || a < 3) || a > 2 => true
case Or( Or(
e1 @ NumericLiteralBinaryComparison(n1, i1), e2 @ NumericLiteralBinaryComparison(n2, i2)),
right @ NumericLiteralBinaryComparison(n3, i3)) =>
case or @ Or(
Or(e1 @ NumLitBinComparison(n1, i1), e2 @ NumLitBinComparison(n2, i2)),
right @ NumLitBinComparison(n3, i3)) =>
if (n3 fastEquals n1) {
Or(Or(e1, right), e2)
} else {
} else if (n3 fastEquals n2) {
Or(Or(e2, right), e1)
} else {
or
}

// (b > 5 && a < 2) && a > 3 => false, (a < 2 && b > 5) && a > 3 => false
case And(And(
e1 @ NumericLiteralBinaryComparison(n1, i1), e2 @ NumericLiteralBinaryComparison(n2, i2)),
right @ NumericLiteralBinaryComparison(n3, i3)) =>
case and @ And(
And(e1 @ NumLitBinComparison(n1, i1), e2 @ NumLitBinComparison(n2, i2)),
right @ NumLitBinComparison(n3, i3)) =>
if (n3 fastEquals n1) {
And(And(e1, right), e2)
} else {
} else if (n3 fastEquals n2) {
And(And(e2, right), e1)
} else {
and
}

// (a < 2 || b > 5) && a > 3 => b > 5 && a > 3
case And(left1@Or(left2, right2), right1) =>
Or(And(left2, right1), And(right2, right1))

// (a && b && c && ...) || (a && b && d && ...) || (a && b && e && ...) ... =>
// using formula: a && (b || c) = (a && b) || (a && c)
case And(
left1 @ Or(left2 @ NumLitBinComparison(n1, i1), right2 @ NumLitBinComparison(n2, i2)),
right1 @ NumLitBinComparison(n3, i3))
if ((n3.fastEquals(n1) && i3 != i1) || (n3.fastEquals(n2) && i3 != i2)) =>
Or(And(left2, right1), And(right2, right1))

// (a < 3 && b > 5) || a > 2 => b > 5 || a > 2.
// using formula: a || (b && c) = (a || b) && (a || c)
case Or(
left1 @ And(left2 @ NumLitBinComparison(n1, i1), right2 @ NumLitBinComparison(n2, i2)),
right1 @ NumLitBinComparison(n3, i3))
if ((n3.fastEquals(n1) && i3 != i1) || (n3.fastEquals(n2) && i3 != i2)) =>
Or(And(left2, right1), And(right2, right1))

/** 4. And/Or whose one child is literal condition, the other is Or/And */
// (a < 2 || b > 5) && a < 2 => a < 2
case And(
left1 @ Or(left2 @ NumLitBinComparison(_, _), right2 @ NumLitBinComparison(_, _)),
right1 @ NumLitBinComparison(_, _))
if (right1 fastEquals left2) || (right1 fastEquals right2) =>
right1

// (a < 3 && b > 5) || a < 3 => a < 3
case Or(
left1 @ And(left2 @ NumLitBinComparison(_, _), right2 @ NumLitBinComparison(_, _)),
right1 @ NumLitBinComparison(_, _))
if (right1 fastEquals left2) || (right1 fastEquals right2) =>
right1

// 5. (a && b && c && ...) || (a && b && d && ...) || (a && b && e && ...) ... =>
// a && b && ((c && ...) || (d && ...) || (e && ...) || ...)
case or @ Or(left, right) =>
val lhsSet = splitConjunctivePredicates(left).toSet
val rhsSet = splitConjunctivePredicates(right).toSet
val common = lhsSet.intersect(rhsSet)
(lhsSet.diff(common).reduceOption(And) ++ rhsSet.diff(common).reduceOption(And))
.reduceOption(Or)
.map(_ :: common.toList)
.getOrElse(common.toList)
.reduce(And)
val ldiff = lhsSet.diff(common)
val rdiff = rhsSet.diff(common)
if (common.size == 0) {
or
}else if ( ldiff.size == 0 || rdiff == 0) {
common.reduce(And)
} else {
(ldiff.reduceOption(And) ++ rdiff.reduceOption(And))
.reduceOption(Or)
.map(_ :: common.toList)
.getOrElse(common.toList)
.reduce(And)
}

// (a || b || c || ...) && (a || b || d || ...) && (a || b || e || ...) ... =>
// 6. (a || b || c || ...) && (a || b || d || ...) && (a || b || e || ...) ... =>
// (a || b) || ((c || ...) && (f || ...) && (e || ...) && ...)
case and @ And(left, right) =>
val lhsSet = splitDisjunctivePredicates(left).toSet
val rhsSet = splitDisjunctivePredicates(right).toSet
val common = lhsSet.intersect(rhsSet)
(lhsSet.diff(common).reduceOption(Or) ++ rhsSet.diff(common).reduceOption(Or))
.reduceOption(And)
.map(_ :: common.toList)
.getOrElse(common.toList)
.reduce(Or)
val ldiff = lhsSet.diff(common)
val rdiff = rhsSet.diff(common)

if (common.size == 0) {
and
}else if (ldiff.size == 0 || rdiff.size == 0) {
common.reduce(Or)
} else {
val x = (ldiff.reduceOption(Or) ++ rdiff.reduceOption(Or))
.reduceOption(And)
.map(_ :: common.toList)
.getOrElse(common.toList)
.reduce(Or)
x
}

case other => other
}
}

private implicit class NumericLiteral(e: Literal) {
def toDouble = Cast(e, DoubleType).eval().asInstanceOf[Double]
}

object NumericLiteralBinaryComparison {
object NumLitBinComparison {
def unapply(e: Expression): Option[(NamedExpression, Interval[Double])] = e match {
case LessThan(n: NamedExpression, l @ Literal(_, _: NumericType)) => Some((n, Interval.below(l.toDouble)))
case LessThan(n: NamedExpression, l @ Literal(_, _: NumericType)) => Some((n, Interval.below(l.toDouble)))
case LessThan(l @ Literal(_, _: NumericType), n: NamedExpression) => Some((n, Interval.atOrAbove(l.toDouble)))

case GreaterThan(n: NamedExpression, l @ Literal(_, _: NumericType)) => Some((n, Interval.above(l.toDouble)))
Expand All @@ -402,6 +458,7 @@ object ConditionSimplification extends Rule[LogicalPlan] with PredicateHelper {
case GreaterThanOrEqual(l @ Literal(_, _: NumericType), n: NamedExpression) => Some((n, Interval.below(l.toDouble)))

case EqualTo(n: NamedExpression, l @ Literal(_, _: NumericType)) => Some((n, Interval.point(l.toDouble)))
case other => None
}
}
}
Expand Down
41 changes: 21 additions & 20 deletions ...src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConditionSimplificationSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class ConditionSimplificationSuite extends PlanTest {
val batches =
Batch("AnalysisNodes", Once,
EliminateAnalysisOperators) ::
Batch("Constant Folding", FixedPoint(10),
Batch("Constant Folding", FixedPoint(50),
NullPropagation,
ConstantFolding,
ConditionSimplification,
Expand All @@ -55,11 +55,12 @@ class ConditionSimplificationSuite extends PlanTest {
comparePlans(optimized, expected)
}

test("literal in front of attribute") {
checkCondition(Literal(1) < 'a || Literal(2) < 'a, 'a > 1)
test("literals in front of attribute") {
checkCondition(Literal(1) < 'a || Literal(2) < 'a, Literal(1) < 'a)
checkCondition(Literal(1) < 'a && Literal(2) < 'a, Literal(2) < 'a)
}

test("combine the same condition") {
test("And/Or with the same conditions") {
checkCondition('a < 1 || 'a < 1, 'a < 1)
checkCondition('a < 1 || 'a < 1 || 'a < 1 || 'a < 1, 'a < 1)
checkCondition('a > 2 && 'a > 2, 'a > 2)
Expand All @@ -69,7 +70,7 @@ class ConditionSimplificationSuite extends PlanTest {

test("combine literal binary comparison") {
checkCondition('a === 1 && 'a < 1)
checkCondition('a === 1 || 'a < 1, 'a <= 1)
checkCondition('a === 1 || 'a < 1, 'a === 1 || 'a < 1)

checkCondition('a === 1 && 'a === 2)
checkCondition('a === 1 || 'a === 2, 'a === 1 || 'a === 2)
Expand All @@ -83,6 +84,10 @@ class ConditionSimplificationSuite extends PlanTest {
checkCondition('a > 3 && 'a > 2, 'a > 3)
checkCondition('a > 3 || 'a > 2, 'a > 2)

checkCondition('a < 2 || 'a === 3 , 'a < 2 || 'a === 3)
checkCondition('a === 3 || 'a > 5, 'a === 3 || 'a > 5)
checkCondition('a < 2 || 'a > 5, 'a < 2 || 'a > 5)

checkCondition('a >= 1 && 'a <= 1, 'a === 1)

}
Expand All @@ -103,56 +108,52 @@ class ConditionSimplificationSuite extends PlanTest {
checkCondition('a < 1 || 'b > 2 || 'a >= 1)
checkCondition('a < 1 && 'b > 2 && 'a >= 1)

checkCondition('a < 2 || 'b > 3 || 'b > 2, 'a < 2 || 'b > 2)
checkCondition('a < 2 && 'b > 3 && 'b > 2, 'a < 2 && 'b > 3)
checkCondition('a < 2 || 'b > 3 || 'b > 2, 'b > 2 || 'a < 2)
checkCondition('a < 2 && 'b > 3 && 'b > 2, 'b > 3 && 'a < 2)

checkCondition('a < 2 || ('b > 3 || 'b > 2), 'b > 2 || 'a < 2)
checkCondition('a < 2 && ('b > 3 && 'b > 2), 'b > 3 && 'a < 2)
checkCondition('a < 2 || ('b > 3 || 'b > 2), 'a < 2 || 'b > 2)
checkCondition('a < 2 && ('b > 3 && 'b > 2), 'a < 2 && 'b > 3)

checkCondition('a < 2 || 'a === 3 || 'a > 5, 'a < 2 || 'a === 3 || 'a > 5)
}

test("combine predicate : 2 difference combine") {
checkCondition(('a < 2 || 'a > 3) && 'a > 4, 'a > 4)
checkCondition(('a < 2 || 'b > 3) && 'a < 2, 'a < 2)

checkCondition('a < 2 || ('a >= 2 && 'b > 1), 'b > 1 || 'a < 2)
checkCondition('a < 2 || ('a === 2 && 'b > 1), 'a < 2 || ('a === 2 && 'b > 1))

checkCondition('a > 3 || ('a > 2 && 'a < 4), 'a > 2)
checkCondition(('a < 2 && 'b > 3) || 'a < 2, 'a < 2)
}

test("multi left, single right") {
checkCondition(('a < 2 || 'a > 3 || 'b > 5) && 'a < 2, 'a < 2)
}

test("multi left, multi right") {
checkCondition(('a < 2 || 'b > 3) && ('a < 2 || 'c > 5), 'a < 2 || ('b > 3 && 'c > 5))
checkCondition(('a < 2 || 'b > 3) && ('a < 2 || 'c > 5), ('b > 3 && 'c > 5) || 'a < 2)

var input: Expression = ('a === 'b || 'b > 3) && ('a === 'b || 'a > 3) && ('a === 'b || 'a < 5)
var expected: Expression = 'a === 'b || ('b > 3 && 'a > 3 && 'a < 5)
var expected: Expression = ('a > 3 && 'a < 5 && 'b > 3) || 'a === 'b
checkCondition(input, expected)

input = ('a === 'b || 'b > 3) && ('a === 'b || 'a > 3) && ('a === 'b || 'a > 1)
expected = 'a === 'b || ('b > 3 && 'a > 3)
expected = ('a > 3 && 'b > 3) || 'a === 'b
checkCondition(input, expected)

input = ('a === 'b && 'b > 3 && 'c > 2) ||
('a === 'b && 'c < 1 && 'a === 5) ||
('a === 'b && 'b < 5 && 'a > 1)

expected = ('a === 'b) &&
expected =
(((('b > 3) && ('c > 2)) ||
(('c < 1) && ('a === 5))) ||
(('b < 5) && ('a > 1)))
(('b < 5) && ('a > 1))) && ('a === 'b)
checkCondition(input, expected)

input = ('a < 2 || 'b > 5 || 'a < 2 || 'b > 1) && ('a < 2 || 'b > 1)
expected = 'a < 2 || 'b > 1
checkCondition(input, expected)

input = ('a === 'b || 'b > 5) && ('a === 'b || 'c > 3) && ('a === 'b || 'b > 1)
expected = ('a === 'b) || ('c > 3 && 'b > 5)
expected = ('b > 5 && 'c > 3) || ('a === 'b)
checkCondition(input, expected)
}
}

0 comments on commit 32a595b

Please sign in to comment.