Skip to content

Commit

Permalink
[SPARK-23356][SQL][TEST] add new test cases for a + 1,a + b and Rand …
Browse files Browse the repository at this point in the history
…in SetOperationSuite

## What changes were proposed in this pull request?

The purpose of this PR is supplement new test cases for a + 1,a + b and Rand in SetOperationSuite.
It comes from the comment of closed PR:apache#20541, thanks.

## How was this patch tested?

add new test cases

Closes apache#23138 from heary-cao/UnionPushTestCases.

Authored-by: caoxuewen <cao.xuewen@zte.com.cn>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
  • Loading branch information
heary-cao authored and jackylee-ch committed Feb 18, 2019
1 parent cf3c302 commit 779a973
Showing 1 changed file with 28 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions.{Alias, And, GreaterThan, GreaterThanOrEqual, If, Literal, ReplicateRows}
import org.apache.spark.sql.catalyst.expressions.{And, GreaterThan, GreaterThanOrEqual, If, Literal, Rand, ReplicateRows}
import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
Expand Down Expand Up @@ -196,4 +196,31 @@ class SetOperationSuite extends PlanTest {
))
comparePlans(expectedPlan, rewrittenPlan)
}

test("SPARK-23356 union: expressions with literal in project list are pushed down") {
val unionQuery = testUnion.select(('a + 1).as("aa"))
val unionOptimized = Optimize.execute(unionQuery.analyze)
val unionCorrectAnswer =
Union(testRelation.select(('a + 1).as("aa")) ::
testRelation2.select(('d + 1).as("aa")) ::
testRelation3.select(('g + 1).as("aa")) :: Nil).analyze
comparePlans(unionOptimized, unionCorrectAnswer)
}

test("SPARK-23356 union: expressions in project list are pushed down") {
val unionQuery = testUnion.select(('a + 'b).as("ab"))
val unionOptimized = Optimize.execute(unionQuery.analyze)
val unionCorrectAnswer =
Union(testRelation.select(('a + 'b).as("ab")) ::
testRelation2.select(('d + 'e).as("ab")) ::
testRelation3.select(('g + 'h).as("ab")) :: Nil).analyze
comparePlans(unionOptimized, unionCorrectAnswer)
}

test("SPARK-23356 union: no pushdown for non-deterministic expression") {
val unionQuery = testUnion.select('a, Rand(10).as("rnd"))
val unionOptimized = Optimize.execute(unionQuery.analyze)
val unionCorrectAnswer = unionQuery.analyze
comparePlans(unionOptimized, unionCorrectAnswer)
}
}

0 comments on commit 779a973

Please sign in to comment.