Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SPARK-3329: [SQL] Don't depend on Hive SET pair ordering. #2220

Closed
wants to merge 3 commits into from
Closed

SPARK-3329: [SQL] Don't depend on Hive SET pair ordering. #2220

Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cf11b0e
Fix flakey HiveQuerySuite test
aarondav Jul 21, 2014
6525d8e
Handle cases where SET returns Rows of (single) strings
willb Sep 2, 2014
3b3e205
Collapse collectResults case match in HiveQuerySuite
willb Sep 3, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 28 additions & 21 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -520,62 +520,69 @@ class HiveQuerySuite extends HiveComparisonTest {
val testKey = "spark.sql.key.usedfortestonly"
val testVal = "test.val.0"
val nonexistentKey = "nonexistent"

val KV = "([^=]+)=([^=]*)".r
def collectResults(rdd: SchemaRDD): Set[(String, String)] =
rdd.collect().map {
case Row(key: String, value: String) => key -> value
case Row(kv: String) => kv match {
case KV(key, value) => key -> value
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor style nit: this could just be: case Row(KV(key,value)) => ... I believe.

}
}.toSet
clear()

// "set" itself returns all config variables currently specified in SQLConf.
// TODO: Should we be listing the default here always? probably...
assert(sql("SET").collect().size == 0)

assertResult(Array(s"$testKey=$testVal")) {
sql(s"SET $testKey=$testVal").collect().map(_.getString(0))
assertResult(Set(testKey -> testVal)) {
collectResults(hql(s"SET $testKey=$testVal"))
}

assert(hiveconf.get(testKey, "") == testVal)
assertResult(Array(s"$testKey=$testVal")) {
sql(s"SET $testKey=$testVal").collect().map(_.getString(0))
assertResult(Set(testKey -> testVal)) {
collectResults(hql("SET"))
}

sql(s"SET ${testKey + testKey}=${testVal + testVal}")
assert(hiveconf.get(testKey + testKey, "") == testVal + testVal)
assertResult(Array(s"$testKey=$testVal", s"${testKey + testKey}=${testVal + testVal}")) {
sql(s"SET").collect().map(_.getString(0))
assertResult(Set(testKey -> testVal, (testKey + testKey) -> (testVal + testVal))) {
collectResults(hql("SET"))
}

// "set key"
assertResult(Array(s"$testKey=$testVal")) {
sql(s"SET $testKey").collect().map(_.getString(0))
assertResult(Set(testKey -> testVal)) {
collectResults(hql(s"SET $testKey"))
}

assertResult(Array(s"$nonexistentKey=<undefined>")) {
sql(s"SET $nonexistentKey").collect().map(_.getString(0))
assertResult(Set(nonexistentKey -> "<undefined>")) {
collectResults(hql(s"SET $nonexistentKey"))
}

// Assert that sql() should have the same effects as sql() by repeating the above using sql().
clear()
assert(sql("SET").collect().size == 0)

assertResult(Array(s"$testKey=$testVal")) {
sql(s"SET $testKey=$testVal").collect().map(_.getString(0))
assertResult(Set(testKey -> testVal)) {
collectResults(sql(s"SET $testKey=$testVal"))
}

assert(hiveconf.get(testKey, "") == testVal)
assertResult(Array(s"$testKey=$testVal")) {
sql("SET").collect().map(_.getString(0))
assertResult(Set(testKey -> testVal)) {
collectResults(sql("SET"))
}

sql(s"SET ${testKey + testKey}=${testVal + testVal}")
assert(hiveconf.get(testKey + testKey, "") == testVal + testVal)
assertResult(Array(s"$testKey=$testVal", s"${testKey + testKey}=${testVal + testVal}")) {
sql("SET").collect().map(_.getString(0))
assertResult(Set(testKey -> testVal, (testKey + testKey) -> (testVal + testVal))) {
collectResults(sql("SET"))
}

assertResult(Array(s"$testKey=$testVal")) {
sql(s"SET $testKey").collect().map(_.getString(0))
assertResult(Set(testKey -> testVal)) {
collectResults(sql(s"SET $testKey"))
}

assertResult(Array(s"$nonexistentKey=<undefined>")) {
sql(s"SET $nonexistentKey").collect().map(_.getString(0))
assertResult(Set(nonexistentKey -> "<undefined>")) {
collectResults(sql(s"SET $nonexistentKey"))
}

clear()
Expand Down