Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SPARK-25406][SQL] For ParquetSchemaPruningSuite.scala, move calls to withSQLConf inside calls to test #22394

Closed
wants to merge 3 commits into from
Closed

[SPARK-25406][SQL] For ParquetSchemaPruningSuite.scala, move calls to withSQLConf inside calls to test #22394

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8cca76b
For ParquetSchemaPruningSuite.scala, move calls to `withSQLConf` inside
mallman Sep 11, 2018
c759aea
We shouldn't expect queries selecting columns using different case to
mallman Sep 11, 2018
1c89637
Clarify the intent of the test methods for case-insensitive pruning
mallman Sep 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 38 additions & 25 deletions .../scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,20 +156,24 @@ class ParquetSchemaPruningSuite
}

private def testSchemaPruning(testName: String)(testThunk: => Unit) {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true") {
test(s"Spark vectorized reader - without partition data column - $testName") {
test(s"Spark vectorized reader - without partition data column - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true") {
mallman marked this conversation as resolved.
Show resolved Hide resolved
withContacts(testThunk)
}
test(s"Spark vectorized reader - with partition data column - $testName") {
}
test(s"Spark vectorized reader - with partition data column - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true") {
withContactsWithDataPartitionColumn(testThunk)
}
}

withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
test(s"Parquet-mr reader - without partition data column - $testName") {
test(s"Parquet-mr reader - without partition data column - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
withContacts(testThunk)
}
test(s"Parquet-mr reader - with partition data column - $testName") {
}
test(s"Parquet-mr reader - with partition data column - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
withContactsWithDataPartitionColumn(testThunk)
}
}
Expand Down Expand Up @@ -209,7 +213,7 @@ class ParquetSchemaPruningSuite
MixedCase(1, "r1c1", MixedCaseColumn("123", 2)) ::
Nil

testMixedCasePruning("select with exact column names") {
testExactCaseQueryPruning("select with exact column names") {
val query = sql("select CoL1, coL2.B from mixedcase")
checkScan(query, "struct<CoL1:string,coL2:struct<B:int>>")
checkAnswer(query.orderBy("id"),
Expand All @@ -218,7 +222,7 @@ class ParquetSchemaPruningSuite
Nil)
}

testMixedCasePruning("select with lowercase column names") {
testMixedCaseQueryPruning("select with lowercase column names") {
val query = sql("select col1, col2.b from mixedcase")
checkScan(query, "struct<CoL1:string,coL2:struct<B:int>>")
checkAnswer(query.orderBy("id"),
Expand All @@ -227,7 +231,7 @@ class ParquetSchemaPruningSuite
Nil)
}

testMixedCasePruning("select with different-case column names") {
testMixedCaseQueryPruning("select with different-case column names") {
val query = sql("select cOL1, cOl2.b from mixedcase")
checkScan(query, "struct<CoL1:string,coL2:struct<B:int>>")
checkAnswer(query.orderBy("id"),
Expand All @@ -236,7 +240,7 @@ class ParquetSchemaPruningSuite
Nil)
}

testMixedCasePruning("filter with different-case column names") {
testMixedCaseQueryPruning("filter with different-case column names") {
val query = sql("select id from mixedcase where Col2.b = 2")
// Pruning with filters is currently unsupported. As-is, the file reader will read the id column
// and the entire coL2 struct. Once pruning with filters has been implemented we can uncomment
Expand All @@ -245,28 +249,37 @@ class ParquetSchemaPruningSuite
checkAnswer(query.orderBy("id"), Row(1) :: Nil)
}

private def testMixedCasePruning(testName: String)(testThunk: => Unit) {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
SQLConf.CASE_SENSITIVE.key -> "true") {
test(s"Spark vectorized reader - case-sensitive parser - mixed-case schema - $testName") {
withMixedCaseData(testThunk)
// Tests schema pruning for a query whose column and field names are exactly the same as the table
// schema's column and field names. N.B. this implies that `testThunk` should pass using either a
// case-sensitive or case-insensitive query parser
private def testExactCaseQueryPruning(testName: String)(testThunk: => Unit) {
test(s"Spark vectorized reader - case-sensitive parser - mixed-case schema - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
SQLConf.CASE_SENSITIVE.key -> "true") {
withMixedCaseData(testThunk)
}
}
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
SQLConf.CASE_SENSITIVE.key -> "false") {
test(s"Parquet-mr reader - case-insensitive parser - mixed-case schema - $testName") {
test(s"Parquet-mr reader - case-sensitive parser - mixed-case schema - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
SQLConf.CASE_SENSITIVE.key -> "true") {
withMixedCaseData(testThunk)
}
}
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
SQLConf.CASE_SENSITIVE.key -> "false") {
test(s"Spark vectorized reader - case-insensitive parser - mixed-case schema - $testName") {
withMixedCaseData(testThunk)
testMixedCaseQueryPruning(testName)(testThunk)
}

// Tests schema pruning for a query whose column and field names may differ in case from the table
// schema's column and field names
private def testMixedCaseQueryPruning(testName: String)(testThunk: => Unit) {
test(s"Spark vectorized reader - case-insensitive parser - mixed-case schema - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
SQLConf.CASE_SENSITIVE.key -> "false") {
withMixedCaseData(testThunk)
}
}
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
SQLConf.CASE_SENSITIVE.key -> "true") {
test(s"Parquet-mr reader - case-sensitive parser - mixed-case schema - $testName") {
test(s"Parquet-mr reader - case-insensitive parser - mixed-case schema - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
SQLConf.CASE_SENSITIVE.key -> "false") {
withMixedCaseData(testThunk)
}
}
Expand Down