Skip to content

Commit

Permalink
[SPARK-26517][SQL][TEST] Avoid duplicate test in ParquetSchemaPruning…
Browse files Browse the repository at this point in the history
…Suite

## What changes were proposed in this pull request?

`testExactCaseQueryPruning` and `testMixedCaseQueryPruning` don't need to set up `PARQUET_VECTORIZED_READER_ENABLED` config. Because `withMixedCaseData` will run against both Spark vectorized reader and Parquet-mr reader.

## How was this patch tested?

Existing test.

Closes #23427 from viirya/fix-parquet-schema-pruning-test.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Sean Owen <sean.owen@databricks.com>
  • Loading branch information
viirya authored and srowen committed Jan 3, 2019
1 parent 88b074f commit 40711ee
Showing 1 changed file with 5 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -312,15 +312,8 @@ class ParquetSchemaPruningSuite
// schema's column and field names. N.B. this implies that `testThunk` should pass using either a
// case-sensitive or case-insensitive query parser
private def testExactCaseQueryPruning(testName: String)(testThunk: => Unit) {
test(s"Spark vectorized reader - case-sensitive parser - mixed-case schema - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
SQLConf.CASE_SENSITIVE.key -> "true") {
withMixedCaseData(testThunk)
}
}
test(s"Parquet-mr reader - case-sensitive parser - mixed-case schema - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
SQLConf.CASE_SENSITIVE.key -> "true") {
test(s"Case-sensitive parser - mixed-case schema - $testName") {
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
withMixedCaseData(testThunk)
}
}
Expand All @@ -330,20 +323,14 @@ class ParquetSchemaPruningSuite
// Tests schema pruning for a query whose column and field names may differ in case from the table
// schema's column and field names
private def testMixedCaseQueryPruning(testName: String)(testThunk: => Unit) {
test(s"Spark vectorized reader - case-insensitive parser - mixed-case schema - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
SQLConf.CASE_SENSITIVE.key -> "false") {
withMixedCaseData(testThunk)
}
}
test(s"Parquet-mr reader - case-insensitive parser - mixed-case schema - $testName") {
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
SQLConf.CASE_SENSITIVE.key -> "false") {
test(s"Case-insensitive parser - mixed-case schema - $testName") {
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
withMixedCaseData(testThunk)
}
}
}

// Tests given test function with Spark vectorized reader and Parquet-mr reader.
private def withMixedCaseData(testThunk: => Unit) {
withParquetTable(mixedCaseData, "mixedcase") {
testThunk
Expand Down

0 comments on commit 40711ee

Please sign in to comment.