Skip to content

Commit

Permalink
[SPARK-3536][SQL] SELECT on empty parquet table throws exception
Browse files Browse the repository at this point in the history
It returns null metadata from parquet if querying on empty parquet file while calculating splits.So added null check and returns the empty splits.

Author : ravipesala ravindra.pesalahuawei.com

Author: ravipesala <ravindra.pesala@huawei.com>

Closes #2456 from ravipesala/SPARK-3536 and squashes the following commits:

1e81a50 [ravipesala] Fixed the issue when querying on empty parquet file.
  • Loading branch information
ravipesala authored and marmbrus committed Sep 23, 2014
1 parent 116016b commit 3b8eefa
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -427,11 +427,15 @@ private[parquet] class FilteringParquetRowInputFormat
s"maxSplitSize or minSplitSie should not be negative: maxSplitSize = $maxSplitSize;" +
s" minSplitSize = $minSplitSize")
}

val splits = mutable.ArrayBuffer.empty[ParquetInputSplit]
val getGlobalMetaData =
classOf[ParquetFileWriter].getDeclaredMethod("getGlobalMetaData", classOf[JList[Footer]])
getGlobalMetaData.setAccessible(true)
val globalMetaData = getGlobalMetaData.invoke(null, footers).asInstanceOf[GlobalMetaData]
// if parquet file is empty, return empty splits.
if (globalMetaData == null) {
return splits
}

val readContext = getReadSupport(configuration).init(
new InitContext(configuration,
Expand All @@ -442,7 +446,6 @@ private[parquet] class FilteringParquetRowInputFormat
classOf[ParquetInputFormat[_]].getDeclaredMethods.find(_.getName == "generateSplits").get
generateSplits.setAccessible(true)

val splits = mutable.ArrayBuffer.empty[ParquetInputSplit]
for (footer <- footers) {
val fs = footer.getFile.getFileSystem(configuration)
val file = footer.getFile
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -789,4 +789,13 @@ class ParquetQuerySuite extends QueryTest with FunSuiteLike with BeforeAndAfterA
assert(result3(0)(1) === "the answer")
Utils.deleteRecursively(tmpdir)
}

test("Querying on empty parquet throws exception (SPARK-3536)") {
val tmpdir = Utils.createTempDir()
Utils.deleteRecursively(tmpdir)
createParquetFile[TestRDDEntry](tmpdir.toString()).registerTempTable("tmpemptytable")
val result1 = sql("SELECT * FROM tmpemptytable").collect()
assert(result1.size === 0)
Utils.deleteRecursively(tmpdir)
}
}

0 comments on commit 3b8eefa

Please sign in to comment.