Skip to content

Commit

Permalink
Fixed the issue when querying on empty parquet file.
Browse files Browse the repository at this point in the history
  • Loading branch information
ravipesala committed Sep 18, 2014
1 parent 7d1a372 commit 1e81a50
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -427,11 +427,15 @@ private[parquet] class FilteringParquetRowInputFormat
s"maxSplitSize or minSplitSie should not be negative: maxSplitSize = $maxSplitSize;" +
s" minSplitSize = $minSplitSize")
}

val splits = mutable.ArrayBuffer.empty[ParquetInputSplit]
val getGlobalMetaData =
classOf[ParquetFileWriter].getDeclaredMethod("getGlobalMetaData", classOf[JList[Footer]])
getGlobalMetaData.setAccessible(true)
val globalMetaData = getGlobalMetaData.invoke(null, footers).asInstanceOf[GlobalMetaData]
// if parquet file is empty, return empty splits.
if (globalMetaData == null) {
return splits
}

val readContext = getReadSupport(configuration).init(
new InitContext(configuration,
Expand All @@ -442,7 +446,6 @@ private[parquet] class FilteringParquetRowInputFormat
classOf[ParquetInputFormat[_]].getDeclaredMethods.find(_.getName == "generateSplits").get
generateSplits.setAccessible(true)

val splits = mutable.ArrayBuffer.empty[ParquetInputSplit]
for (footer <- footers) {
val fs = footer.getFile.getFileSystem(configuration)
val file = footer.getFile
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -789,4 +789,13 @@ class ParquetQuerySuite extends QueryTest with FunSuiteLike with BeforeAndAfterA
assert(result3(0)(1) === "the answer")
Utils.deleteRecursively(tmpdir)
}

test("Querying on empty parquet throws exception (SPARK-3536)") {
val tmpdir = Utils.createTempDir()
Utils.deleteRecursively(tmpdir)
createParquetFile[TestRDDEntry](tmpdir.toString()).registerTempTable("tmpemptytable")
val result1 = sql("SELECT * FROM tmpemptytable").collect()
assert(result1.size === 0)
Utils.deleteRecursively(tmpdir)
}
}

0 comments on commit 1e81a50

Please sign in to comment.