From 8b216c4593dc39f49cb9b1abdaae1a720a183866 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Thu, 28 May 2015 22:48:50 -0700 Subject: [PATCH] Guard against negative offsets and lengths in FileSegment --- .../org/apache/spark/storage/BlockObjectWriter.scala | 2 ++ .../scala/org/apache/spark/storage/FileSegment.scala | 2 ++ .../apache/spark/storage/BlockObjectWriterSuite.scala | 9 +++++++++ 3 files changed, 13 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala b/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala index da6abac7ed267..7eeabd1e0489c 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala @@ -171,6 +171,8 @@ private[spark] class DiskBlockObjectWriter( finalPosition = file.length() // In certain compression codecs, more bytes are written after close() is called writeMetrics.incShuffleBytesWritten(finalPosition - reportedPosition) + } else { + finalPosition = file.length() } commitAndCloseHasBeenCalled = true } diff --git a/core/src/main/scala/org/apache/spark/storage/FileSegment.scala b/core/src/main/scala/org/apache/spark/storage/FileSegment.scala index 95e2d688d9b17..021a9facfb0b2 100644 --- a/core/src/main/scala/org/apache/spark/storage/FileSegment.scala +++ b/core/src/main/scala/org/apache/spark/storage/FileSegment.scala @@ -24,6 +24,8 @@ import java.io.File * based off an offset and a length. */ private[spark] class FileSegment(val file: File, val offset: Long, val length: Long) { + require(offset >= 0, s"File segment offset cannot be negative (got $offset)") + require(length >= 0, s"File segment length cannot be negative (got $length)") override def toString: String = { "(name=%s, offset=%d, length=%d)".format(file.getName, offset, length) } diff --git a/core/src/test/scala/org/apache/spark/storage/BlockObjectWriterSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockObjectWriterSuite.scala index 6522034d0b0e9..378e45249aabd 100644 --- a/core/src/test/scala/org/apache/spark/storage/BlockObjectWriterSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/BlockObjectWriterSuite.scala @@ -160,4 +160,13 @@ class BlockObjectWriterSuite extends FunSuite with BeforeAndAfterEach { } writer.close() } + + test("commitAndClose() without ever opening or writing") { + val file = new File(tempDir, "somefile") + val writeMetrics = new ShuffleWriteMetrics() + val writer = new DiskBlockObjectWriter(new TestBlockId("0"), file, + new JavaSerializer(new SparkConf()).newInstance(), 1024, os => os, true, writeMetrics) + writer.commitAndClose() + assert(writer.fileSegment().length === 0) + } }