From 782c7dec435f85219b911e4b12caa83b5348a587 Mon Sep 17 00:00:00 2001 From: Tianshuo Deng Date: Tue, 18 Nov 2014 10:45:58 -0800 Subject: [PATCH] make elementsRead private, fix comment --- .../apache/spark/util/collection/Spillable.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala index d3b8964c88915..ef37b8f63ceab 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala @@ -36,11 +36,11 @@ private[spark] trait Spillable[C] { protected def spill(collection: C): Unit // Number of elements read from input since last spill - protected var elementsRead: Long = 0 + protected def elementsRead = _elementsRead - // subclass calls this method to notify reading an element - // it's used to check spilling frequency - protected def addElementsRead = elementsRead += 1 + // Called by subclasses every time a record is read + // It's used for checking spilling frequency + protected def addElementsRead() = { _elementsRead += 1 } // Memory manager that can be used to acquire/release memory private[this] val shuffleMemoryManager = SparkEnv.get.shuffleMemoryManager @@ -48,6 +48,9 @@ private[spark] trait Spillable[C] { // What threshold of elementsRead we start estimating collection size at private[this] val trackMemoryThreshold = 1000 + // Number of elements read from input since last spill + private[this] var _elementsRead = 0L + // How much of the shared memory pool this collection has claimed private[this] var myMemoryThreshold = 0L @@ -80,7 +83,7 @@ private[spark] trait Spillable[C] { spill(collection) - elementsRead = 0 + _elementsRead = 0 // Keep track of spills, and release memory _memoryBytesSpilled += currentMemory releaseMemoryForThisThread()