Skip to content

Commit

Permalink
address comments
Browse files Browse the repository at this point in the history
  • Loading branch information
cloud-fan committed May 7, 2019
1 parent 9606410 commit 115c072
Showing 1 changed file with 6 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,8 @@ private[spark] class TaskSetManager(
private val addedJars = HashMap[String, Long](sched.sc.addedJars.toSeq: _*)
private val addedFiles = HashMap[String, Long](sched.sc.addedFiles.toSeq: _*)

// Quantile of tasks at which to start speculation
val speculationQuantile = conf.get(SPECULATION_QUANTILE)
val speculationMultiplier = conf.get(SPECULATION_MULTIPLIER)

val maxResultSize = conf.get(config.MAX_RESULT_SIZE)

val speculationEnabled = conf.get(SPECULATION_ENABLED)

// Serializer for closures and tasks.
val env = SparkEnv.get
val ser = env.closureSerializer.newInstance()
Expand All @@ -80,6 +74,12 @@ private[spark] class TaskSetManager(
val numTasks = tasks.length
val copiesRunning = new Array[Int](numTasks)

val speculationEnabled = conf.get(SPECULATION_ENABLED)
// Quantile of tasks at which to start speculation
val speculationQuantile = conf.get(SPECULATION_QUANTILE)
val speculationMultiplier = conf.get(SPECULATION_MULTIPLIER)
val minFinishedForSpeculation = math.max((speculationQuantile * numTasks).floor.toInt, 1)

// For each task, tracks whether a copy of the task has succeeded. A task will also be
// marked as "succeeded" if it failed with a fetch failure, in which case it should not
// be re-run because the missing map data needs to be regenerated first.
Expand Down Expand Up @@ -1032,7 +1032,6 @@ private[spark] class TaskSetManager(
return false
}
var foundTasks = false
val minFinishedForSpeculation = math.max((speculationQuantile * numTasks).floor.toInt, 1)
logDebug("Checking for speculative tasks: minFinished = " + minFinishedForSpeculation)

// It's possible that a task is marked as completed by the scheduler, then the size of
Expand Down

0 comments on commit 115c072

Please sign in to comment.