Skip to content

Commit

Permalink
[SPARK-32003] Clean up test.
Browse files Browse the repository at this point in the history
  • Loading branch information
wypoon committed Jun 18, 2020
1 parent 4c0b98c commit 4b18369
Showing 1 changed file with 6 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -560,21 +560,20 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
runEvent(ExecutorLost("hostA-exec", ExecutorExited(-100, false, "Container marked as failed")))

// The MapOutputTracker has all the shuffle files
val initialMapStatuses = mapOutputTracker.shuffleStatuses(shuffleId).mapStatuses
assert(initialMapStatuses.count(_ != null) === 3)
assert(initialMapStatuses.count(s => s != null && s.location.executorId == "hostA-exec") === 2)
assert(initialMapStatuses.count(s => s != null && s.location.executorId == "hostB-exec") === 1)
val mapStatuses = mapOutputTracker.shuffleStatuses(shuffleId).mapStatuses
assert(mapStatuses.count(_ != null) === 3)
assert(mapStatuses.count(s => s != null && s.location.executorId == "hostA-exec") === 2)
assert(mapStatuses.count(s => s != null && s.location.executorId == "hostB-exec") === 1)

// Now a fetch failure from the lost executor occurs
complete(taskSets(1), Seq(
(FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 0, 0, "ignored"), null)
))

// Shuffle files for hostA-exec should be lost
val mapStatuses = mapOutputTracker.shuffleStatuses(shuffleId).mapStatuses
assert(mapStatuses.count(_ != null) === 1)
assert(initialMapStatuses.count(s => s != null && s.location.executorId == "hostA-exec") === 0)
assert(initialMapStatuses.count(s => s != null && s.location.executorId == "hostB-exec") === 1)
assert(mapStatuses.count(s => s != null && s.location.executorId == "hostA-exec") === 0)
assert(mapStatuses.count(s => s != null && s.location.executorId == "hostB-exec") === 1)
}

test("zero split job") {
Expand Down

0 comments on commit 4b18369

Please sign in to comment.