Skip to content

Commit

Permalink
Fix table sorting on all jobs page.
Browse files Browse the repository at this point in the history
It now sorts based on the time that the scheduler
learns about the job rather than the time that the
first stage begins executing.  This is necessary
because the earliest stage that a job depends on
may not necessarily be executed (e.g. if it lies
behind a shuffle boundary for a shuffle that’s
already been computed, or if some RDD has been
cached, etc.).
  • Loading branch information
JoshRosen committed Nov 19, 2014
1 parent 5eb39dc commit d69c775
Showing 1 changed file with 3 additions and 11 deletions.
14 changes: 3 additions & 11 deletions core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,6 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
private val startTime: Option[Long] = parent.sc.map(_.startTime)
private val listener = parent.listener

private def getSubmissionTime(job: JobUIData): Option[Long] = {
for (
firstStageId <- job.stageIds.headOption;
firstStageInfo <- listener.stageIdToInfo.get(firstStageId);
submitTime <- firstStageInfo.submissionTime
) yield submitTime
}

private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = {
val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined)

Expand Down Expand Up @@ -107,11 +99,11 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
val now = System.currentTimeMillis

val activeJobsTable =
jobsTable(activeJobs.sortBy(getSubmissionTime(_).getOrElse(-1L)).reverse)
jobsTable(activeJobs.sortBy(_.startTime.getOrElse(-1L)).reverse)
val completedJobsTable =
jobsTable(completedJobs.sortBy(getSubmissionTime(_).getOrElse(-1L)).reverse)
jobsTable(completedJobs.sortBy(_.endTime.getOrElse(-1L)).reverse)
val failedJobsTable =
jobsTable(failedJobs.sortBy(getSubmissionTime(_).getOrElse(-1L)).reverse)
jobsTable(failedJobs.sortBy(_.endTime.getOrElse(-1L)).reverse)

val summary: NodeSeq =
<div>
Expand Down

0 comments on commit d69c775

Please sign in to comment.