From ee33d52b49fbbcee796a88387f297e459047a76c Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Sun, 3 May 2015 16:36:49 -0700 Subject: [PATCH] Separate HTML generating code from listener --- .../apache/spark/ui/static/spark-dag-viz.js | 3 ++ .../scala/org/apache/spark/ui/UIUtils.scala | 39 +++++++++++++++++++ .../org/apache/spark/ui/jobs/JobPage.scala | 2 +- .../org/apache/spark/ui/jobs/StagePage.scala | 4 +- .../spark/ui/viz/VisualizationListener.scala | 36 +---------------- 5 files changed, 48 insertions(+), 36 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js index 0dec1b2476c1c..c10daab73f5ed 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js +++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js @@ -105,6 +105,9 @@ function toggleDagViz(forJob) { * div#dag-viz-graph > * svg > * g#cluster_stage_[stageId] + * + * Note that the input metadata is populated by o.a.s.ui.UIUtils.showDagViz. + * Any changes in the input format here must be reflected there. */ function renderDagViz(forJob) { diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 80e93977b6e15..3b2b944da725b 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -23,6 +23,7 @@ import java.util.{Locale, Date} import scala.xml.{Node, Text} import org.apache.spark.Logging +import org.apache.spark.ui.viz.VizGraph /** Utility functions for generating XML pages with spark content. */ private[spark] object UIUtils extends Logging { @@ -329,4 +330,42 @@ private[spark] object UIUtils extends Logging {
} + + /** Return a "DAG visualization" DOM element that expands into a visualization for a stage. */ + def showDagVizForStage(stageId: Int, graph: Option[VizGraph]): Seq[Node] = { + showDagViz(graph.toSeq, forJob = false) + } + + /** Return a "DAG visualization" DOM element that expands into a visualization for a job. */ + def showDagVizForJob(jobId: Int, graphs: Seq[VizGraph]): Seq[Node] = { + showDagViz(graphs, forJob = true) + } + + /** + * Return a "DAG visualization" DOM element that expands into a visualization on the UI. + * + * This populates metadata necessary for generating the visualization on the front-end in + * a format that is expected by spark-dag-viz.js. Any changes in the format here must be + * reflected there. + */ + private def showDagViz(graphs: Seq[VizGraph], forJob: Boolean): Seq[Node] = { +
+ + + DAG visualization + +
+
+ { + graphs.map { g => + + } + } +
+
+ } } diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala index f3a515901d7ce..3c18a5d875079 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala @@ -309,7 +309,7 @@ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") { content ++= makeTimeline(activeStages ++ completedStages ++ failedStages, executorListener.executorIdToData, appStartTime) - content ++= vizListener.showVizElementForJob(jobId) + content ++= UIUtils.showDagVizForJob(jobId, vizListener.getVizGraphsForJob(jobId)) if (shouldShowActiveStages) { content ++=

Active Stages ({activeStages.size})

++ diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index 95fc1c962e219..12c1a88512d43 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -171,6 +171,8 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { + val dagViz = UIUtils.showDagVizForStage(stageId, vizListener.getVizGraphForStage(stageId)) + val accumulableHeaders: Seq[String] = Seq("Accumulable", "Value") def accumulableRow(acc: AccumulableInfo): Elem = {acc.name}{acc.value} @@ -436,7 +438,7 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { val content = summary ++ showAdditionalMetrics ++ - vizListener.showVizElementForStage(stageId) ++ + dagViz ++

Summary Metrics for {numCompleted} Completed Tasks

++
{summaryTable.getOrElse("No tasks have reported metrics yet.")}
++

Aggregated Metrics by Executor

++ executorTable.toNodeSeq ++ diff --git a/core/src/main/scala/org/apache/spark/ui/viz/VisualizationListener.scala b/core/src/main/scala/org/apache/spark/ui/viz/VisualizationListener.scala index 23d1acd3afff1..e6ea15787b907 100644 --- a/core/src/main/scala/org/apache/spark/ui/viz/VisualizationListener.scala +++ b/core/src/main/scala/org/apache/spark/ui/viz/VisualizationListener.scala @@ -36,47 +36,15 @@ private[ui] class VisualizationListener(conf: SparkConf) extends SparkListener { private val retainedStages = conf.getInt("spark.ui.retainedStages", SparkUI.DEFAULT_RETAINED_STAGES) - /** Construct a "DAG visualization" DOM element that expands into a visualization for a stage. */ - def showVizElementForStage(stageId: Int): Seq[Node] = { - showVizElement(getVizGraphForStage(stageId).toSeq, forJob = false) - } - - /** Construct a "DAG visualization" DOM element that expands into a visualization for a job. */ - def showVizElementForJob(jobId: Int): Seq[Node] = { - showVizElement(getVizGraphsForJob(jobId), forJob = true) - } - - /** Construct a "DAG visualization" DOM element that expands into a visualization on the UI. */ - private def showVizElement(graphs: Seq[VizGraph], forJob: Boolean): Seq[Node] = { -
- - - DAG visualization - -
-
- { - graphs.map { g => - - } - } -
-
- } - /** Return the graph metadata for the given stage, or None if no such information exists. */ - private def getVizGraphsForJob(jobId: Int): Seq[VizGraph] = { + def getVizGraphsForJob(jobId: Int): Seq[VizGraph] = { jobIdToStageIds.get(jobId) .map { sids => sids.flatMap { sid => stageIdToGraph.get(sid) } } .getOrElse { Seq.empty } } /** Return the graph metadata for the given stage, or None if no such information exists. */ - private def getVizGraphForStage(stageId: Int): Option[VizGraph] = { + def getVizGraphForStage(stageId: Int): Option[VizGraph] = { stageIdToGraph.get(stageId) }