Skip to content

Commit

Permalink
[SPARK-49007][CORE] Improve MasterPage to support custom title
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?

This PR aims to improve `MasterPage` to support custom title.

### Why are the changes needed?

When there exists multiple Spark clusters, custom title can be more helpful than the spark master address because it can contain semantics like the role of the clusters. In addition, the URL field in the same page already provides the spark master information even when we use a custom title.

**BEFORE**
```
sbin/start-master.sh
```
![Screenshot 2024-07-25 at 14 01 11](https://github.com/user-attachments/assets/7055d700-4bd6-4785-a535-2f8ce6dba47d)

**AFTER**
```
SPARK_MASTER_OPTS='-Dspark.master.ui.title="Projext X Staging Cluster"' sbin/start-master.sh
```
![Screenshot 2024-07-25 at 14 05 38](https://github.com/user-attachments/assets/f7e45fd6-fa2b-4547-ae39-1403b1e910d9)

### Does this PR introduce _any_ user-facing change?

### How was this patch tested?

Pass the CIs with newly added test case.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes apache#47491 from dongjoon-hyun/SPARK-49007.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
  • Loading branch information
dongjoon-hyun committed Jul 26, 2024
1 parent e73ede7 commit 2363aec
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,13 @@ import org.apache.spark.deploy.DeployMessages.{KillDriverResponse, MasterStateRe
import org.apache.spark.deploy.JsonProtocol
import org.apache.spark.deploy.StandaloneResourceUtils._
import org.apache.spark.deploy.master._
import org.apache.spark.internal.config.UI.MASTER_UI_TITLE
import org.apache.spark.ui.{UIUtils, WebUIPage}
import org.apache.spark.util.Utils

private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
private val master = parent.masterEndpointRef
private val title = parent.master.conf.get(MASTER_UI_TITLE)
private val jsonFieldPattern = "/json/([a-zA-Z]+).*".r

def getMasterState: MasterStateResponse = {
Expand Down Expand Up @@ -266,7 +268,7 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
}
</div>;

UIUtils.basicSparkPage(request, content, "Spark Master at " + state.uri)
UIUtils.basicSparkPage(request, content, title.getOrElse("Spark Master at " + state.uri))
}

private def workerRow(showResourceColumn: Boolean): WorkerInfo => Seq[Node] = worker => {
Expand Down
7 changes: 7 additions & 0 deletions core/src/main/scala/org/apache/spark/internal/config/UI.scala
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,13 @@ private[spark] object UI {
.checkValues(Set("ALLOW", "LOCAL", "DENY"))
.createWithDefault("LOCAL")

val MASTER_UI_TITLE = ConfigBuilder("spark.master.ui.title")
.version("4.0.0")
.doc("Specifies the title of the Master UI page. If unset, `Spark Master at <MasterURL>` " +
"is used by default.")
.stringConf
.createOptional

val UI_SQL_GROUP_SUB_EXECUTION_ENABLED = ConfigBuilder("spark.ui.groupSQLSubExecutionEnabled")
.doc("Whether to group sub executions together in SQL UI when they belong to the same " +
"root execution")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,4 +140,22 @@ class MasterWorkerUISuite extends MasterSuiteBase {
System.getProperties().remove("spark.ui.proxyBase")
}
}

test("SPARK-49007: Support custom master web ui title") {
implicit val formats = org.json4s.DefaultFormats
val title = "Spark Custom Title"
val conf = new SparkConf().set(MASTER_UI_TITLE, title)
val localCluster = LocalSparkCluster(2, 2, 512, conf)
localCluster.start()
val masterUrl = s"http://${Utils.localHostNameForURI()}:${localCluster.masterWebUIPort}"
try {
eventually(timeout(50.seconds), interval(100.milliseconds)) {
val html = Utils
.tryWithResource(Source.fromURL(s"$masterUrl/"))(_.getLines().mkString("\n"))
html should include (title)
}
} finally {
localCluster.stop()
}
}
}

0 comments on commit 2363aec

Please sign in to comment.