From 7c0e2ebbd630838bcf92eafdfe9a8045343c5347 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 25 Jul 2024 20:23:48 -0700 Subject: [PATCH] [SPARK-49007][CORE] Improve `MasterPage` to support custom title ### What changes were proposed in this pull request? This PR aims to improve `MasterPage` to support custom title. ### Why are the changes needed? When there exists multiple Spark clusters, custom title can be more helpful than the spark master address because it can contain semantics like the role of the clusters. In addition, the URL field in the same page already provides the spark master information even when we use a custom title. **BEFORE** ``` sbin/start-master.sh ``` ![Screenshot 2024-07-25 at 14 01 11](https://github.com/user-attachments/assets/7055d700-4bd6-4785-a535-2f8ce6dba47d) **AFTER** ``` SPARK_MASTER_OPTS='-Dspark.master.ui.title="Projext X Staging Cluster"' sbin/start-master.sh ``` ![Screenshot 2024-07-25 at 14 05 38](https://github.com/user-attachments/assets/f7e45fd6-fa2b-4547-ae39-1403b1e910d9) ### Does this PR introduce _any_ user-facing change? ### How was this patch tested? Pass the CIs with newly added test case. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #47491 from dongjoon-hyun/SPARK-49007. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- .../spark/deploy/master/ui/MasterPage.scala | 4 +++- .../org/apache/spark/internal/config/UI.scala | 7 +++++++ .../deploy/master/MasterWorkerUISuite.scala | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala index 1248b1c368e71..eb858e93aa7fb 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala @@ -26,11 +26,13 @@ import org.apache.spark.deploy.DeployMessages.{KillDriverResponse, MasterStateRe import org.apache.spark.deploy.JsonProtocol import org.apache.spark.deploy.StandaloneResourceUtils._ import org.apache.spark.deploy.master._ +import org.apache.spark.internal.config.UI.MASTER_UI_TITLE import org.apache.spark.ui.{UIUtils, WebUIPage} import org.apache.spark.util.Utils private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") { private val master = parent.masterEndpointRef + private val title = parent.master.conf.get(MASTER_UI_TITLE) private val jsonFieldPattern = "/json/([a-zA-Z]+).*".r def getMasterState: MasterStateResponse = { @@ -266,7 +268,7 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") { } ; - UIUtils.basicSparkPage(request, content, "Spark Master at " + state.uri) + UIUtils.basicSparkPage(request, content, title.getOrElse("Spark Master at " + state.uri)) } private def workerRow(showResourceColumn: Boolean): WorkerInfo => Seq[Node] = worker => { diff --git a/core/src/main/scala/org/apache/spark/internal/config/UI.scala b/core/src/main/scala/org/apache/spark/internal/config/UI.scala index c4c5796a2b278..e02cb7a968314 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/UI.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/UI.scala @@ -244,6 +244,13 @@ private[spark] object UI { .checkValues(Set("ALLOW", "LOCAL", "DENY")) .createWithDefault("LOCAL") + val MASTER_UI_TITLE = ConfigBuilder("spark.master.ui.title") + .version("4.0.0") + .doc("Specifies the title of the Master UI page. If unset, `Spark Master at ` " + + "is used by default.") + .stringConf + .createOptional + val UI_SQL_GROUP_SUB_EXECUTION_ENABLED = ConfigBuilder("spark.ui.groupSQLSubExecutionEnabled") .doc("Whether to group sub executions together in SQL UI when they belong to the same " + "root execution") diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterWorkerUISuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterWorkerUISuite.scala index 428539068a107..5f206f611fe6b 100644 --- a/core/src/test/scala/org/apache/spark/deploy/master/MasterWorkerUISuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterWorkerUISuite.scala @@ -140,4 +140,22 @@ class MasterWorkerUISuite extends MasterSuiteBase { System.getProperties().remove("spark.ui.proxyBase") } } + + test("SPARK-49007: Support custom master web ui title") { + implicit val formats = org.json4s.DefaultFormats + val title = "Spark Custom Title" + val conf = new SparkConf().set(MASTER_UI_TITLE, title) + val localCluster = LocalSparkCluster(2, 2, 512, conf) + localCluster.start() + val masterUrl = s"http://${Utils.localHostNameForURI()}:${localCluster.masterWebUIPort}" + try { + eventually(timeout(50.seconds), interval(100.milliseconds)) { + val html = Utils + .tryWithResource(Source.fromURL(s"$masterUrl/"))(_.getLines().mkString("\n")) + html should include (title) + } + } finally { + localCluster.stop() + } + } }