From 2286e6bdc417cf9caca4beabd37ff3d025019ac3 Mon Sep 17 00:00:00 2001 From: WangTao Date: Wed, 5 Nov 2014 23:19:52 +0800 Subject: [PATCH] ignore spark.driver.host in yarn-cluster mode --- .../main/scala/org/apache/spark/SparkContext.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 3cdaa6a9cc8a8..0c21b0316303a 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -175,10 +175,6 @@ class SparkContext(config: SparkConf) extends SparkStatusAPI with Logging { logInfo("Spark configuration:\n" + conf.toDebugString) } - // Set Spark driver host and port system properties - conf.setIfMissing("spark.driver.host", Utils.localHostName()) - conf.setIfMissing("spark.driver.port", "0") - val jars: Seq[String] = conf.getOption("spark.jars").map(_.split(",")).map(_.filter(_.size != 0)).toSeq.flatten @@ -206,6 +202,14 @@ class SparkContext(config: SparkConf) extends SparkStatusAPI with Logging { if (master == "yarn-client") System.setProperty("SPARK_YARN_MODE", "true") + // Set Spark driver host and port system properties. Ignore host setting in yarn-cluster mode. + if (master == "yarn-cluster") { + conf.set("spark.driver.host", Utils.localHostName()) + } else { + conf.setIfMissing("spark.driver.host", Utils.localHostName()) + } + conf.setIfMissing("spark.driver.port", "0") + // An asynchronous listener bus for Spark events private[spark] val listenerBus = new LiveListenerBus