ignore spark.driver.host in yarn-cluster mode

markhamstra · Nov 5, 2014 · 2286e6b · 2286e6b
1 parent 5f13759
commit 2286e6b
Showing 1 changed file with 8 additions and 4 deletions.
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -175,10 +175,6 @@ class SparkContext(config: SparkConf) extends SparkStatusAPI with Logging {
     logInfo("Spark configuration:\n" + conf.toDebugString)
   }
 
-  // Set Spark driver host and port system properties
-  conf.setIfMissing("spark.driver.host", Utils.localHostName())
-  conf.setIfMissing("spark.driver.port", "0")
-
   val jars: Seq[String] =
     conf.getOption("spark.jars").map(_.split(",")).map(_.filter(_.size != 0)).toSeq.flatten
 
@@ -206,6 +202,14 @@ class SparkContext(config: SparkConf) extends SparkStatusAPI with Logging {
 
   if (master == "yarn-client") System.setProperty("SPARK_YARN_MODE", "true")
 
+  // Set Spark driver host and port system properties. Ignore host setting in yarn-cluster mode.
+  if (master == "yarn-cluster") {
+    conf.set("spark.driver.host", Utils.localHostName())
+  } else {
+    conf.setIfMissing("spark.driver.host", Utils.localHostName())
+  }
+  conf.setIfMissing("spark.driver.port", "0")
+
   // An asynchronous listener bus for Spark events
   private[spark] val listenerBus = new LiveListenerBus