From eaad42d434792639424292fdc17f07eba5d3646e Mon Sep 17 00:00:00 2001 From: "James Z.M. Gao" Date: Tue, 11 Mar 2014 11:07:28 +0800 Subject: [PATCH] remove staging dir when app quiting for yarn-cluster mode In yarn-cluster, the driver is actually running as 'yarn' user. When posting jobs from other users, we need give stagingDir a full path, so the driver could remove the correct directory if needed. --- .../main/scala/org/apache/spark/deploy/yarn/ClientBase.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index 57e5761cba896..5b0a659f36925 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -273,7 +273,7 @@ trait ClientBase extends Logging { ClientBase.populateClasspath(yarnConf, sparkConf, log4jConfLocalRes != null, env) env("SPARK_YARN_MODE") = "true" - env("SPARK_YARN_STAGING_DIR") = stagingDir + env("SPARK_YARN_STAGING_DIR") = FileSystem.get(conf).getHomeDirectory() + Path.SEPARATOR + stagingDir env("SPARK_USER") = UserGroupInformation.getCurrentUser().getShortUserName() // Set the environment variables to be passed on to the executors.