Add in admin acls

core/src/main/scala/org/apache/spark/SecurityManager.scala

@@ -45,10 +45,14 @@ import org.apache.spark.deploy.SparkHadoopUtil
  * control the behavior of the acls. Note that the person who started the application
  * always has view access to the UI.
  *
- * Spark has a set of modify acls that controls which users have permission to modify a single
- * application. This would include things like killing the application. By default the person who
- * started the application has modify access. For modify access through the UI, you must have a
- * filter that does authentication in place for the modify acls to work properly.
+ * Spark has a set of modify acls (`spark.modify.acls`) that controls which users have permission
+ * to  modify a single application. This would include things like killing the application. By
+ * default the person who started the application has modify access. For modify access through
+ * the UI, you must have a filter that does authentication in place for the modify acls to work
+ * properly.
+ *
+ * Spark also has a set of admin acls (`spark.admin.acls`) which is a set of users/administrators
+ * who always have permission to view or modify the Spark application.
  *
  * Spark does not currently support encryption after authentication.
  *
@@ -146,15 +150,20 @@ private[spark] class SecurityManager(sparkConf: SparkConf) extends Logging {
   private var aclsOn = sparkConf.getOption("spark.acls.enable").getOrElse(
     sparkConf.get("spark.ui.acls.enable", "false")).toBoolean
 
+  // admin acls should be set before view or modify acls
+  private var adminAcls: Set[String] =
+    stringToSet(sparkConf.get("spark.admin.acls", ""))
+
   private var viewAcls: Set[String] = _
 
   // list of users who have permission to modify the application. This should
   // apply to both UI and CLI for things like killing the application.
   private var modifyAcls: Set[String] = _
 
   // always add the current user and SPARK_USER to the viewAcls
-  private val defaultAclUsers = Seq[String](System.getProperty("user.name", ""),
+  private val defaultAclUsers = Set[String](System.getProperty("user.name", ""),
     Option(System.getenv("SPARK_USER")).getOrElse(""))
+
   setViewAcls(defaultAclUsers, sparkConf.get("spark.ui.view.acls", ""))
   setModifyAcls(defaultAclUsers, sparkConf.get("spark.modify.acls", ""))
 
@@ -183,24 +192,40 @@ private[spark] class SecurityManager(sparkConf: SparkConf) extends Logging {
     )
   }
 
-  private[spark] def setViewAcls(defaultUsers: Seq[String], allowedUsers: String) {
-    viewAcls = (defaultUsers ++ allowedUsers.split(',')).map(_.trim()).filter(!_.isEmpty).toSet
+  /**
+   * Split a comma separated String, filter out any empty items, and return a Set of strings
+   */
+  private def stringToSet(list: String): Set[String] = {
+    (list.split(',')).map(_.trim()).filter(!_.isEmpty).toSet
+  }
+
+  private[spark] def setViewAcls(defaultUsers: Set[String], allowedUsers: String) {
+    viewAcls = (adminAcls ++ defaultUsers ++ stringToSet(allowedUsers))
     logInfo("Changing view acls to: " + viewAcls.mkString(","))
   }
 
   private[spark] def setViewAcls(defaultUser: String, allowedUsers: String) {
-    setViewAcls(Seq[String](defaultUser), allowedUsers)
+    setViewAcls(Set[String](defaultUser), allowedUsers)
   }
 
   private[spark] def getViewAcls: String = viewAcls.mkString(",")
 
-  private[spark] def setModifyAcls(defaultUsers: Seq[String], allowedUsers: String) {
-    modifyAcls = (defaultUsers ++ allowedUsers.split(',')).map(_.trim()).filter(!_.isEmpty).toSet
+  private[spark] def setModifyAcls(defaultUsers: Set[String], allowedUsers: String) {
+    modifyAcls = (adminAcls ++ defaultUsers ++ stringToSet(allowedUsers))
     logInfo("Changing modify acls to: " + modifyAcls.mkString(","))
   }
 
   private[spark] def getModifyAcls: String = modifyAcls.mkString(",")
 
+  /**
+   * Admin acls should be set before the view or modify acls.  If you modify the admin
+   * acls you should also set the view and modify acls again to pick up the changes.
+   */
+  private[spark] def setAdminAcls(adminUsers: String) {
+    adminAcls = stringToSet(adminUsers)
+    logInfo("Changing admin acls to: " + adminAcls.mkString(","))
+  }
+
   private[spark] def setAcls(aclSetting: Boolean) {
     aclsOn = aclSetting
     logInfo("Changing acls enabled to: " + aclsOn)
@@ -252,7 +277,8 @@ private[spark] class SecurityManager(sparkConf: SparkConf) extends Logging {
   /**
    * Checks the given user against the view acl list to see if they have
    * authorization to view the UI. If the UI acls are disabled
-   * via spark.acls.enable, all users have view access.
+   * via spark.acls.enable, all users have view access. If the user is null
+   * it is assumed authentication isn't turned on and all users have access.
    *
    * @param user to see if is authorized
    * @return true is the user has permission, otherwise false
@@ -266,7 +292,8 @@ private[spark] class SecurityManager(sparkConf: SparkConf) extends Logging {
   /**
    * Checks the given user against the modify acl list to see if they have
    * authorization to modify the application. If the UI acls are disabled
-   * via spark.acls.enable, all users have modify access.
+   * via spark.acls.enable, all users have modify access. If the user is null
+   * it is assumed authentication isn't turned on and all users have access.
    *
    * @param user to see if is authorized
    * @return true is the user has permission, otherwise false

core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala

@@ -186,7 +186,9 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis
 
     if (ui != null) {
       val uiAclsEnabled = conf.getBoolean("spark.history.ui.acls.enable", false)
-      ui.getSecurityManager.setUIAcls(uiAclsEnabled)
+      ui.getSecurityManager.setAcls(uiAclsEnabled)
+      // make sure to set admin acls before view acls so properly picked up
+      ui.getSecurityManager.setAdminAcls(appListener.adminAcls)
       ui.getSecurityManager.setViewAcls(appListener.sparkUser, appListener.viewAcls)
     }
     (appInfo, ui)

core/src/main/scala/org/apache/spark/scheduler/ApplicationEventListener.scala

@@ -29,6 +29,7 @@ private[spark] class ApplicationEventListener extends SparkListener {
   var startTime = -1L
   var endTime = -1L
   var viewAcls = ""
+  var adminAcls = ""
   var enableViewAcls = false
 
   def applicationStarted = startTime != -1
@@ -55,7 +56,9 @@ private[spark] class ApplicationEventListener extends SparkListener {
       val environmentDetails = environmentUpdate.environmentDetails
       val allProperties = environmentDetails("Spark Properties").toMap
       viewAcls = allProperties.getOrElse("spark.ui.view.acls", "")
-      enableViewAcls = allProperties.getOrElse("spark.ui.acls.enable", "false").toBoolean
+      adminAcls = allProperties.getOrElse("spark.admin.acls", "")
+      enableViewAcls = allProperties.getOrElse("spark.acls.enable",
+        allProperties.getOrElse("spark.ui.acls.enable", "false")).toBoolean
     }
   }
 }

core/src/test/scala/org/apache/spark/SecurityManagerSuite.scala

@@ -51,7 +51,7 @@ class SecurityManagerSuite extends FunSuite {
 
     securityManager.setAcls(true)
     assert(securityManager.aclsEnabled() === true)
-    securityManager.setViewAcls(ArrayBuffer[String]("user5"), "user6,user7")
+    securityManager.setViewAcls(Set[String]("user5"), "user6,user7")
     assert(securityManager.checkUIViewPermissions("user1") === false)
     assert(securityManager.checkUIViewPermissions("user5") === true)
     assert(securityManager.checkUIViewPermissions("user6") === true)
@@ -63,6 +63,7 @@ class SecurityManagerSuite extends FunSuite {
   test("set security modify acls") {
     val conf = new SparkConf
     conf.set("spark.modify.acls", "user1,user2")
+
     val securityManager = new SecurityManager(conf);
     securityManager.setAcls(true)
     assert(securityManager.aclsEnabled() === true)
@@ -74,7 +75,7 @@ class SecurityManagerSuite extends FunSuite {
 
     securityManager.setAcls(true)
     assert(securityManager.aclsEnabled() === true)
-    securityManager.setModifyAcls(ArrayBuffer[String]("user5"), "user6,user7")
+    securityManager.setModifyAcls(Set("user5"), "user6,user7")
     assert(securityManager.checkModifyPermissions("user1") === false)
     assert(securityManager.checkModifyPermissions("user5") === true)
     assert(securityManager.checkModifyPermissions("user6") === true)
@@ -83,6 +84,47 @@ class SecurityManagerSuite extends FunSuite {
     assert(securityManager.checkModifyPermissions(null) === true)
   }
 
+  test("set security admin acls") {
+    val conf = new SparkConf
+    conf.set("spark.admin.acls", "user1,user2")
+    conf.set("spark.ui.view.acls", "user3")
+    conf.set("spark.modify.acls", "user4")
+
+    val securityManager = new SecurityManager(conf);
+    securityManager.setAcls(true)
+    assert(securityManager.aclsEnabled() === true)
+
+    assert(securityManager.checkModifyPermissions("user1") === true)
+    assert(securityManager.checkModifyPermissions("user2") === true)
+    assert(securityManager.checkModifyPermissions("user4") === true)
+    assert(securityManager.checkModifyPermissions("user3") === false)
+    assert(securityManager.checkModifyPermissions("user5") === false)
+    assert(securityManager.checkModifyPermissions(null) === true)
+    assert(securityManager.checkUIViewPermissions("user1") === true)
+    assert(securityManager.checkUIViewPermissions("user2") === true)
+    assert(securityManager.checkUIViewPermissions("user3") === true)
+    assert(securityManager.checkUIViewPermissions("user4") === false)
+    assert(securityManager.checkUIViewPermissions("user5") === false)
+    assert(securityManager.checkUIViewPermissions(null) === true)
+
+    securityManager.setAdminAcls("user6")
+    securityManager.setViewAcls(Set[String]("user8"), "user9")
+    securityManager.setModifyAcls(Set("user11"), "user9")
+    assert(securityManager.checkModifyPermissions("user6") === true)
+    assert(securityManager.checkModifyPermissions("user11") === true)
+    assert(securityManager.checkModifyPermissions("user9") === true)
+    assert(securityManager.checkModifyPermissions("user1") === false)
+    assert(securityManager.checkModifyPermissions("user4") === false)
+    assert(securityManager.checkModifyPermissions(null) === true)
+    assert(securityManager.checkUIViewPermissions("user6") === true)
+    assert(securityManager.checkUIViewPermissions("user8") === true)
+    assert(securityManager.checkUIViewPermissions("user9") === true)
+    assert(securityManager.checkUIViewPermissions("user1") === false)
+    assert(securityManager.checkUIViewPermissions("user3") === false)
+    assert(securityManager.checkUIViewPermissions(null) === true)
+
+  }
+
 
 }
 

docs/configuration.md

@@ -768,6 +768,15 @@ Apart from these, the following properties are also available, and may be useful
     user that started the Spark job has access to modify it (kill it for example).
   </td>
 </tr>
+<tr>
+  <td><code>spark.admin.acls</code></td>
+  <td>Empty</td>
+  <td>
+    Comma separated list of users/administrators that have view and modify access to all Spark jobs.
+    This can be used if you run on a shared cluster and have a set of administrators or devs who
+    help debug when things work.
+  </td>
+</tr>
 </table>
 
 #### Spark Streaming

docs/security.md

@@ -10,7 +10,9 @@ Spark currently supports authentication via a shared secret. Authentication can
 
 The Spark UI can also be secured by using [javax servlet filters](http://docs.oracle.com/javaee/6/api/javax/servlet/Filter.html) via the `spark.ui.filters` setting. A user may want to secure the UI if it has data that other users should not be allowed to see. The javax servlet filter specified by the user can authenticate the user and then once the user is logged in, Spark can compare that user versus the view ACLs to make sure they are authorized to view the UI. The configs `spark.acls.enable` and `spark.ui.view.acls` control the behavior of the ACLs. Note that the user who started the application always has view access to the UI.  On YARN, the Spark UI uses the standard YARN web application proxy mechanism and will authenticate via any installed Hadoop filters.
 
-Spark also supports modify ACLs to control who has access to modify a running Spark application.  This includes things like killing the application or a task. This is controlled by the configs `spark.acls.enable` and `spark.modify.acls`.
+Spark also supports modify ACLs to control who has access to modify a running Spark application.  This includes things like killing the application or a task. This is controlled by the configs `spark.acls.enable` and `spark.modify.acls`. Note that if you are authenticating the web UI, in order to use the kill button on the web UI it might be necessary to add the users in the modify acls to the view acls also. On YARN, the modify acls are passed in and control who has modify access via YARN interfaces.
+
+Spark allows for a set of administrators to be specified in the acls who always have view and modify permissions to all the applications. is controlled by the config `spark.admin.acls`. This is useful on a shared cluster where you might have administrators or support staff who help users debug applications.
 
 If your applications are using event logging, the directory where the event logs go (`spark.eventLog.dir`) should be manually created and have the proper permissions set on it. If you want those log files secured, the permissions should be set to `drwxrwxrwxt` for that directory. The owner of the directory should be the super user who is running the history server and the group permissions should be restricted to super user group. This will allow all users to write to the directory but will prevent unprivileged users from removing or renaming a file unless they own the file or directory. The event log files will be created by Spark with permissions such that only the user and group have read and write access.
 

yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala

@@ -37,10 +37,7 @@ import org.apache.hadoop.yarn.api.protocolrecords._
 import org.apache.hadoop.yarn.api.records._
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.hadoop.yarn.util.Records
-import org.apache.spark._
-import scala.util.Failure
-import scala.Some
-import scala.util.Success
+import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkContext, SparkException}
 
 /**
  * The entry point (starting in Client#main() and Client#run()) for launching Spark on YARN. The
@@ -420,6 +417,7 @@ trait ClientBase extends Logging {
 
     setupSecurityToken(amContainer)
 
+    // send the acl settings into YARN to control who has access via YARN interfaces
     val securityManager = new SecurityManager(sparkConf)
     val acls = Map[ApplicationAccessType, String] (
       ApplicationAccessType.VIEW_APP -> securityManager.getViewAcls,