diff --git a/cli/dcos_spark/version.py b/cli/dcos_spark/version.py index 2910900f28c55..32cba3268f8ea 100644 --- a/cli/dcos_spark/version.py +++ b/cli/dcos_spark/version.py @@ -1 +1 @@ -version = 'SNAPSHOT' +version = '0.5.19' diff --git a/conf/spark-env.sh b/conf/spark-env.sh index 1e2a4c9a4b34f..94aa68e4ad4c8 100755 --- a/conf/spark-env.sh +++ b/conf/spark-env.sh @@ -7,17 +7,24 @@ # moves those config files into the standard directory. In DCOS, the # CLI reads the "SPARK_HDFS_CONFIG_URL" marathon label in order to set # spark.mesos.uris + mkdir -p "${HADOOP_CONF_DIR}" [ -f "${MESOS_SANDBOX}/hdfs-site.xml" ] && cp "${MESOS_SANDBOX}/hdfs-site.xml" "${HADOOP_CONF_DIR}" [ -f "${MESOS_SANDBOX}/core-site.xml" ] && cp "${MESOS_SANDBOX}/core-site.xml" "${HADOOP_CONF_DIR}" -MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so +cd $MESOS_SANDBOX + +MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so # Support environments without DNS if [ -n "$LIBPROCESS_IP" ]; then SPARK_LOCAL_IP=${LIBPROCESS_IP} fi +# I first set this to MESOS_SANDBOX, as a Workaround for MESOS-5866 +# But this fails now due to MESOS-6391, so I'm setting it to /tmp +MESOS_DIRECTORY=/tmp + # Options read when launching programs locally with # ./bin/run-example or ./bin/spark-submit # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files diff --git a/docker/Dockerfile b/docker/Dockerfile index b7793c0294d74..ba94999757b50 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,7 +18,7 @@ # docker build -t spark:git-`git rev-parse --short HEAD` . # Basing from Mesos image so the Mesos native library is present. -FROM mesosphere/mesos-modules-private:git-e348e3f +FROM mesosphere/mesos-modules-private:dcos-ee-mesos-modules-1.8.5-rc2 MAINTAINER Michael Gummelt # Set environment variables. @@ -27,24 +27,25 @@ ENV DEBCONF_NONINTERACTIVE_SEEN "true" # Upgrade package index and install basic commands. RUN apt-get update && \ - apt-get install -y software-properties-common runit nginx + apt-get install -y \ + software-properties-common \ + runit \ + nginx + RUN add-apt-repository ppa:openjdk-r/ppa RUN apt-get update && \ apt-get install -y openjdk-8-jdk curl ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 -ENV MESOS_NATIVE_JAVA_LIBRARY /usr/local/lib/libmesos.so +ENV MESOS_NATIVE_JAVA_LIBRARY /usr/lib/libmesos.so ENV HADOOP_CONF_DIR /etc/hadoop RUN mkdir /etc/hadoop -ADD dist /opt/spark/dist ADD runit/service /var/lib/runit/service ADD runit/init.sh /sbin/init.sh ADD nginx /etc/nginx -#RUN ln -sf /usr/lib/libmesos.so /usr/lib/libmesos-0.23.1.so - # The following symlinks are hacks to make spark-class work under the # restricted PATH (/usr/bin) set by the DCOS # --executor-environment-variables option @@ -55,4 +56,13 @@ RUN ln -s /bin/grep /usr/bin/grep RUN ln -s /var/lib/runit/service/spark /etc/service/spark RUN ln -s /var/lib/runit/service/nginx /etc/service/nginx +RUN chmod -R ugo+rw /etc/nginx +RUN chmod -R ugo+rw /etc/service +RUN chmod -R ugo+rw /var/lib/ +RUN chmod -R ugo+rw /var/run/ +RUN chmod -R ugo+rw /var/log/ + +ADD dist /opt/spark/dist +RUN chmod -R ugo+rw /opt/spark/dist + WORKDIR /opt/spark/dist diff --git a/docker/runit/service/spark/run b/docker/runit/service/spark/run index 8338fc9bf0177..2becf9f979c2a 100755 --- a/docker/runit/service/spark/run +++ b/docker/runit/service/spark/run @@ -4,64 +4,83 @@ set -x exec 2>&1 -export APPLICATION_WEB_PROXY_BASE="${DISPATCHER_UI_WEB_PROXY_BASE}" - -cd /opt/spark/dist - -export SPARK_DAEMON_JAVA_OPTS="" -if [ "${DCOS_SERVICE_NAME}" != "spark" ]; then - export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.zookeeper.dir=/spark_mesos_dispatcher_${DCOS_SERVICE_NAME}" -fi - -if [ "$SPARK_DISPATCHER_MESOS_ROLE" != "" ]; then - export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.role=$SPARK_DISPATCHER_MESOS_ROLE" -fi - -if [ "$SPARK_DISPATCHER_MESOS_PRINCIPAL" != "" ]; then - export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.principal=$SPARK_DISPATCHER_MESOS_PRINCIPAL" -fi - -if [ "$SPARK_DISPATCHER_MESOS_SECRET" != "" ]; then - export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.secret=$SPARK_DISPATCHER_MESOS_SECRET" -fi - - - -HISTORY_SERVER_CONF="" -if [ "${ENABLE_HISTORY_SERVER:=false}" = "true" ]; then - HISTORY_SERVER_CONF="spark.mesos.historyServer.url=${HISTORY_SERVER_WEB_PROXY_BASE}" -fi - -sed "s,,${HISTORY_SERVER_CONF}," \ - conf/mesos-cluster-dispatcher.properties.template >conf/mesos-cluster-dispatcher.properties +function export_daemon_opts() { + export SPARK_DAEMON_JAVA_OPTS="" + if [ "${DCOS_SERVICE_NAME}" != "spark" ]; then + export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.zookeeper.dir=/spark_mesos_dispatcher_${DCOS_SERVICE_NAME}" + fi + + if [ "$SPARK_DISPATCHER_MESOS_ROLE" != "" ]; then + export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.role=$SPARK_DISPATCHER_MESOS_ROLE" + fi + + if [ "$SPARK_DISPATCHER_MESOS_PRINCIPAL" != "" ]; then + export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.principal=$SPARK_DISPATCHER_MESOS_PRINCIPAL" + fi + + if [ "$SPARK_DISPATCHER_MESOS_SECRET" != "" ]; then + export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.secret=$SPARK_DISPATCHER_MESOS_SECRET" + fi +} -sed "s,,${SPARK_LOG_LEVEL}," \ - conf/log4j.properties.template >conf/log4j.properties +function set_log_level() { + sed "s,,${SPARK_LOG_LEVEL}," \ + /opt/spark/dist/conf/log4j.properties.template >/opt/spark/dist/conf/log4j.properties +} function add_if_non_empty() { if [ -n "$2" ]; then - echo "$1=$2" >> conf/mesos-cluster-dispatcher.properties + echo "$1=$2" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties fi } -if [ "${SPARK_SSL_KEYSTOREBASE64}" != "" ]; then - echo "${SPARK_SSL_KEYSTOREBASE64}" | base64 -d > /tmp/dispatcher-keystore.jks - add_if_non_empty spark.ssl.keyStore /tmp/dispatcher-keystore.jks -fi +function configure_properties() { + HISTORY_SERVER_CONF="" + if [ "${ENABLE_HISTORY_SERVER:=false}" = "true" ]; then + HISTORY_SERVER_CONF="spark.mesos.historyServer.url=${HISTORY_SERVER_WEB_PROXY_BASE}" + fi + + sed "s,,${HISTORY_SERVER_CONF}," \ + /opt/spark/dist/conf/mesos-cluster-dispatcher.properties.template >/opt/spark/dist/conf/mesos-cluster-dispatcher.properties + + if [ "${SPARK_SSL_KEYSTOREBASE64}" != "" ]; then + echo "${SPARK_SSL_KEYSTOREBASE64}" | base64 -d > /tmp/dispatcher-keystore.jks + add_if_non_empty spark.ssl.keyStore /tmp/dispatcher-keystore.jks + fi + + if [ "${SPARK_SSL_TRUSTSTOREBASE64}" != "" ]; then + echo "${SPARK_SSL_TRUSTSTOREBASE64}" | base64 -d > /tmp/dispatcher-truststore.jks + add_if_non_empty spark.ssl.trustStore /tmp/dispatcher-truststore.jks + fi + + add_if_non_empty spark.ssl.enabled "${SPARK_SSL_ENABLED}" + add_if_non_empty spark.ssl.keyPassword "${SPARK_SSL_KEYPASSWORD}" + add_if_non_empty spark.ssl.keyStorePassword "${SPARK_SSL_KEYSTOREPASSWORD}" + add_if_non_empty spark.ssl.trustStorePassword "${SPARK_SSL_TRUSTSTOREPASSWORD}" + add_if_non_empty spark.ssl.protocol "${SPARK_SSL_PROTOCOL}" + add_if_non_empty spark.ssl.enabledAlgorithms "${SPARK_SSL_ENABLEDALGORITHMS}" + + # write defaults + if [ "${DCOS_SERVICE_ACCOUNT_CREDENTIAL}" != "" ]; then + # write defaults using both property names, since 2.0 uses one and 2.1 uses the other + echo "spark.mesos.dispatcher.driverDefault.spark.mesos.driverEnv.MESOS_MODULES=file:///opt/mesosphere/etc/mesos-scheduler-modules/dcos_authenticatee_module.json" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties + echo "spark.mesos.cluster.taskProperty.spark.mesos.driverEnv.MESOS_MODULES=file:///opt/mesosphere/etc/mesos-scheduler-modules/dcos_authenticatee_module.json" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties + + echo "spark.mesos.dispatcher.driverDefault.spark.mesos.driverEnv.MESOS_AUTHENTICATEE=com_mesosphere_dcos_ClassicRPCAuthenticatee" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties + echo "spark.mesos.cluster.taskProperty.spark.mesos.driverEnv.MESOS_AUTHENTICATEE=com_mesosphere_dcos_ClassicRPCAuthenticatee" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties + + echo "spark.mesos.dispatcher.driverDefault.spark.mesos.principal=${SPARK_DISPATCHER_MESOS_PRINCIPAL}" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties + echo "spark.mesos.cluster.taskProperty.spark.mesos.principal=${SPARK_DISPATCHER_MESOS_PRINCIPAL}" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties + fi +} -if [ "${SPARK_SSL_TRUSTSTOREBASE64}" != "" ]; then - echo "${SPARK_SSL_TRUSTSTOREBASE64}" | base64 -d > /tmp/dispatcher-truststore.jks - add_if_non_empty spark.ssl.trustStore /tmp/dispatcher-truststore.jks -fi -add_if_non_empty spark.ssl.enabled "${SPARK_SSL_ENABLED}" -add_if_non_empty spark.ssl.keyPassword "${SPARK_SSL_KEYPASSWORD}" -add_if_non_empty spark.ssl.keyStorePassword "${SPARK_SSL_KEYSTOREPASSWORD}" -add_if_non_empty spark.ssl.trustStorePassword "${SPARK_SSL_TRUSTSTOREPASSWORD}" -add_if_non_empty spark.ssl.protocol "${SPARK_SSL_PROTOCOL}" -add_if_non_empty spark.ssl.enabledAlgorithms "${SPARK_SSL_ENABLEDALGORITHMS}" +export APPLICATION_WEB_PROXY_BASE="${DISPATCHER_UI_WEB_PROXY_BASE}" +set_log_level +export_daemon_opts +configure_properties +ZK="master.mesos:2181" -export ZK="master.mesos:2181" exec /opt/spark/dist/bin/spark-class \ org.apache.spark.deploy.mesos.MesosClusterDispatcher \ --port "${DISPATCHER_PORT}" \ @@ -70,4 +89,4 @@ exec /opt/spark/dist/bin/spark-class \ --zk "${ZK}" \ --host "${HOST}" \ --name "${DCOS_SERVICE_NAME}" \ - --properties-file "conf/mesos-cluster-dispatcher.properties" + --properties-file "/opt/spark/dist/conf/mesos-cluster-dispatcher.properties" diff --git a/docs/user-docs.md b/docs/user-docs.md index b1aa9b38b217b..dbc7d62f36687 100644 --- a/docs/user-docs.md +++ b/docs/user-docs.md @@ -276,7 +276,114 @@ to the history server entry for that job. -### SSL +### Security + +#### Mesos + +##### SSL + + + + + + + +
+ `security.mesos.ssl.enabled` + + Set to true to enable SSL on Mesos communication (default: false). +
+ + +##### Authentication + +When running in +[DC/OS strict security mode](https://docs.mesosphere.com/latest/administration/id-and-access-mgt/), +Both the dispatcher and jobs must authenticate to Mesos using a [DC/OS +Service Account](https://docs.mesosphere.com/1.8/administration/id-and-access-mgt/service-auth/). +Follow these instructions to authenticate in strict mode: + +1. Create a Service Account + +Instructions +[here](https://docs.mesosphere.com/1.8/administration/id-and-access-mgt/service-auth/universe-service-auth/). + +2. Assign Permissions + +First, allow Spark to run tasks as root: + +``` +$ curl -k -L -X PUT \ + -H "Authorization: token=$(dcos config show core.dcos_acs_token)" \ + "$(dcos config show core.dcos_url)/acs/api/v1/acls/dcos:mesos:master:task:user:root" \ + -d '{"description":"Allows root to execute tasks"}' \ + -H 'Content-Type: application/json' + +$ curl -k -L -X PUT \ + -H "Authorization: token=$(dcos config show core.dcos_acs_token)" \ + "$(dcos config show core.dcos_url)/acs/api/v1/acls/dcos:mesos:master:task:user:root/users/${SERVICE_ACCOUNT_NAME}/create" +``` + +Now you must allow Spark to register under the desired role. This is +the value used for `service.role` when installing Spark (default: +`*`): + +``` +$ export ROLE= +$ curl -k -L -X PUT \ + -H "Authorization: token=$(dcos config show core.dcos_acs_token)" \ + "$(dcos config show core.dcos_url)/acs/api/v1/acls/dcos:mesos:master:framework:role:${ROLE}" \ + -d '{"description":"Allows ${ROLE} to register as a framework with the Mesos master"}' \ + -H 'Content-Type: application/json' + +$ curl -k -L -X PUT \ + -H "Authorization: token=$(dcos config show core.dcos_acs_token)" \ + "$(dcos config show core.dcos_url)/acs/api/v1/acls/dcos:mesos:master:framework:role:${ROLE}/users/${SERVICE_ACCOUNT_NAME}/create" +``` + +3. Install Spark + +``` +$ dcos package install spark --options=config.json +``` + +Where `config.json` contains the following JSON. Replace +`` with the name of your service account, and +`` with the name of the DC/OS secret containing your +service account's private key. These values were created in Step #1 +above. + +``` +{ + "service": { + "principal": "", + "user": "nobody" + }, + "security": { + "mesos": { + "authentication": { + "secret_name": "" + } + } + } +} +``` + +4. Submit a Job + +We've now installed the Spark Dispatcher, which is authenticating +itself to the Mesos master. Spark jobs are also frameworks which must +authenticate. The dispatcher will pass the secret along to the jobs, +so all that's left to do is configure our jobs to use DC/OS authentication: + +``` +$ PROPS="-Dspark.mesos.driverEnv.MESOS_MODULES=file:///opt/mesosphere/etc/mesos-scheduler-modules/dcos_authenticatee_module.json " +$ PROPS+="-Dspark.mesos.driverEnv.MESOS_AUTHENTICATEE=com_mesosphere_dcos_ClassicRPCAuthenticatee " +$ PROPS+="-Dspark.mesos.principal=" +$ dcos spark run --submit-args="${PROPS} ..." +``` + +#### Spark SSL SSL support in DC/OS Spark encrypts the following channels: diff --git a/package/config.json b/package/config.json index a999a92e3e6f4..2908487a66879 100644 --- a/package/config.json +++ b/package/config.json @@ -116,6 +116,22 @@ } } }, + "mesos": { + "description": "Mesos scheduler configuration properties.", + "type": "object", + "properties": { + "authentication": { + "description": "Mesos scheduler dcos-oauth configuration.", + "type": "object", + "properties": { + "secret_name": { + "description": "Name of the secret used to authenticate with the Mesos Master.", + "type": "string" + } + } + } + } + }, "ssl": { "description": "Spark SSL certificates and private key configuration.", "type": "object", diff --git a/package/marathon.json.mustache b/package/marathon.json.mustache index 32bc4db5280df..fb72163237be5 100644 --- a/package/marathon.json.mustache +++ b/package/marathon.json.mustache @@ -4,15 +4,10 @@ "mem": {{service.mem}}, "cmd": "/sbin/init.sh", "env": { - "DCOS_SERVICE_NAME": "{{service.name}}", - "SPARK_HDFS_CONFIG_URL": "{{hdfs.config-url}}", - "SPARK_USER": "{{service.user}}", - "SPARK_DISPATCHER_MESOS_ROLE": "{{service.role}}", - "SPARK_DISPATCHER_MESOS_PRINCIPAL": "{{service.principal}}", - "SPARK_DISPATCHER_MESOS_SECRET": "{{service.secret}}", {{#security.kerberos.krb5conf}} "SPARK_MESOS_KRB5_CONF_BASE64": "{{security.kerberos.krb5conf}}", {{/security.kerberos.krb5conf}} + {{#security.ssl.enabled}} "SPARK_SSL_ENABLED": "{{security.ssl.enabled}}", "SPARK_SSL_KEYSTOREBASE64": "{{security.ssl.keyStoreBase64}}", @@ -23,6 +18,19 @@ "SPARK_SSL_PROTOCOL": "{{security.ssl.protocol}}", "SPARK_SSL_ENABLEDALGORITHMS": "{{security.ssl.enabledAlgorithms}}", {{/security.ssl.enabled}} + +{{#security.mesos.authentication.secret_name}} + "DCOS_SERVICE_ACCOUNT_CREDENTIAL": { "secret": "serviceCredential" }, + "MESOS_MODULES": "file:///opt/mesosphere/etc/mesos-scheduler-modules/dcos_authenticatee_module.json", + "MESOS_AUTHENTICATEE": "com_mesosphere_dcos_ClassicRPCAuthenticatee", +{{/security.mesos.authentication.secret_name}} + + "DCOS_SERVICE_NAME": "{{service.name}}", + "SPARK_HDFS_CONFIG_URL": "{{hdfs.config-url}}", + "SPARK_USER": "{{service.user}}", + "SPARK_DISPATCHER_MESOS_ROLE": "{{service.role}}", + "SPARK_DISPATCHER_MESOS_PRINCIPAL": "{{service.principal}}", + "SPARK_DISPATCHER_MESOS_SECRET": "{{service.secret}}", "ENABLE_HISTORY_SERVER": "{{history-server.enabled}}", "HISTORY_LOG_DIR": "{{history-server.log-dir}}", "HISTORY_CLEANER_ENABLED": "{{history-server.cleaner-enabled}}", @@ -32,6 +40,15 @@ "HISTORY_KRB_KEYTAB": "{{history-server.kerberos.keytab}}", "SPARK_LOG_LEVEL": "{{service.log-level}}" }, + +{{#security.mesos.authentication.secret_name}} + "secrets": { + "serviceCredential": { + "source": "{{security.mesos.authentication.secret_name}}" + } + }, +{{/security.mesos.authentication.secret_name}} + "ports": [ 0, 0, @@ -43,6 +60,14 @@ "docker": { "image": "{{resource.assets.container.docker.spark_docker}}", "network": "HOST", +{{#service.user}} + "parameters": [ + { + "key": "user", + "value": "{{service.user}}" + } + ], +{{/service.user}} "forcePullImage": true } }, @@ -78,6 +103,9 @@ "DCOS_SERVICE_SCHEME": "http" {{/security.ssl.enabled}} }, +{{#service.user}} + "user": "{{service.user}}", +{{/service.user}} "uris": [ {{#hdfs.config-url}} "{{hdfs.config-url}}/hdfs-site.xml",