diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 1823edbe0f53d..867cae5af9886 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -67,6 +67,26 @@
netty-transport-native-kqueue
osx-x86_64
+
+ io.netty
+ netty-tcnative-boringssl-static
+ linux-x86_64
+
+
+ io.netty
+ netty-tcnative-boringssl-static
+ linux-aarch_64
+
+
+ io.netty
+ netty-tcnative-boringssl-static
+ osx-aarch_64
+
+
+ io.netty
+ netty-tcnative-boringssl-static
+ osx-x86_64
+
@@ -147,12 +167,24 @@
log4j-slf4j2-impl
test
+
+ org.bouncycastle
+ bcprov-jdk15on
+ ${bouncycastle.version}
+ test
+
+
+ org.bouncycastle
+ bcpkix-jdk15on
+ ${bouncycastle.version}
+ test
+
org.apache.spark
spark-common-utils_${scala.binary.version}
${project.version}
-
+
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index 29c341f8c3525..e7a7342bf3026 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -54,6 +54,13 @@
test-jar
test
+
+ org.apache.spark
+ spark-network-common_${scala.binary.version}
+ ${project.version}
+ test-jar
+ test
+
org.apache.mesos
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
index e5a6a5f1ef166..063bd6c1770e7 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
@@ -172,7 +172,8 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
// This method is factored out for testability
protected def getShuffleClient(): MesosExternalBlockStoreClient = {
new MesosExternalBlockStoreClient(
- SparkTransportConf.fromSparkConf(conf, "shuffle"),
+ SparkTransportConf.fromSparkConfWithSslOptions(
+ conf, "shuffle", sslOptions = Some(securityManager.getSSLOptions("rpc"))),
securityManager,
securityManager.isAuthenticationEnabled(),
conf.get(config.SHUFFLE_REGISTRATION_TIMEOUT))
diff --git a/resource-managers/mesos/src/test/resources/certchain.pem b/resource-managers/mesos/src/test/resources/certchain.pem
new file mode 100644
index 0000000000000..1004cacc9bf9a
--- /dev/null
+++ b/resource-managers/mesos/src/test/resources/certchain.pem
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICsDCCAZgCCQD7yXTHZWZZlDANBgkqhkiG9w0BAQsFADAaMQswCQYDVQQGEwJV
+UzELMAkGA1UECAwCQ0EwHhcNMjMwNjIwMTczMjAzWhcNMzMwNjE3MTczMjAzWjAa
+MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDH4DO8IP/7xZgpzmYrBaqzsnpamq54cXP8JdQUOXP/dmh8myGg
+CUau/nNdpPNr1Od2iUvf1Z9OW+KcHdNAL/zcwe1ehU3d6/M+UinDtfbEb4HSyQ31
+9AIlPSUq+pJAlsAGJYERLGHPBNXEay0r0+TR0cd9CfSN79rXUMag40pZC3zdxXmY
+JpSkhNuiYfa+Z9TgXoki5MzNiyH12gAb9tO8tr55BnE5s/QujOp7LMjlf6VkE7Bp
+hqj1UbcHmFw7U9jyLDfi98uIvlEDFCwXARdmLxxaYAOqdgZ3TtjBvbugVRpRFQiw
+haFzkiok9bh+MclKQBKvF0ArHmMLHkcCd5oPAgMBAAEwDQYJKoZIhvcNAQELBQAD
+ggEBADYIPLwlnuH6rTbkjeZFYC2UXjNesbUe1TXbsBo9DDHJUSFjNNDDAUpSzhxb
+q6nMvex7tnTvTjAgOQR/qwAueAfcXHWe0EKvn4Y6yJERepSsYg5bSYBt+UJxW89R
+JRLmzBFxEJy1YhsqGCh+I2wRoRz8ZGokDyqcrAlwlzXYVDfNC4wUo14Cm+s90yc3
+2I/roX/MWec8QbEbr25psAYVnRdUL1mzCeQMc83A8Y0SDPfF5ECFhvFXkVaDTULO
+RddXWJoC4K5RuGa6yvpb75I8VTE3fwE2ykSgPuMShNZREDCuszkpPjjFumq9pCOJ
+nUO1huCqjxC1ehPe/9/jgmzoVX4=
+-----END CERTIFICATE-----
diff --git a/resource-managers/mesos/src/test/resources/key.pem b/resource-managers/mesos/src/test/resources/key.pem
new file mode 100644
index 0000000000000..77122755bfdaf
--- /dev/null
+++ b/resource-managers/mesos/src/test/resources/key.pem
@@ -0,0 +1,29 @@
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIE6TAbBgkqhkiG9w0BBQMwDgQIGBIe7ugOgfACAggABIIEyJgkzYc/ixcvwLJC
+eTzGOVwk+F1cqM4H63FOxIjroaxisceqoBmty6Rf4PJ1C9nprkSs6G/SkupbNUUB
+YiWmsQ91orllbHsczAc+qaa0tmommwgt27ZrfXdXBxDB0mJWQTijkVHWfyTqcXmC
+oeWlvTFsilA4CoVakryZQScl3qH/aN5zazg9gjx2xNCRwFexeccC7TqICJkPtnJC
++6wrSby2A9HlJs/MdtYyhfN360GDKvQygnw+wQj+san8EV5s7I7b45SsdEx5vOxP
++AKc6h7loWJkLrJFqDGtfqY/TY76t+sQpinS7R3sA4uYaT1bIx8Feu0GcIbnr3NS
+54St9hNfOgEDmWKFj0ZmMTEISOujj8hNTKYdc0Z/dx/+izqNCuLEXiJjLDIPxfRJ
+EfeYG0/4fBxdeZgIwIVVsUXX4eSzXhguuiwulNhRkUzKhH9aNrp1fw5t0hgsJOPx
+O8Y2sAtDL6KUHx+rt0ejrHYXK1+BOUCHcZiHCGmLCCQlrcX5TWDYPkhtrbtbxS6m
+p9aVxq7pcyxelVlUBXtWeYvcOHGueEd7QQL16uYbhrTHFIwx2Pw//LNIgyJTBNu7
+hxm2jica15PaALtSsYDRhsE9VkWawW9AXeBWOnEj7YKrT5ejrQLI9eCxtolRbVNd
+gSR5r7MQkPssjCU+pdsCl98e0mxVnq8eMDtZYSIDGLEyPfJNCsxDanzUuxSfBP8K
+cIDzREINA/QiuhyGxxBB8dR6k+kl1LNVy8FA7RYAe84MLW2wuaFMQirMDtTLo/Wt
+/AatxW4WKlCvfd/nC8O5xlzmF5qffhgmS8xYDL/w1G3Uxo21dA4gGipH9uw91wqi
+YaSidtcVs6JbHpUddmO5AiEiSBbbwqNgaOxNdur1WYWZHDNWvCKL9sqQy/HtDLHy
+Tzyuw8GtrpB2BKfWWwbAApqvjcqgjitEXk2Nw/L3qWfWmVStP9ys5rz27UWRhMFi
+Go/HrVh7heOxK16ei5tp2OyRLSfDBZ7+IlpbbnR26BPdBE08cuBo/ELOfifnYTTS
+V4CKLMiG7RxtdAddkFKO+GgNW1nNHppBBhJzDvuBcFuUfB+AdnymZTlA8RFha7aW
+zwtg6I1ABdMGPn+wzMhkkutDtSCWpkBRddJPcB1mwmRdp/2WC3NxuaMQX3anQG36
+9m2sxWUmT8ZLGvFDHIwGbRPT5zzcvIJV/xhZdCxhg/7tgLikBZBB8TmtDBck+wq+
+DPIEQkr4rObCi9xphSpHvPBGhdI5v8xbKEGLcPzVMW0hjaHouvQipEXC+ASrn/sG
+nytZkyt1DD8KG27wlcrDl/RDCcjNvlKkgKPme3pPsDcD+qX+eqjQ4OM9AexW+VLZ
+ZUa84/Fh6yjbuPF3vtCVRwFJURzhKMVG3Fcs7C3iczCOUNDOar9k0yCrmbACF9Wm
+kSD5lXPXe1fFq0xi21Isuz+FH4A5CR/tHc2i+avQhYs9FvaqzLiaNmLaZKrhX5uy
+dJXYtLruhgwBjv4eo6GXm8/WHFG6r4iaq6NEimQoT41MH+uJr9nAiBWg397JoHpG
+jheCZDpZBAVuEz8NUdWP7mu64DQsjKeY6okwMlXlcSUKlMnx8QCtEMTj/JF7E7dT
+bHYe30+OIWl3X88v9Q==
+-----END ENCRYPTED PRIVATE KEY-----
diff --git a/resource-managers/mesos/src/test/resources/keystore b/resource-managers/mesos/src/test/resources/keystore
new file mode 100644
index 0000000000000..f8310e39ba1e0
Binary files /dev/null and b/resource-managers/mesos/src/test/resources/keystore differ
diff --git a/resource-managers/mesos/src/test/resources/truststore b/resource-managers/mesos/src/test/resources/truststore
new file mode 100644
index 0000000000000..a6b1d46e1f391
Binary files /dev/null and b/resource-managers/mesos/src/test/resources/truststore differ
diff --git a/resource-managers/mesos/src/test/resources/untrusted-keystore b/resource-managers/mesos/src/test/resources/untrusted-keystore
new file mode 100644
index 0000000000000..6015b02caa128
Binary files /dev/null and b/resource-managers/mesos/src/test/resources/untrusted-keystore differ
diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
index 2b7272a490376..79b65ddd869cc 100644
--- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
+++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
@@ -34,6 +34,7 @@ import org.apache.spark.{LocalSparkContext, SecurityManager, SparkConf, SparkCon
import org.apache.spark.deploy.mesos.{config => mesosConfig}
import org.apache.spark.internal.config._
import org.apache.spark.network.shuffle.mesos.MesosExternalBlockStoreClient
+import org.apache.spark.network.ssl.SslSampleConfigs
import org.apache.spark.resource.ResourceProfile
import org.apache.spark.rpc.{RpcAddress, RpcEndpointRef}
import org.apache.spark.scheduler.TaskSchedulerImpl
@@ -46,7 +47,7 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
with BeforeAndAfter
with ScalaFutures {
- private var sparkConf: SparkConf = _
+ protected var sparkConf: SparkConf = _
private var driver: SchedulerDriver = _
private var taskScheduler: TaskSchedulerImpl = _
private var backend: MesosCoarseGrainedSchedulerBackend = _
@@ -807,7 +808,7 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
backend
}
- private def initializeSparkConf(
+ protected def initializeSparkConf(
sparkConfVars: Map[String, String] = null,
home: String = "/path"): Unit = {
sparkConf = (new SparkConf)
@@ -841,3 +842,13 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
backend = createSchedulerBackend(taskScheduler, driver, externalShuffleClient)
}
}
+
+class SslMesosCoarseGrainedSchedulerBackendSuite extends MesosCoarseGrainedSchedulerBackendSuite {
+ override def initializeSparkConf(
+ sparkConfVars: Map[String, String] = null,
+ home: String = "/path"): Unit = {
+ super.initializeSparkConf(sparkConfVars, home)
+ val updatedConfigs = SslSampleConfigs.createDefaultConfigMap()
+ updatedConfigs.entrySet().forEach(entry => sparkConf.set(entry.getKey, entry.getValue))
+ }
+}
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index e58ab1ea25050..6071d557b5a56 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -91,6 +91,13 @@
test-jar
test
+
+ org.apache.spark
+ spark-network-common_${scala.binary.version}
+ ${project.version}
+ test-jar
+ test
+
org.apache.hadoop
hadoop-client-api
diff --git a/resource-managers/yarn/src/test/resources/certchain.pem b/resource-managers/yarn/src/test/resources/certchain.pem
new file mode 100644
index 0000000000000..1004cacc9bf9a
--- /dev/null
+++ b/resource-managers/yarn/src/test/resources/certchain.pem
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICsDCCAZgCCQD7yXTHZWZZlDANBgkqhkiG9w0BAQsFADAaMQswCQYDVQQGEwJV
+UzELMAkGA1UECAwCQ0EwHhcNMjMwNjIwMTczMjAzWhcNMzMwNjE3MTczMjAzWjAa
+MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDH4DO8IP/7xZgpzmYrBaqzsnpamq54cXP8JdQUOXP/dmh8myGg
+CUau/nNdpPNr1Od2iUvf1Z9OW+KcHdNAL/zcwe1ehU3d6/M+UinDtfbEb4HSyQ31
+9AIlPSUq+pJAlsAGJYERLGHPBNXEay0r0+TR0cd9CfSN79rXUMag40pZC3zdxXmY
+JpSkhNuiYfa+Z9TgXoki5MzNiyH12gAb9tO8tr55BnE5s/QujOp7LMjlf6VkE7Bp
+hqj1UbcHmFw7U9jyLDfi98uIvlEDFCwXARdmLxxaYAOqdgZ3TtjBvbugVRpRFQiw
+haFzkiok9bh+MclKQBKvF0ArHmMLHkcCd5oPAgMBAAEwDQYJKoZIhvcNAQELBQAD
+ggEBADYIPLwlnuH6rTbkjeZFYC2UXjNesbUe1TXbsBo9DDHJUSFjNNDDAUpSzhxb
+q6nMvex7tnTvTjAgOQR/qwAueAfcXHWe0EKvn4Y6yJERepSsYg5bSYBt+UJxW89R
+JRLmzBFxEJy1YhsqGCh+I2wRoRz8ZGokDyqcrAlwlzXYVDfNC4wUo14Cm+s90yc3
+2I/roX/MWec8QbEbr25psAYVnRdUL1mzCeQMc83A8Y0SDPfF5ECFhvFXkVaDTULO
+RddXWJoC4K5RuGa6yvpb75I8VTE3fwE2ykSgPuMShNZREDCuszkpPjjFumq9pCOJ
+nUO1huCqjxC1ehPe/9/jgmzoVX4=
+-----END CERTIFICATE-----
diff --git a/resource-managers/yarn/src/test/resources/key.pem b/resource-managers/yarn/src/test/resources/key.pem
new file mode 100644
index 0000000000000..77122755bfdaf
--- /dev/null
+++ b/resource-managers/yarn/src/test/resources/key.pem
@@ -0,0 +1,29 @@
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIE6TAbBgkqhkiG9w0BBQMwDgQIGBIe7ugOgfACAggABIIEyJgkzYc/ixcvwLJC
+eTzGOVwk+F1cqM4H63FOxIjroaxisceqoBmty6Rf4PJ1C9nprkSs6G/SkupbNUUB
+YiWmsQ91orllbHsczAc+qaa0tmommwgt27ZrfXdXBxDB0mJWQTijkVHWfyTqcXmC
+oeWlvTFsilA4CoVakryZQScl3qH/aN5zazg9gjx2xNCRwFexeccC7TqICJkPtnJC
++6wrSby2A9HlJs/MdtYyhfN360GDKvQygnw+wQj+san8EV5s7I7b45SsdEx5vOxP
++AKc6h7loWJkLrJFqDGtfqY/TY76t+sQpinS7R3sA4uYaT1bIx8Feu0GcIbnr3NS
+54St9hNfOgEDmWKFj0ZmMTEISOujj8hNTKYdc0Z/dx/+izqNCuLEXiJjLDIPxfRJ
+EfeYG0/4fBxdeZgIwIVVsUXX4eSzXhguuiwulNhRkUzKhH9aNrp1fw5t0hgsJOPx
+O8Y2sAtDL6KUHx+rt0ejrHYXK1+BOUCHcZiHCGmLCCQlrcX5TWDYPkhtrbtbxS6m
+p9aVxq7pcyxelVlUBXtWeYvcOHGueEd7QQL16uYbhrTHFIwx2Pw//LNIgyJTBNu7
+hxm2jica15PaALtSsYDRhsE9VkWawW9AXeBWOnEj7YKrT5ejrQLI9eCxtolRbVNd
+gSR5r7MQkPssjCU+pdsCl98e0mxVnq8eMDtZYSIDGLEyPfJNCsxDanzUuxSfBP8K
+cIDzREINA/QiuhyGxxBB8dR6k+kl1LNVy8FA7RYAe84MLW2wuaFMQirMDtTLo/Wt
+/AatxW4WKlCvfd/nC8O5xlzmF5qffhgmS8xYDL/w1G3Uxo21dA4gGipH9uw91wqi
+YaSidtcVs6JbHpUddmO5AiEiSBbbwqNgaOxNdur1WYWZHDNWvCKL9sqQy/HtDLHy
+Tzyuw8GtrpB2BKfWWwbAApqvjcqgjitEXk2Nw/L3qWfWmVStP9ys5rz27UWRhMFi
+Go/HrVh7heOxK16ei5tp2OyRLSfDBZ7+IlpbbnR26BPdBE08cuBo/ELOfifnYTTS
+V4CKLMiG7RxtdAddkFKO+GgNW1nNHppBBhJzDvuBcFuUfB+AdnymZTlA8RFha7aW
+zwtg6I1ABdMGPn+wzMhkkutDtSCWpkBRddJPcB1mwmRdp/2WC3NxuaMQX3anQG36
+9m2sxWUmT8ZLGvFDHIwGbRPT5zzcvIJV/xhZdCxhg/7tgLikBZBB8TmtDBck+wq+
+DPIEQkr4rObCi9xphSpHvPBGhdI5v8xbKEGLcPzVMW0hjaHouvQipEXC+ASrn/sG
+nytZkyt1DD8KG27wlcrDl/RDCcjNvlKkgKPme3pPsDcD+qX+eqjQ4OM9AexW+VLZ
+ZUa84/Fh6yjbuPF3vtCVRwFJURzhKMVG3Fcs7C3iczCOUNDOar9k0yCrmbACF9Wm
+kSD5lXPXe1fFq0xi21Isuz+FH4A5CR/tHc2i+avQhYs9FvaqzLiaNmLaZKrhX5uy
+dJXYtLruhgwBjv4eo6GXm8/WHFG6r4iaq6NEimQoT41MH+uJr9nAiBWg397JoHpG
+jheCZDpZBAVuEz8NUdWP7mu64DQsjKeY6okwMlXlcSUKlMnx8QCtEMTj/JF7E7dT
+bHYe30+OIWl3X88v9Q==
+-----END ENCRYPTED PRIVATE KEY-----
diff --git a/resource-managers/yarn/src/test/resources/keystore b/resource-managers/yarn/src/test/resources/keystore
new file mode 100644
index 0000000000000..f8310e39ba1e0
Binary files /dev/null and b/resource-managers/yarn/src/test/resources/keystore differ
diff --git a/resource-managers/yarn/src/test/resources/truststore b/resource-managers/yarn/src/test/resources/truststore
new file mode 100644
index 0000000000000..a6b1d46e1f391
Binary files /dev/null and b/resource-managers/yarn/src/test/resources/truststore differ
diff --git a/resource-managers/yarn/src/test/resources/untrusted-keystore b/resource-managers/yarn/src/test/resources/untrusted-keystore
new file mode 100644
index 0000000000000..6015b02caa128
Binary files /dev/null and b/resource-managers/yarn/src/test/resources/untrusted-keystore differ
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/SslYarnShuffleServiceSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/SslYarnShuffleServiceSuite.scala
new file mode 100644
index 0000000000000..41a4331ab1e4f
--- /dev/null
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/SslYarnShuffleServiceSuite.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.yarn
+
+import org.apache.spark.network.ssl.SslSampleConfigs
+
+class SslYarnShuffleServiceWithRocksDBBackendSuite
+ extends YarnShuffleServiceWithRocksDBBackendSuite {
+
+ /**
+ * Override to add "spark.ssl.rpc.*" configuration parameters...
+ */
+ override def beforeEach(): Unit = {
+ super.beforeEach()
+ val updatedConfigs = SslSampleConfigs.createDefaultConfigMapForRpcNamespace()
+ updatedConfigs.entrySet().forEach(entry => yarnConfig.set(entry.getKey, entry.getValue))
+ }
+}