diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 1823edbe0f53d..867cae5af9886 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -67,6 +67,26 @@ netty-transport-native-kqueue osx-x86_64 + + io.netty + netty-tcnative-boringssl-static + linux-x86_64 + + + io.netty + netty-tcnative-boringssl-static + linux-aarch_64 + + + io.netty + netty-tcnative-boringssl-static + osx-aarch_64 + + + io.netty + netty-tcnative-boringssl-static + osx-x86_64 + @@ -147,12 +167,24 @@ log4j-slf4j2-impl test + + org.bouncycastle + bcprov-jdk15on + ${bouncycastle.version} + test + + + org.bouncycastle + bcpkix-jdk15on + ${bouncycastle.version} + test + org.apache.spark spark-common-utils_${scala.binary.version} ${project.version} - + diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index 29c341f8c3525..e7a7342bf3026 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -54,6 +54,13 @@ test-jar test + + org.apache.spark + spark-network-common_${scala.binary.version} + ${project.version} + test-jar + test + org.apache.mesos diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala index e5a6a5f1ef166..063bd6c1770e7 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala @@ -172,7 +172,8 @@ private[spark] class MesosCoarseGrainedSchedulerBackend( // This method is factored out for testability protected def getShuffleClient(): MesosExternalBlockStoreClient = { new MesosExternalBlockStoreClient( - SparkTransportConf.fromSparkConf(conf, "shuffle"), + SparkTransportConf.fromSparkConfWithSslOptions( + conf, "shuffle", sslOptions = Some(securityManager.getSSLOptions("rpc"))), securityManager, securityManager.isAuthenticationEnabled(), conf.get(config.SHUFFLE_REGISTRATION_TIMEOUT)) diff --git a/resource-managers/mesos/src/test/resources/certchain.pem b/resource-managers/mesos/src/test/resources/certchain.pem new file mode 100644 index 0000000000000..1004cacc9bf9a --- /dev/null +++ b/resource-managers/mesos/src/test/resources/certchain.pem @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICsDCCAZgCCQD7yXTHZWZZlDANBgkqhkiG9w0BAQsFADAaMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0EwHhcNMjMwNjIwMTczMjAzWhcNMzMwNjE3MTczMjAzWjAa +MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDH4DO8IP/7xZgpzmYrBaqzsnpamq54cXP8JdQUOXP/dmh8myGg +CUau/nNdpPNr1Od2iUvf1Z9OW+KcHdNAL/zcwe1ehU3d6/M+UinDtfbEb4HSyQ31 +9AIlPSUq+pJAlsAGJYERLGHPBNXEay0r0+TR0cd9CfSN79rXUMag40pZC3zdxXmY +JpSkhNuiYfa+Z9TgXoki5MzNiyH12gAb9tO8tr55BnE5s/QujOp7LMjlf6VkE7Bp +hqj1UbcHmFw7U9jyLDfi98uIvlEDFCwXARdmLxxaYAOqdgZ3TtjBvbugVRpRFQiw +haFzkiok9bh+MclKQBKvF0ArHmMLHkcCd5oPAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggEBADYIPLwlnuH6rTbkjeZFYC2UXjNesbUe1TXbsBo9DDHJUSFjNNDDAUpSzhxb +q6nMvex7tnTvTjAgOQR/qwAueAfcXHWe0EKvn4Y6yJERepSsYg5bSYBt+UJxW89R +JRLmzBFxEJy1YhsqGCh+I2wRoRz8ZGokDyqcrAlwlzXYVDfNC4wUo14Cm+s90yc3 +2I/roX/MWec8QbEbr25psAYVnRdUL1mzCeQMc83A8Y0SDPfF5ECFhvFXkVaDTULO +RddXWJoC4K5RuGa6yvpb75I8VTE3fwE2ykSgPuMShNZREDCuszkpPjjFumq9pCOJ +nUO1huCqjxC1ehPe/9/jgmzoVX4= +-----END CERTIFICATE----- diff --git a/resource-managers/mesos/src/test/resources/key.pem b/resource-managers/mesos/src/test/resources/key.pem new file mode 100644 index 0000000000000..77122755bfdaf --- /dev/null +++ b/resource-managers/mesos/src/test/resources/key.pem @@ -0,0 +1,29 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIE6TAbBgkqhkiG9w0BBQMwDgQIGBIe7ugOgfACAggABIIEyJgkzYc/ixcvwLJC +eTzGOVwk+F1cqM4H63FOxIjroaxisceqoBmty6Rf4PJ1C9nprkSs6G/SkupbNUUB +YiWmsQ91orllbHsczAc+qaa0tmommwgt27ZrfXdXBxDB0mJWQTijkVHWfyTqcXmC +oeWlvTFsilA4CoVakryZQScl3qH/aN5zazg9gjx2xNCRwFexeccC7TqICJkPtnJC ++6wrSby2A9HlJs/MdtYyhfN360GDKvQygnw+wQj+san8EV5s7I7b45SsdEx5vOxP ++AKc6h7loWJkLrJFqDGtfqY/TY76t+sQpinS7R3sA4uYaT1bIx8Feu0GcIbnr3NS +54St9hNfOgEDmWKFj0ZmMTEISOujj8hNTKYdc0Z/dx/+izqNCuLEXiJjLDIPxfRJ +EfeYG0/4fBxdeZgIwIVVsUXX4eSzXhguuiwulNhRkUzKhH9aNrp1fw5t0hgsJOPx +O8Y2sAtDL6KUHx+rt0ejrHYXK1+BOUCHcZiHCGmLCCQlrcX5TWDYPkhtrbtbxS6m +p9aVxq7pcyxelVlUBXtWeYvcOHGueEd7QQL16uYbhrTHFIwx2Pw//LNIgyJTBNu7 +hxm2jica15PaALtSsYDRhsE9VkWawW9AXeBWOnEj7YKrT5ejrQLI9eCxtolRbVNd +gSR5r7MQkPssjCU+pdsCl98e0mxVnq8eMDtZYSIDGLEyPfJNCsxDanzUuxSfBP8K +cIDzREINA/QiuhyGxxBB8dR6k+kl1LNVy8FA7RYAe84MLW2wuaFMQirMDtTLo/Wt +/AatxW4WKlCvfd/nC8O5xlzmF5qffhgmS8xYDL/w1G3Uxo21dA4gGipH9uw91wqi +YaSidtcVs6JbHpUddmO5AiEiSBbbwqNgaOxNdur1WYWZHDNWvCKL9sqQy/HtDLHy +Tzyuw8GtrpB2BKfWWwbAApqvjcqgjitEXk2Nw/L3qWfWmVStP9ys5rz27UWRhMFi +Go/HrVh7heOxK16ei5tp2OyRLSfDBZ7+IlpbbnR26BPdBE08cuBo/ELOfifnYTTS +V4CKLMiG7RxtdAddkFKO+GgNW1nNHppBBhJzDvuBcFuUfB+AdnymZTlA8RFha7aW +zwtg6I1ABdMGPn+wzMhkkutDtSCWpkBRddJPcB1mwmRdp/2WC3NxuaMQX3anQG36 +9m2sxWUmT8ZLGvFDHIwGbRPT5zzcvIJV/xhZdCxhg/7tgLikBZBB8TmtDBck+wq+ +DPIEQkr4rObCi9xphSpHvPBGhdI5v8xbKEGLcPzVMW0hjaHouvQipEXC+ASrn/sG +nytZkyt1DD8KG27wlcrDl/RDCcjNvlKkgKPme3pPsDcD+qX+eqjQ4OM9AexW+VLZ +ZUa84/Fh6yjbuPF3vtCVRwFJURzhKMVG3Fcs7C3iczCOUNDOar9k0yCrmbACF9Wm +kSD5lXPXe1fFq0xi21Isuz+FH4A5CR/tHc2i+avQhYs9FvaqzLiaNmLaZKrhX5uy +dJXYtLruhgwBjv4eo6GXm8/WHFG6r4iaq6NEimQoT41MH+uJr9nAiBWg397JoHpG +jheCZDpZBAVuEz8NUdWP7mu64DQsjKeY6okwMlXlcSUKlMnx8QCtEMTj/JF7E7dT +bHYe30+OIWl3X88v9Q== +-----END ENCRYPTED PRIVATE KEY----- diff --git a/resource-managers/mesos/src/test/resources/keystore b/resource-managers/mesos/src/test/resources/keystore new file mode 100644 index 0000000000000..f8310e39ba1e0 Binary files /dev/null and b/resource-managers/mesos/src/test/resources/keystore differ diff --git a/resource-managers/mesos/src/test/resources/truststore b/resource-managers/mesos/src/test/resources/truststore new file mode 100644 index 0000000000000..a6b1d46e1f391 Binary files /dev/null and b/resource-managers/mesos/src/test/resources/truststore differ diff --git a/resource-managers/mesos/src/test/resources/untrusted-keystore b/resource-managers/mesos/src/test/resources/untrusted-keystore new file mode 100644 index 0000000000000..6015b02caa128 Binary files /dev/null and b/resource-managers/mesos/src/test/resources/untrusted-keystore differ diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala index 2b7272a490376..79b65ddd869cc 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala @@ -34,6 +34,7 @@ import org.apache.spark.{LocalSparkContext, SecurityManager, SparkConf, SparkCon import org.apache.spark.deploy.mesos.{config => mesosConfig} import org.apache.spark.internal.config._ import org.apache.spark.network.shuffle.mesos.MesosExternalBlockStoreClient +import org.apache.spark.network.ssl.SslSampleConfigs import org.apache.spark.resource.ResourceProfile import org.apache.spark.rpc.{RpcAddress, RpcEndpointRef} import org.apache.spark.scheduler.TaskSchedulerImpl @@ -46,7 +47,7 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite with BeforeAndAfter with ScalaFutures { - private var sparkConf: SparkConf = _ + protected var sparkConf: SparkConf = _ private var driver: SchedulerDriver = _ private var taskScheduler: TaskSchedulerImpl = _ private var backend: MesosCoarseGrainedSchedulerBackend = _ @@ -807,7 +808,7 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite backend } - private def initializeSparkConf( + protected def initializeSparkConf( sparkConfVars: Map[String, String] = null, home: String = "/path"): Unit = { sparkConf = (new SparkConf) @@ -841,3 +842,13 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite backend = createSchedulerBackend(taskScheduler, driver, externalShuffleClient) } } + +class SslMesosCoarseGrainedSchedulerBackendSuite extends MesosCoarseGrainedSchedulerBackendSuite { + override def initializeSparkConf( + sparkConfVars: Map[String, String] = null, + home: String = "/path"): Unit = { + super.initializeSparkConf(sparkConfVars, home) + val updatedConfigs = SslSampleConfigs.createDefaultConfigMap() + updatedConfigs.entrySet().forEach(entry => sparkConf.set(entry.getKey, entry.getValue)) + } +} diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index e58ab1ea25050..6071d557b5a56 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -91,6 +91,13 @@ test-jar test + + org.apache.spark + spark-network-common_${scala.binary.version} + ${project.version} + test-jar + test + org.apache.hadoop hadoop-client-api diff --git a/resource-managers/yarn/src/test/resources/certchain.pem b/resource-managers/yarn/src/test/resources/certchain.pem new file mode 100644 index 0000000000000..1004cacc9bf9a --- /dev/null +++ b/resource-managers/yarn/src/test/resources/certchain.pem @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICsDCCAZgCCQD7yXTHZWZZlDANBgkqhkiG9w0BAQsFADAaMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0EwHhcNMjMwNjIwMTczMjAzWhcNMzMwNjE3MTczMjAzWjAa +MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDH4DO8IP/7xZgpzmYrBaqzsnpamq54cXP8JdQUOXP/dmh8myGg +CUau/nNdpPNr1Od2iUvf1Z9OW+KcHdNAL/zcwe1ehU3d6/M+UinDtfbEb4HSyQ31 +9AIlPSUq+pJAlsAGJYERLGHPBNXEay0r0+TR0cd9CfSN79rXUMag40pZC3zdxXmY +JpSkhNuiYfa+Z9TgXoki5MzNiyH12gAb9tO8tr55BnE5s/QujOp7LMjlf6VkE7Bp +hqj1UbcHmFw7U9jyLDfi98uIvlEDFCwXARdmLxxaYAOqdgZ3TtjBvbugVRpRFQiw +haFzkiok9bh+MclKQBKvF0ArHmMLHkcCd5oPAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggEBADYIPLwlnuH6rTbkjeZFYC2UXjNesbUe1TXbsBo9DDHJUSFjNNDDAUpSzhxb +q6nMvex7tnTvTjAgOQR/qwAueAfcXHWe0EKvn4Y6yJERepSsYg5bSYBt+UJxW89R +JRLmzBFxEJy1YhsqGCh+I2wRoRz8ZGokDyqcrAlwlzXYVDfNC4wUo14Cm+s90yc3 +2I/roX/MWec8QbEbr25psAYVnRdUL1mzCeQMc83A8Y0SDPfF5ECFhvFXkVaDTULO +RddXWJoC4K5RuGa6yvpb75I8VTE3fwE2ykSgPuMShNZREDCuszkpPjjFumq9pCOJ +nUO1huCqjxC1ehPe/9/jgmzoVX4= +-----END CERTIFICATE----- diff --git a/resource-managers/yarn/src/test/resources/key.pem b/resource-managers/yarn/src/test/resources/key.pem new file mode 100644 index 0000000000000..77122755bfdaf --- /dev/null +++ b/resource-managers/yarn/src/test/resources/key.pem @@ -0,0 +1,29 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIE6TAbBgkqhkiG9w0BBQMwDgQIGBIe7ugOgfACAggABIIEyJgkzYc/ixcvwLJC +eTzGOVwk+F1cqM4H63FOxIjroaxisceqoBmty6Rf4PJ1C9nprkSs6G/SkupbNUUB +YiWmsQ91orllbHsczAc+qaa0tmommwgt27ZrfXdXBxDB0mJWQTijkVHWfyTqcXmC +oeWlvTFsilA4CoVakryZQScl3qH/aN5zazg9gjx2xNCRwFexeccC7TqICJkPtnJC ++6wrSby2A9HlJs/MdtYyhfN360GDKvQygnw+wQj+san8EV5s7I7b45SsdEx5vOxP ++AKc6h7loWJkLrJFqDGtfqY/TY76t+sQpinS7R3sA4uYaT1bIx8Feu0GcIbnr3NS +54St9hNfOgEDmWKFj0ZmMTEISOujj8hNTKYdc0Z/dx/+izqNCuLEXiJjLDIPxfRJ +EfeYG0/4fBxdeZgIwIVVsUXX4eSzXhguuiwulNhRkUzKhH9aNrp1fw5t0hgsJOPx +O8Y2sAtDL6KUHx+rt0ejrHYXK1+BOUCHcZiHCGmLCCQlrcX5TWDYPkhtrbtbxS6m +p9aVxq7pcyxelVlUBXtWeYvcOHGueEd7QQL16uYbhrTHFIwx2Pw//LNIgyJTBNu7 +hxm2jica15PaALtSsYDRhsE9VkWawW9AXeBWOnEj7YKrT5ejrQLI9eCxtolRbVNd +gSR5r7MQkPssjCU+pdsCl98e0mxVnq8eMDtZYSIDGLEyPfJNCsxDanzUuxSfBP8K +cIDzREINA/QiuhyGxxBB8dR6k+kl1LNVy8FA7RYAe84MLW2wuaFMQirMDtTLo/Wt +/AatxW4WKlCvfd/nC8O5xlzmF5qffhgmS8xYDL/w1G3Uxo21dA4gGipH9uw91wqi +YaSidtcVs6JbHpUddmO5AiEiSBbbwqNgaOxNdur1WYWZHDNWvCKL9sqQy/HtDLHy +Tzyuw8GtrpB2BKfWWwbAApqvjcqgjitEXk2Nw/L3qWfWmVStP9ys5rz27UWRhMFi +Go/HrVh7heOxK16ei5tp2OyRLSfDBZ7+IlpbbnR26BPdBE08cuBo/ELOfifnYTTS +V4CKLMiG7RxtdAddkFKO+GgNW1nNHppBBhJzDvuBcFuUfB+AdnymZTlA8RFha7aW +zwtg6I1ABdMGPn+wzMhkkutDtSCWpkBRddJPcB1mwmRdp/2WC3NxuaMQX3anQG36 +9m2sxWUmT8ZLGvFDHIwGbRPT5zzcvIJV/xhZdCxhg/7tgLikBZBB8TmtDBck+wq+ +DPIEQkr4rObCi9xphSpHvPBGhdI5v8xbKEGLcPzVMW0hjaHouvQipEXC+ASrn/sG +nytZkyt1DD8KG27wlcrDl/RDCcjNvlKkgKPme3pPsDcD+qX+eqjQ4OM9AexW+VLZ +ZUa84/Fh6yjbuPF3vtCVRwFJURzhKMVG3Fcs7C3iczCOUNDOar9k0yCrmbACF9Wm +kSD5lXPXe1fFq0xi21Isuz+FH4A5CR/tHc2i+avQhYs9FvaqzLiaNmLaZKrhX5uy +dJXYtLruhgwBjv4eo6GXm8/WHFG6r4iaq6NEimQoT41MH+uJr9nAiBWg397JoHpG +jheCZDpZBAVuEz8NUdWP7mu64DQsjKeY6okwMlXlcSUKlMnx8QCtEMTj/JF7E7dT +bHYe30+OIWl3X88v9Q== +-----END ENCRYPTED PRIVATE KEY----- diff --git a/resource-managers/yarn/src/test/resources/keystore b/resource-managers/yarn/src/test/resources/keystore new file mode 100644 index 0000000000000..f8310e39ba1e0 Binary files /dev/null and b/resource-managers/yarn/src/test/resources/keystore differ diff --git a/resource-managers/yarn/src/test/resources/truststore b/resource-managers/yarn/src/test/resources/truststore new file mode 100644 index 0000000000000..a6b1d46e1f391 Binary files /dev/null and b/resource-managers/yarn/src/test/resources/truststore differ diff --git a/resource-managers/yarn/src/test/resources/untrusted-keystore b/resource-managers/yarn/src/test/resources/untrusted-keystore new file mode 100644 index 0000000000000..6015b02caa128 Binary files /dev/null and b/resource-managers/yarn/src/test/resources/untrusted-keystore differ diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/SslYarnShuffleServiceSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/SslYarnShuffleServiceSuite.scala new file mode 100644 index 0000000000000..41a4331ab1e4f --- /dev/null +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/SslYarnShuffleServiceSuite.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.yarn + +import org.apache.spark.network.ssl.SslSampleConfigs + +class SslYarnShuffleServiceWithRocksDBBackendSuite + extends YarnShuffleServiceWithRocksDBBackendSuite { + + /** + * Override to add "spark.ssl.rpc.*" configuration parameters... + */ + override def beforeEach(): Unit = { + super.beforeEach() + val updatedConfigs = SslSampleConfigs.createDefaultConfigMapForRpcNamespace() + updatedConfigs.entrySet().forEach(entry => yarnConfig.set(entry.getKey, entry.getValue)) + } +}