From e2e7223667becab04f4ec3160a32756e60c31c44 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 17 Aug 2018 13:16:34 +0100 Subject: [PATCH 01/53] add spec template configs --- .../scala/org/apache/spark/deploy/k8s/Config.scala | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index 4442333c573cc..137d2cb4643fa 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -212,6 +212,19 @@ private[spark] object Config extends Logging { "Ensure that major Python version is either Python2 or Python3") .createWithDefault("2") + val KUBERNETES_DRIVER_PODTEMPLATE_FILE = + ConfigBuilder("spark.kubernetes.driver.podTemplateFile") + .doc("File containing a template pod spec for the driver") + .stringConf + .createOptional + + val KUBERNETES_EXECUTOR_PODTEMPLATE_FILE = + ConfigBuilder("spark.kubernetes.executor.podTemplateFile") + .doc("File containing a template pod spec for executors") + .stringConf + .createOptional + + val KUBERNETES_AUTH_SUBMISSION_CONF_PREFIX = "spark.kubernetes.authenticate.submission" From ea4dde6e3733e092160143c781afa627b084a1b8 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 17 Aug 2018 15:09:56 +0100 Subject: [PATCH 02/53] start from template for driver --- .../deploy/k8s/KubernetesDriverSpec.scala | 11 ++++++----- .../submit/KubernetesClientApplication.scala | 18 ++++++++++++------ .../k8s/submit/KubernetesDriverBuilder.scala | 7 +++++-- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala index 0c5ae022f4070..94515f73e389e 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.deploy.k8s -import io.fabric8.kubernetes.api.model.HasMetadata +import io.fabric8.kubernetes.api.model.{HasMetadata, Pod} private[spark] case class KubernetesDriverSpec( pod: SparkPod, @@ -24,8 +24,9 @@ private[spark] case class KubernetesDriverSpec( systemProperties: Map[String, String]) private[spark] object KubernetesDriverSpec { - def initialSpec(initialProps: Map[String, String]): KubernetesDriverSpec = KubernetesDriverSpec( - SparkPod.initialPod(), - Seq.empty, - initialProps) + def initialSpec(initialConf: KubernetesConf[KubernetesDriverSpecificConf]): KubernetesDriverSpec = + KubernetesDriverSpec( + SparkPod.initialPod(), + Seq.empty, + initialConf.sparkConf.getAll.toMap) } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala index 9398faee2ea5c..241c2dc9442f1 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala @@ -16,18 +16,18 @@ */ package org.apache.spark.deploy.k8s.submit -import java.io.StringWriter +import java.io.{File, StringWriter} import java.util.{Collections, UUID} import java.util.Properties import io.fabric8.kubernetes.api.model._ import io.fabric8.kubernetes.client.KubernetesClient + import scala.collection.mutable import scala.util.control.NonFatal - import org.apache.spark.SparkConf import org.apache.spark.deploy.SparkApplication -import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverSpecificConf, KubernetesUtils, SparkKubernetesClientFactory} +import org.apache.spark.deploy.k8s._ import org.apache.spark.deploy.k8s.Config._ import org.apache.spark.deploy.k8s.Constants._ import org.apache.spark.internal.Logging @@ -96,7 +96,6 @@ private[spark] object ClientArguments { * @param watcher a watcher that monitors and logs the application status */ private[spark] class Client( - builder: KubernetesDriverBuilder, kubernetesConf: KubernetesConf[KubernetesDriverSpecificConf], kubernetesClient: KubernetesClient, waitForAppCompletion: Boolean, @@ -105,6 +104,15 @@ private[spark] class Client( kubernetesResourceNamePrefix: String) extends Logging { def run(): Unit = { + + val builder = kubernetesConf.sparkConf.get(KUBERNETES_DRIVER_PODTEMPLATE_FILE) + .map(new File(_)) + .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => + KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod( + kubernetesClient.pods().load(file).get(), + new ContainerBuilder().build() + )))) + .getOrElse(new KubernetesDriverBuilder()) val resolvedDriverSpec = builder.buildFromFeatures(kubernetesConf) val configMapName = s"$kubernetesResourceNamePrefix-driver-conf-map" val configMap = buildConfigMap(configMapName, resolvedDriverSpec.systemProperties) @@ -224,7 +232,6 @@ private[spark] class KubernetesClientApplication extends SparkApplication { clientArguments.mainClass, clientArguments.driverArgs, clientArguments.maybePyFiles) - val builder = new KubernetesDriverBuilder val namespace = kubernetesConf.namespace() // The master URL has been checked for validity already in SparkSubmit. // We just need to get rid of the "k8s://" prefix here. @@ -241,7 +248,6 @@ private[spark] class KubernetesClientApplication extends SparkApplication { None, None)) { kubernetesClient => val client = new Client( - builder, kubernetesConf, kubernetesClient, waitForAppCompletion, diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index 7208e3d377593..f9c7d055061df 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -47,7 +47,10 @@ private[spark] class KubernetesDriverBuilder( providePythonStep: ( KubernetesConf[KubernetesDriverSpecificConf] => PythonDriverFeatureStep) = - new PythonDriverFeatureStep(_)) { + new PythonDriverFeatureStep(_), + provideInitialSpec: KubernetesConf[KubernetesDriverSpecificConf] + => KubernetesDriverSpec = + KubernetesDriverSpec.initialSpec) { def buildFromFeatures( kubernetesConf: KubernetesConf[KubernetesDriverSpecificConf]): KubernetesDriverSpec = { @@ -77,7 +80,7 @@ private[spark] class KubernetesDriverBuilder( val allFeatures = (baseFeatures :+ bindingsStep) ++ secretFeature ++ envSecretFeature ++ volumesFeature - var spec = KubernetesDriverSpec.initialSpec(kubernetesConf.sparkConf.getAll.toMap) + var spec = provideInitialSpec() for (feature <- allFeatures) { val configuredPod = feature.configurePod(spec.pod) val addedSystemProperties = feature.getAdditionalPodSystemProperties() From 4f088dbacdfa34593a4d2dcccb03bb32ad58b0f7 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 17 Aug 2018 16:09:20 +0100 Subject: [PATCH 03/53] start from template for executor --- .../k8s/submit/KubernetesClientApplication.scala | 2 +- .../cluster/k8s/KubernetesClusterManager.scala | 15 +++++++++++---- .../cluster/k8s/KubernetesExecutorBuilder.scala | 5 +++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala index 241c2dc9442f1..c8cbba7d32752 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala @@ -110,7 +110,7 @@ private[spark] class Client( .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod( kubernetesClient.pods().load(file).get(), - new ContainerBuilder().build() + new ContainerBuilder().build() // TODO(osatici): infer container from pod )))) .getOrElse(new KubernetesDriverBuilder()) val resolvedDriverSpec = builder.buildFromFeatures(kubernetesConf) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index 9999c62c878df..18865bd6a8ceb 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -20,10 +20,10 @@ import java.io.File import java.util.concurrent.TimeUnit import com.google.common.cache.CacheBuilder +import io.fabric8.kubernetes.api.model.ContainerBuilder import io.fabric8.kubernetes.client.Config - import org.apache.spark.SparkContext -import org.apache.spark.deploy.k8s.{KubernetesUtils, SparkKubernetesClientFactory} +import org.apache.spark.deploy.k8s.{KubernetesUtils, SparkKubernetesClientFactory, SparkPod} import org.apache.spark.deploy.k8s.Config._ import org.apache.spark.deploy.k8s.Constants._ import org.apache.spark.internal.Logging @@ -79,15 +79,22 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit val removedExecutorsCache = CacheBuilder.newBuilder() .expireAfterWrite(3, TimeUnit.MINUTES) .build[java.lang.Long, java.lang.Long]() + val builder = sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) + .map(new File(_)) + .map(file => new KubernetesExecutorBuilder(provideInitialPod = () => + SparkPod( + kubernetesClient.pods().load(file).get(), + new ContainerBuilder().build()))) // TODO(osatici): infer container from pod + .getOrElse(new KubernetesExecutorBuilder()()) val executorPodsLifecycleEventHandler = new ExecutorPodsLifecycleManager( sc.conf, - new KubernetesExecutorBuilder(), + builder, kubernetesClient, snapshotsStore, removedExecutorsCache) val executorPodsAllocator = new ExecutorPodsAllocator( - sc.conf, new KubernetesExecutorBuilder(), kubernetesClient, snapshotsStore, new SystemClock()) + sc.conf, builder, kubernetesClient, snapshotsStore, new SystemClock()) val podsWatchEventSource = new ExecutorPodsWatchSnapshotSource( snapshotsStore, diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala index 364b6fb367722..628d3652509dd 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala @@ -35,7 +35,8 @@ private[spark] class KubernetesExecutorBuilder( new LocalDirsFeatureStep(_), provideVolumesStep: (KubernetesConf[_ <: KubernetesRoleSpecificConf] => MountVolumesFeatureStep) = - new MountVolumesFeatureStep(_)) { + new MountVolumesFeatureStep(_), + provideInitialPod: () => SparkPod = SparkPod.initialPod) { def buildFromFeatures( kubernetesConf: KubernetesConf[KubernetesExecutorSpecificConf]): SparkPod = { @@ -53,7 +54,7 @@ private[spark] class KubernetesExecutorBuilder( val allFeatures = baseFeatures ++ secretFeature ++ secretEnvFeature ++ volumesFeature - var executorPod = SparkPod.initialPod() + var executorPod = provideInitialPod() for (feature <- allFeatures) { executorPod = feature.configurePod(executorPod) } From f2f9a4410cc4907f4c935bc50e96887f687d38bd Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 17 Aug 2018 16:34:48 +0100 Subject: [PATCH 04/53] wip --- .../k8s/features/TemplateVolumeStep.scala | 41 +++++++++++++++++++ .../k8s/KubernetesClusterManager.scala | 1 + 2 files changed, 42 insertions(+) create mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala new file mode 100644 index 0000000000000..4ee2defedfa54 --- /dev/null +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.features + +import io.fabric8.kubernetes.api.model.{HasMetadata, PodBuilder} +import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesRoleSpecificConf, SparkPod} + +private[spark] class TemplateVolumeStep( + conf: KubernetesConf[_ <: KubernetesRoleSpecificConf]) + extends KubernetesFeatureConfigStep { + def configurePod(pod: SparkPod): SparkPod = { + val podWithVolume = new PodBuilder(pod.pod) + .editSpec() + .addNewVolume() + // TODO(osatici): create the volume for the executor podspec template here + .endVolume() + .endSpec() + .build() + SparkPod(podWithVolume, pod.container) + } + + def getAdditionalPodSystemProperties(): Map[String, String] = { + // TODO(osatici): remap executor podspec template config to point to the mount created above + } + + def getAdditionalKubernetesResources(): Seq[HasMetadata] = Seq.empty +} diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index 18865bd6a8ceb..190d996cb2495 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit import com.google.common.cache.CacheBuilder import io.fabric8.kubernetes.api.model.ContainerBuilder import io.fabric8.kubernetes.client.Config + import org.apache.spark.SparkContext import org.apache.spark.deploy.k8s.{KubernetesUtils, SparkKubernetesClientFactory, SparkPod} import org.apache.spark.deploy.k8s.Config._ From 368d0a4b73b9d6d3f4b5a30bb77d7bdd1b46c689 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 17 Aug 2018 21:57:35 +0100 Subject: [PATCH 05/53] volume executor podspec template --- .../apache/spark/deploy/k8s/Constants.scala | 6 +++++ .../k8s/features/TemplateVolumeStep.scala | 24 +++++++++++++------ .../k8s/KubernetesExecutorBuilder.scala | 12 ++++++++-- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala index f82cd7fd02e12..e578366e74a55 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala @@ -72,6 +72,12 @@ private[spark] object Constants { val ENV_PYSPARK_ARGS = "PYSPARK_APP_ARGS" val ENV_PYSPARK_MAJOR_PYTHON_VERSION = "PYSPARK_MAJOR_PYTHON_VERSION" + // Pod spec templates + val EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME = "podSpecTemplate.yml" + val EXECUTOR_POD_SPEC_TEMPLATE_FILE = + s"$SPARK_CONF_DIR_INTERNAL/$EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME" + val POD_TEMPLATE_VOLUME = "podspec-volume" + // Miscellaneous val KUBERNETES_MASTER_INTERNAL_URL = "https://kubernetes.default.svc" val DRIVER_CONTAINER_NAME = "spark-kubernetes-driver" diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala index 4ee2defedfa54..cfdb0537aee07 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala @@ -16,26 +16,36 @@ */ package org.apache.spark.deploy.k8s.features -import io.fabric8.kubernetes.api.model.{HasMetadata, PodBuilder} -import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesRoleSpecificConf, SparkPod} +import io.fabric8.kubernetes.api.model.{Config => _, _} + +import org.apache.spark.deploy.k8s._ private[spark] class TemplateVolumeStep( conf: KubernetesConf[_ <: KubernetesRoleSpecificConf]) extends KubernetesFeatureConfigStep { def configurePod(pod: SparkPod): SparkPod = { + require(conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) + val podTemplateFile = conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get val podWithVolume = new PodBuilder(pod.pod) .editSpec() .addNewVolume() - // TODO(osatici): create the volume for the executor podspec template here + .withName(Constants.POD_TEMPLATE_VOLUME) + .withHostPath(new HostPathVolumeSource(podTemplateFile)) .endVolume() .endSpec() .build() - SparkPod(podWithVolume, pod.container) - } - def getAdditionalPodSystemProperties(): Map[String, String] = { - // TODO(osatici): remap executor podspec template config to point to the mount created above + val containerWithVolume = new ContainerBuilder(pod.container) + .withVolumeMounts(new VolumeMountBuilder() + .withName(Constants.POD_TEMPLATE_VOLUME) + .withMountPath(Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE) + .build()) + .build() + SparkPod(podWithVolume, containerWithVolume) } + def getAdditionalPodSystemProperties(): Map[String, String] = Map[String, String]( + Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE.key -> Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE) + def getAdditionalKubernetesResources(): Seq[HasMetadata] = Seq.empty } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala index 628d3652509dd..b0026c039aa1d 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.scheduler.cluster.k8s -import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesExecutorSpecificConf, KubernetesRoleSpecificConf, SparkPod} +import org.apache.spark.deploy.k8s._ import org.apache.spark.deploy.k8s.features._ import org.apache.spark.deploy.k8s.features.{BasicExecutorFeatureStep, EnvSecretsFeatureStep, LocalDirsFeatureStep, MountSecretsFeatureStep} @@ -36,6 +36,9 @@ private[spark] class KubernetesExecutorBuilder( provideVolumesStep: (KubernetesConf[_ <: KubernetesRoleSpecificConf] => MountVolumesFeatureStep) = new MountVolumesFeatureStep(_), + provideTemplateVolumeStep: (KubernetesConf[_ <: KubernetesRoleSpecificConf] + => TemplateVolumeStep) = + new TemplateVolumeStep(_), provideInitialPod: () => SparkPod = SparkPod.initialPod) { def buildFromFeatures( @@ -51,8 +54,13 @@ private[spark] class KubernetesExecutorBuilder( val volumesFeature = if (kubernetesConf.roleVolumes.nonEmpty) { Seq(provideVolumesStep(kubernetesConf)) } else Nil + val templateVolumeStep = if ( + kubernetesConf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) { + Seq(provideTemplateVolumeStep(kubernetesConf)) + } else Nil - val allFeatures = baseFeatures ++ secretFeature ++ secretEnvFeature ++ volumesFeature + val allFeatures = + baseFeatures ++ secretFeature ++ secretEnvFeature ++ volumesFeature ++ templateVolumeStep var executorPod = provideInitialPod() for (feature <- allFeatures) { From 0005ea5b69faccc20df20faccb6909429b87f736 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 17 Aug 2018 22:18:08 +0100 Subject: [PATCH 06/53] move logic to apply functions --- .../submit/KubernetesClientApplication.scala | 12 ++--------- .../k8s/submit/KubernetesDriverBuilder.scala | 21 ++++++++++++++++++- .../k8s/KubernetesClusterManager.scala | 15 ++++++------- .../k8s/KubernetesExecutorBuilder.scala | 19 +++++++++++++++++ 4 files changed, 47 insertions(+), 20 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala index c8cbba7d32752..2e8eada70ceaf 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala @@ -104,16 +104,8 @@ private[spark] class Client( kubernetesResourceNamePrefix: String) extends Logging { def run(): Unit = { - - val builder = kubernetesConf.sparkConf.get(KUBERNETES_DRIVER_PODTEMPLATE_FILE) - .map(new File(_)) - .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => - KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod( - kubernetesClient.pods().load(file).get(), - new ContainerBuilder().build() // TODO(osatici): infer container from pod - )))) - .getOrElse(new KubernetesDriverBuilder()) - val resolvedDriverSpec = builder.buildFromFeatures(kubernetesConf) + val resolvedDriverSpec = KubernetesDriverBuilder(kubernetesClient, kubernetesConf.sparkConf) + .buildFromFeatures(kubernetesConf) val configMapName = s"$kubernetesResourceNamePrefix-driver-conf-map" val configMap = buildConfigMap(configMapName, resolvedDriverSpec.systemProperties) // The include of the ENV_VAR for "SPARK_CONF_DIR" is to allow for the diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index f9c7d055061df..012e039c20228 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -16,7 +16,13 @@ */ package org.apache.spark.deploy.k8s.submit -import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverSpec, KubernetesDriverSpecificConf, KubernetesRoleSpecificConf} +import java.io.File + +import io.fabric8.kubernetes.api.model.ContainerBuilder +import io.fabric8.kubernetes.client.KubernetesClient + +import org.apache.spark.SparkConf +import org.apache.spark.deploy.k8s._ import org.apache.spark.deploy.k8s.features._ import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep} @@ -93,3 +99,16 @@ private[spark] class KubernetesDriverBuilder( spec } } + +private[spark] object KubernetesDriverBuilder { + def apply(kubernetesClient: KubernetesClient, conf: SparkConf): KubernetesDriverBuilder = { + conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE) + .map(new File(_)) + .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => + KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod( + kubernetesClient.pods().load(file).get(), + new ContainerBuilder().build() // TODO(osatici): infer container from pod + )))) + .getOrElse(new KubernetesDriverBuilder()) + } +} diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index 190d996cb2495..1fdfd01177417 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -80,22 +80,19 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit val removedExecutorsCache = CacheBuilder.newBuilder() .expireAfterWrite(3, TimeUnit.MINUTES) .build[java.lang.Long, java.lang.Long]() - val builder = sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) - .map(new File(_)) - .map(file => new KubernetesExecutorBuilder(provideInitialPod = () => - SparkPod( - kubernetesClient.pods().load(file).get(), - new ContainerBuilder().build()))) // TODO(osatici): infer container from pod - .getOrElse(new KubernetesExecutorBuilder()()) val executorPodsLifecycleEventHandler = new ExecutorPodsLifecycleManager( sc.conf, - builder, + KubernetesExecutorBuilder(kubernetesClient, sc.conf), kubernetesClient, snapshotsStore, removedExecutorsCache) val executorPodsAllocator = new ExecutorPodsAllocator( - sc.conf, builder, kubernetesClient, snapshotsStore, new SystemClock()) + sc.conf, + KubernetesExecutorBuilder(kubernetesClient, sc.conf), + kubernetesClient, + snapshotsStore, + new SystemClock()) val podsWatchEventSource = new ExecutorPodsWatchSnapshotSource( snapshotsStore, diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala index b0026c039aa1d..833ea4ab88430 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala @@ -16,6 +16,12 @@ */ package org.apache.spark.scheduler.cluster.k8s +import java.io.File + +import io.fabric8.kubernetes.api.model.ContainerBuilder +import io.fabric8.kubernetes.client.KubernetesClient + +import org.apache.spark.SparkConf import org.apache.spark.deploy.k8s._ import org.apache.spark.deploy.k8s.features._ import org.apache.spark.deploy.k8s.features.{BasicExecutorFeatureStep, EnvSecretsFeatureStep, LocalDirsFeatureStep, MountSecretsFeatureStep} @@ -69,3 +75,16 @@ private[spark] class KubernetesExecutorBuilder( executorPod } } + +private[spark] object KubernetesExecutorBuilder { + def apply(kubernetesClient: KubernetesClient, conf: SparkConf): KubernetesExecutorBuilder = { + conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) + .map(new File(_)) + .map(file => new KubernetesExecutorBuilder(provideInitialPod = () => { + val pod = kubernetesClient.pods().load(file).get() + // TODO(osatici) find container + SparkPod(pod, new ContainerBuilder().build()) + })) + .getOrElse(new KubernetesExecutorBuilder()) + } +} From dda5cc9429eb3c92d1c12d41aafc0781e48fdbd8 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 17 Aug 2018 22:30:39 +0100 Subject: [PATCH 07/53] find containers --- .../org/apache/spark/deploy/k8s/Constants.scala | 1 + .../k8s/features/BasicExecutorFeatureStep.scala | 2 +- .../deploy/k8s/submit/KubernetesDriverBuilder.scala | 13 ++++++++----- .../cluster/k8s/KubernetesExecutorBuilder.scala | 7 +++++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala index e578366e74a55..cb73358158892 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala @@ -81,5 +81,6 @@ private[spark] object Constants { // Miscellaneous val KUBERNETES_MASTER_INTERNAL_URL = "https://kubernetes.default.svc" val DRIVER_CONTAINER_NAME = "spark-kubernetes-driver" + val EXECUTOR_CONTAINER_NAME = "executor" val MEMORY_OVERHEAD_MIN_MIB = 384L } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala index c37f713c56de1..d413db4000bed 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala @@ -129,7 +129,7 @@ private[spark] class BasicExecutorFeatureStep( } val executorContainer = new ContainerBuilder(pod.container) - .withName("executor") + .withName(Constants.EXECUTOR_CONTAINER_NAME) .withImage(executorContainerImage) .withImagePullPolicy(kubernetesConf.imagePullPolicy()) .withNewResources() diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index 012e039c20228..5575dafa73c3f 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -104,11 +104,14 @@ private[spark] object KubernetesDriverBuilder { def apply(kubernetesClient: KubernetesClient, conf: SparkConf): KubernetesDriverBuilder = { conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE) .map(new File(_)) - .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => - KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod( - kubernetesClient.pods().load(file).get(), - new ContainerBuilder().build() // TODO(osatici): infer container from pod - )))) + .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => { + val pod = kubernetesClient.pods().load(file).get() + val container = pod.getSpec.getContainers.stream() + .filter(_.getName == Constants.DRIVER_CONTAINER_NAME) + .findFirst() + .orElseGet(() => new ContainerBuilder().build()) + KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod(pod, container)) + })) .getOrElse(new KubernetesDriverBuilder()) } } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala index 833ea4ab88430..3969e3047000e 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala @@ -82,8 +82,11 @@ private[spark] object KubernetesExecutorBuilder { .map(new File(_)) .map(file => new KubernetesExecutorBuilder(provideInitialPod = () => { val pod = kubernetesClient.pods().load(file).get() - // TODO(osatici) find container - SparkPod(pod, new ContainerBuilder().build()) + val container = pod.getSpec.getContainers.stream() + .filter(_.getName == Constants.EXECUTOR_CONTAINER_NAME) + .findFirst() + .orElseGet(() => new ContainerBuilder().build()) + SparkPod(pod, container) })) .getOrElse(new KubernetesExecutorBuilder()) } From d0f41aa18318a35fdcf6f2fbdd969ad3d5ad04da Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 17 Aug 2018 22:33:34 +0100 Subject: [PATCH 08/53] style --- .../spark/deploy/k8s/submit/KubernetesClientApplication.scala | 1 + .../spark/deploy/k8s/submit/KubernetesDriverBuilder.scala | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala index 2e8eada70ceaf..c6311036726be 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala @@ -25,6 +25,7 @@ import io.fabric8.kubernetes.client.KubernetesClient import scala.collection.mutable import scala.util.control.NonFatal + import org.apache.spark.SparkConf import org.apache.spark.deploy.SparkApplication import org.apache.spark.deploy.k8s._ diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index 5575dafa73c3f..e8c87a8e68639 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -109,7 +109,7 @@ private[spark] object KubernetesDriverBuilder { val container = pod.getSpec.getContainers.stream() .filter(_.getName == Constants.DRIVER_CONTAINER_NAME) .findFirst() - .orElseGet(() => new ContainerBuilder().build()) + .orElseGet(() => new ContainerBuilder().build()) KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod(pod, container)) })) .getOrElse(new KubernetesDriverBuilder()) From c4c1231f4c1545a01243c4f3c4d195af9dacee43 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 17 Aug 2018 22:33:54 +0100 Subject: [PATCH 09/53] remove import --- .../spark/deploy/k8s/submit/KubernetesClientApplication.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala index c6311036726be..306be8901afdc 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.deploy.k8s.submit -import java.io.{File, StringWriter} +import java.io.StringWriter import java.util.{Collections, UUID} import java.util.Properties From 74de0e5f793562c11f3fb23f610fd139d34f5661 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Tue, 21 Aug 2018 16:50:27 +0100 Subject: [PATCH 10/53] compiles --- .../submit/KubernetesClientApplication.scala | 4 +--- .../k8s/submit/KubernetesDriverBuilder.scala | 18 +++++++++++++----- .../k8s/KubernetesExecutorBuilder.scala | 16 +++++----------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala index 306be8901afdc..3cecfeea12bc8 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala @@ -17,12 +17,10 @@ package org.apache.spark.deploy.k8s.submit import java.io.StringWriter -import java.util.{Collections, UUID} -import java.util.Properties +import java.util.{Collections, Properties, UUID} import io.fabric8.kubernetes.api.model._ import io.fabric8.kubernetes.client.KubernetesClient - import scala.collection.mutable import scala.util.control.NonFatal diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index e8c87a8e68639..3d3659a93a52e 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -20,6 +20,7 @@ import java.io.File import io.fabric8.kubernetes.api.model.ContainerBuilder import io.fabric8.kubernetes.client.KubernetesClient +import scala.collection.JavaConverters._ import org.apache.spark.SparkConf import org.apache.spark.deploy.k8s._ @@ -54,6 +55,9 @@ private[spark] class KubernetesDriverBuilder( KubernetesConf[KubernetesDriverSpecificConf] => PythonDriverFeatureStep) = new PythonDriverFeatureStep(_), + provideTemplateVolumeStep: (KubernetesConf[_ <: KubernetesRoleSpecificConf] + => TemplateVolumeStep) = + new TemplateVolumeStep(_), provideInitialSpec: KubernetesConf[KubernetesDriverSpecificConf] => KubernetesDriverSpec = KubernetesDriverSpec.initialSpec) { @@ -75,6 +79,10 @@ private[spark] class KubernetesDriverBuilder( val volumesFeature = if (kubernetesConf.roleVolumes.nonEmpty) { Seq(provideVolumesStep(kubernetesConf)) } else Nil + val templateVolumeFeature = if ( + kubernetesConf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) { + Seq(provideTemplateVolumeStep(kubernetesConf)) + } else Nil val bindingsStep = kubernetesConf.roleSpecificConf.mainAppResource.map { case JavaMainAppResource(_) => @@ -84,9 +92,9 @@ private[spark] class KubernetesDriverBuilder( .getOrElse(provideJavaStep(kubernetesConf)) val allFeatures = (baseFeatures :+ bindingsStep) ++ - secretFeature ++ envSecretFeature ++ volumesFeature + secretFeature ++ envSecretFeature ++ volumesFeature ++ templateVolumeFeature - var spec = provideInitialSpec() + var spec = provideInitialSpec(kubernetesConf) for (feature <- allFeatures) { val configuredPod = feature.configurePod(spec.pod) val addedSystemProperties = feature.getAdditionalPodSystemProperties() @@ -106,10 +114,10 @@ private[spark] object KubernetesDriverBuilder { .map(new File(_)) .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => { val pod = kubernetesClient.pods().load(file).get() - val container = pod.getSpec.getContainers.stream() + val container = pod.getSpec.getContainers.asScala .filter(_.getName == Constants.DRIVER_CONTAINER_NAME) - .findFirst() - .orElseGet(() => new ContainerBuilder().build()) + .headOption + .getOrElse(new ContainerBuilder().build()) KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod(pod, container)) })) .getOrElse(new KubernetesDriverBuilder()) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala index 3969e3047000e..44df7f05ccb1e 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala @@ -20,6 +20,7 @@ import java.io.File import io.fabric8.kubernetes.api.model.ContainerBuilder import io.fabric8.kubernetes.client.KubernetesClient +import scala.collection.JavaConverters._ import org.apache.spark.SparkConf import org.apache.spark.deploy.k8s._ @@ -42,9 +43,6 @@ private[spark] class KubernetesExecutorBuilder( provideVolumesStep: (KubernetesConf[_ <: KubernetesRoleSpecificConf] => MountVolumesFeatureStep) = new MountVolumesFeatureStep(_), - provideTemplateVolumeStep: (KubernetesConf[_ <: KubernetesRoleSpecificConf] - => TemplateVolumeStep) = - new TemplateVolumeStep(_), provideInitialPod: () => SparkPod = SparkPod.initialPod) { def buildFromFeatures( @@ -60,13 +58,9 @@ private[spark] class KubernetesExecutorBuilder( val volumesFeature = if (kubernetesConf.roleVolumes.nonEmpty) { Seq(provideVolumesStep(kubernetesConf)) } else Nil - val templateVolumeStep = if ( - kubernetesConf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) { - Seq(provideTemplateVolumeStep(kubernetesConf)) - } else Nil val allFeatures = - baseFeatures ++ secretFeature ++ secretEnvFeature ++ volumesFeature ++ templateVolumeStep + baseFeatures ++ secretFeature ++ secretEnvFeature ++ volumesFeature var executorPod = provideInitialPod() for (feature <- allFeatures) { @@ -82,10 +76,10 @@ private[spark] object KubernetesExecutorBuilder { .map(new File(_)) .map(file => new KubernetesExecutorBuilder(provideInitialPod = () => { val pod = kubernetesClient.pods().load(file).get() - val container = pod.getSpec.getContainers.stream() + val container = pod.getSpec.getContainers.asScala .filter(_.getName == Constants.EXECUTOR_CONTAINER_NAME) - .findFirst() - .orElseGet(() => new ContainerBuilder().build()) + .headOption + .getOrElse(new ContainerBuilder().build()) SparkPod(pod, container) })) .getOrElse(new KubernetesExecutorBuilder()) From 205ddd39170e52ff336a7da53ef94cc50fae876d Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Tue, 21 Aug 2018 18:08:51 +0100 Subject: [PATCH 11/53] tests pass --- .../deploy/k8s/submit/KubernetesClientApplication.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala index 3cecfeea12bc8..8f30bce7319ea 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala @@ -95,6 +95,7 @@ private[spark] object ClientArguments { * @param watcher a watcher that monitors and logs the application status */ private[spark] class Client( + builder: KubernetesDriverBuilder, kubernetesConf: KubernetesConf[KubernetesDriverSpecificConf], kubernetesClient: KubernetesClient, waitForAppCompletion: Boolean, @@ -103,8 +104,7 @@ private[spark] class Client( kubernetesResourceNamePrefix: String) extends Logging { def run(): Unit = { - val resolvedDriverSpec = KubernetesDriverBuilder(kubernetesClient, kubernetesConf.sparkConf) - .buildFromFeatures(kubernetesConf) + val resolvedDriverSpec = builder.buildFromFeatures(kubernetesConf) val configMapName = s"$kubernetesResourceNamePrefix-driver-conf-map" val configMap = buildConfigMap(configMapName, resolvedDriverSpec.systemProperties) // The include of the ENV_VAR for "SPARK_CONF_DIR" is to allow for the @@ -239,6 +239,7 @@ private[spark] class KubernetesClientApplication extends SparkApplication { None, None)) { kubernetesClient => val client = new Client( + KubernetesDriverBuilder(kubernetesClient, kubernetesConf.sparkConf), kubernetesConf, kubernetesClient, waitForAppCompletion, From 4ae6fc6fabcab6974f0460e8a54a14407eee32e6 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Tue, 21 Aug 2018 18:09:01 +0100 Subject: [PATCH 12/53] adding TemplateVolumeStepSuite --- .../features/TemplateVolumeStepSuite.scala | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStepSuite.scala diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStepSuite.scala new file mode 100644 index 0000000000000..6ed6723c343fc --- /dev/null +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStepSuite.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.features + +import org.mockito.Mockito +import org.scalatest.BeforeAndAfter + +import org.apache.spark.{SparkConf, SparkFunSuite} +import org.apache.spark.deploy.k8s._ + +class TemplateVolumeStepSuite extends SparkFunSuite with BeforeAndAfter { + private val podTemplateLocalFile = "/path/to/file" + private var sparkConf: SparkConf = _ + private var kubernetesConf : KubernetesConf[_ <: KubernetesRoleSpecificConf] = _ + + before { + sparkConf = Mockito.mock(classOf[SparkConf]) + kubernetesConf = KubernetesConf( + sparkConf, + KubernetesDriverSpecificConf( + None, + "app-name", + "main", + Seq.empty), + "resource", + "app-id", + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Nil, + Seq.empty[String]) + } + + test("Mounts executor template volume if config specified") { + Mockito.doReturn(Option(podTemplateLocalFile)).when(sparkConf) + .get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) + val step = new TemplateVolumeStep(kubernetesConf) + val configuredPod = step.configurePod(SparkPod.initialPod()) + + assert(configuredPod.pod.getSpec.getVolumes.size() === 1) + assert(configuredPod.pod.getSpec.getVolumes.get(0).getName === Constants.POD_TEMPLATE_VOLUME) + assert(configuredPod.pod.getSpec.getVolumes.get(0).getHostPath.getPath === "/path/to/file") + assert(configuredPod.container.getVolumeMounts.size() === 1) + assert(configuredPod.container.getVolumeMounts.get(0).getName === Constants.POD_TEMPLATE_VOLUME) + assert(configuredPod.container.getVolumeMounts.get(0).getMountPath === + Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE) + } +} From c0bcfeaaa3e44b8fe9da0a287f9db995a1848c93 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Tue, 21 Aug 2018 22:30:32 +0100 Subject: [PATCH 13/53] DriverBuilder test --- .../submit/KubernetesDriverBuilderSuite.scala | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala index 046e578b94629..16e8b27611132 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.deploy.k8s.submit +import org.mockito.Mockito + import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.deploy.k8s._ import org.apache.spark.deploy.k8s.features._ @@ -33,6 +35,7 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { private val PYSPARK_STEP_TYPE = "pyspark-bindings" private val ENV_SECRETS_STEP_TYPE = "env-secrets" private val MOUNT_VOLUMES_STEP_TYPE = "mount-volumes" + private val TEMPLATE_VOLUME_STEP_TYPE = "template-volume" private val basicFeatureStep = KubernetesFeaturesTestUtils.getMockConfigStepForStepType( BASIC_STEP_TYPE, classOf[BasicDriverFeatureStep]) @@ -61,6 +64,10 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { private val mountVolumesStep = KubernetesFeaturesTestUtils.getMockConfigStepForStepType( MOUNT_VOLUMES_STEP_TYPE, classOf[MountVolumesFeatureStep]) + private val templateVolumeStep = KubernetesFeaturesTestUtils.getMockConfigStepForStepType( + TEMPLATE_VOLUME_STEP_TYPE, classOf[TemplateVolumeStep] + ) + private val builderUnderTest: KubernetesDriverBuilder = new KubernetesDriverBuilder( _ => basicFeatureStep, @@ -71,7 +78,8 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { _ => localDirsStep, _ => mountVolumesStep, _ => javaStep, - _ => pythonStep) + _ => pythonStep, + _ => templateVolumeStep) test("Apply fundamental steps all the time.") { val conf = KubernetesConf( @@ -211,6 +219,36 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { JAVA_STEP_TYPE) } + test("Apply template volume step if executor template is present.") { + val sparkConf = Mockito.spy(new SparkConf(false)) + Mockito.doReturn(Option("filename")).when(sparkConf) + .get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) + val conf = KubernetesConf( + sparkConf, + KubernetesDriverSpecificConf( + Some(JavaMainAppResource("example.jar")), + "test-app", + "main", + Seq.empty), + "prefix", + "appId", + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Nil, + Seq.empty[String]) + validateStepTypesApplied( + builderUnderTest.buildFromFeatures(conf), + BASIC_STEP_TYPE, + CREDENTIALS_STEP_TYPE, + SERVICE_STEP_TYPE, + LOCAL_DIRS_STEP_TYPE, + JAVA_STEP_TYPE, + TEMPLATE_VOLUME_STEP_TYPE) + } + private def validateStepTypesApplied(resolvedSpec: KubernetesDriverSpec, stepTypes: String*) : Unit = { From b9e426377552a99d695b49c7568acf9b7d3d87ac Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Wed, 22 Aug 2018 00:15:41 +0100 Subject: [PATCH 14/53] WIP trying to write tests for KubernetesDriverBuilder constructor --- .../deploy/k8s/KubernetesDriverSpec.scala | 2 +- .../k8s/submit/KubernetesDriverBuilder.scala | 33 +++++--- .../submit/KubernetesDriverBuilderSuite.scala | 82 ++++++++++++++++++- 3 files changed, 99 insertions(+), 18 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala index 94515f73e389e..2deb4e1ccb110 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.deploy.k8s -import io.fabric8.kubernetes.api.model.{HasMetadata, Pod} +import io.fabric8.kubernetes.api.model.HasMetadata private[spark] case class KubernetesDriverSpec( pod: SparkPod, diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index 3d3659a93a52e..ea1c45a96361f 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -26,8 +26,9 @@ import org.apache.spark.SparkConf import org.apache.spark.deploy.k8s._ import org.apache.spark.deploy.k8s.features._ import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep} +import org.apache.spark.internal.Logging -private[spark] class KubernetesDriverBuilder( +private[spark] class KubernetesDriverBuilder ( provideBasicStep: (KubernetesConf[KubernetesDriverSpecificConf]) => BasicDriverFeatureStep = new BasicDriverFeatureStep(_), provideCredentialsStep: (KubernetesConf[KubernetesDriverSpecificConf]) @@ -108,18 +109,24 @@ private[spark] class KubernetesDriverBuilder( } } -private[spark] object KubernetesDriverBuilder { +private[spark] object KubernetesDriverBuilder extends Logging { def apply(kubernetesClient: KubernetesClient, conf: SparkConf): KubernetesDriverBuilder = { - conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE) - .map(new File(_)) - .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => { - val pod = kubernetesClient.pods().load(file).get() - val container = pod.getSpec.getContainers.asScala - .filter(_.getName == Constants.DRIVER_CONTAINER_NAME) - .headOption - .getOrElse(new ContainerBuilder().build()) - KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod(pod, container)) - })) - .getOrElse(new KubernetesDriverBuilder()) + try { + conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE) + .map(new File(_)) + .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => { + val pod = kubernetesClient.pods().load(file).get() + val container = pod.getSpec.getContainers.asScala + .filter(_.getName == Constants.DRIVER_CONTAINER_NAME) + .headOption + .getOrElse(new ContainerBuilder().build()) + KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod(pod, container)) + })) + .getOrElse(new KubernetesDriverBuilder()) + } catch { + case e: Exception => + logWarning(s"Encountered exception while attempting to load initial pod spec from file", e) + new KubernetesDriverBuilder() + } } } diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala index 16e8b27611132..617cc185f3743 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala @@ -16,10 +16,17 @@ */ package org.apache.spark.deploy.k8s.submit -import org.mockito.Mockito +import java.io.File + +import io.fabric8.kubernetes.api.model.{DoneablePod, Pod, PodBuilder, PodList} +import io.fabric8.kubernetes.client.KubernetesClient +import io.fabric8.kubernetes.client.dsl.{MixedOperation, PodResource} +import org.mockito.Matchers._ +import org.mockito.Mockito._ import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.deploy.k8s._ +import org.apache.spark.deploy.k8s.Config.CONTAINER_IMAGE import org.apache.spark.deploy.k8s.features._ import org.apache.spark.deploy.k8s.features.{BasicDriverFeatureStep, DriverKubernetesCredentialsFeatureStep, DriverServiceFeatureStep, EnvSecretsFeatureStep, KubernetesFeaturesTestUtils, LocalDirsFeatureStep, MountSecretsFeatureStep} import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep} @@ -220,8 +227,8 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { } test("Apply template volume step if executor template is present.") { - val sparkConf = Mockito.spy(new SparkConf(false)) - Mockito.doReturn(Option("filename")).when(sparkConf) + val sparkConf = spy(new SparkConf(false)) + doReturn(Option("filename")).when(sparkConf) .get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) val conf = KubernetesConf( sparkConf, @@ -249,7 +256,6 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { TEMPLATE_VOLUME_STEP_TYPE) } - private def validateStepTypesApplied(resolvedSpec: KubernetesDriverSpec, stepTypes: String*) : Unit = { assert(resolvedSpec.systemProperties.size === stepTypes.size) @@ -260,4 +266,72 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { assert(resolvedSpec.systemProperties(stepType) === stepType) } } + + test("Start with empty pod if template is not specified") { + val kubernetesClient = mock(classOf[KubernetesClient]) + val driverBuilder = KubernetesDriverBuilder.apply(kubernetesClient, new SparkConf()) + verify(kubernetesClient, never()).pods() + } + + test("Starts with template if specified") { + val spec = getSpecWithPodTemplate( + new PodBuilder() + .withNewMetadata() + .addToLabels("test-label-key", "test-label-value") + .endMetadata() + .withNewSpec() + .addNewContainer() + .withName(Constants.DRIVER_CONTAINER_NAME) + .endContainer() + .endSpec() + .build()) + + assert(spec.pod.pod.getMetadata.getLabels.containsKey("test-label-key")) + assert(spec.pod.pod.getMetadata.getLabels.get("test-label-key") === "test-label-value") + assert(spec.pod.container.getName === Constants.DRIVER_CONTAINER_NAME) + } + + test("Starts with empty pod if bad template") { + val spec = getSpecWithPodTemplate( + new PodBuilder() + .withNewMetadata() + .addToLabels("test-label-key", "test-label-value") + .endMetadata() + .build()) + + assert(!spec.pod.pod.getMetadata.getLabels.containsKey("test-label-key")) + } + + private def getSpecWithPodTemplate(pod: Pod) : KubernetesDriverSpec = { + val kubernetesClient = mock(classOf[KubernetesClient]) + val pods = + mock(classOf[MixedOperation[Pod, PodList, DoneablePod, PodResource[Pod, DoneablePod]]]) + val podResource = mock(classOf[PodResource[Pod, DoneablePod]]) + when(kubernetesClient.pods()).thenReturn(pods) + when(pods.load(any(classOf[File]))).thenReturn(podResource) + when(podResource.get()).thenReturn(pod) + + val sparkConf = new SparkConf(false) + .set(CONTAINER_IMAGE, "spark-driver:latest") + .set(Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE, "template-file.yaml") + + val kubernetesConf = new KubernetesConf( + sparkConf, + KubernetesDriverSpecificConf( + Some(JavaMainAppResource("example.jar")), + "test-app", + "main", + Seq.empty), + "prefix", + "appId", + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Nil, + Seq.empty[String]) + + KubernetesDriverBuilder.apply(kubernetesClient, sparkConf).buildFromFeatures(kubernetesConf) + } } From c5e1ea07c3ed45206983d3109748e47f89320c13 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Wed, 22 Aug 2018 00:33:15 +0100 Subject: [PATCH 15/53] fix test --- .../k8s/submit/KubernetesDriverBuilder.scala | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index ea1c45a96361f..c250ad7b40436 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -111,22 +111,23 @@ private[spark] class KubernetesDriverBuilder ( private[spark] object KubernetesDriverBuilder extends Logging { def apply(kubernetesClient: KubernetesClient, conf: SparkConf): KubernetesDriverBuilder = { - try { - conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE) - .map(new File(_)) - .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => { + conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE) + .map(new File(_)) + .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => { + try { val pod = kubernetesClient.pods().load(file).get() val container = pod.getSpec.getContainers.asScala .filter(_.getName == Constants.DRIVER_CONTAINER_NAME) .headOption .getOrElse(new ContainerBuilder().build()) KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod(pod, container)) - })) - .getOrElse(new KubernetesDriverBuilder()) - } catch { - case e: Exception => - logWarning(s"Encountered exception while attempting to load initial pod spec from file", e) - new KubernetesDriverBuilder() - } + } catch { + case e: Exception => + logWarning( + s"Encountered exception while attempting to load initial pod spec from file", e) + KubernetesDriverSpec.initialSpec(conf) + } + })) + .getOrElse(new KubernetesDriverBuilder()) } } From 56a6b3264da76b9d1d214a42cad2bc83a7b76bc9 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Wed, 22 Aug 2018 01:14:31 +0100 Subject: [PATCH 16/53] fix test, and move loading logic to util method --- .../spark/deploy/k8s/KubernetesUtils.scala | 28 +++++++++++++++++-- .../k8s/submit/KubernetesDriverBuilder.scala | 19 ++++++------- .../k8s/KubernetesExecutorBuilder.scala | 12 +++----- .../submit/KubernetesDriverBuilderSuite.scala | 19 +++++++------ 4 files changed, 48 insertions(+), 30 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 588cd9d40f9a0..1719c14e46bf4 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -16,10 +16,16 @@ */ package org.apache.spark.deploy.k8s -import org.apache.spark.SparkConf +import java.io.File + +import io.fabric8.kubernetes.client.KubernetesClient +import scala.collection.JavaConverters._ + +import org.apache.spark.{SparkConf, SparkException} +import org.apache.spark.internal.Logging import org.apache.spark.util.Utils -private[spark] object KubernetesUtils { +private[spark] object KubernetesUtils extends Logging { /** * Extract and parse Spark configuration properties with a given name prefix and @@ -59,5 +65,23 @@ private[spark] object KubernetesUtils { } } + def loadPodFromTemplate(kubernetesClient: KubernetesClient, + templateFile: File, + containerName: String): SparkPod = { + try { + val pod = kubernetesClient.pods().load(templateFile).get() + val container = pod.getSpec.getContainers.asScala + .filter(_.getName == containerName) + .headOption + require(container.isDefined) + SparkPod(pod, container.get) + } catch { + case e: Exception => + logError( + s"Encountered exception while attempting to load initial pod spec from file", e) + throw new SparkException("Could not load driver pod from template file.", e) + } + } + def parseMasterUrl(url: String): String = url.substring("k8s://".length) } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index c250ad7b40436..08a1546a1bfe0 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -18,11 +18,9 @@ package org.apache.spark.deploy.k8s.submit import java.io.File -import io.fabric8.kubernetes.api.model.ContainerBuilder import io.fabric8.kubernetes.client.KubernetesClient -import scala.collection.JavaConverters._ -import org.apache.spark.SparkConf +import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.deploy.k8s._ import org.apache.spark.deploy.k8s.features._ import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep} @@ -115,17 +113,16 @@ private[spark] object KubernetesDriverBuilder extends Logging { .map(new File(_)) .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => { try { - val pod = kubernetesClient.pods().load(file).get() - val container = pod.getSpec.getContainers.asScala - .filter(_.getName == Constants.DRIVER_CONTAINER_NAME) - .headOption - .getOrElse(new ContainerBuilder().build()) - KubernetesDriverSpec.initialSpec(conf).copy(pod = SparkPod(pod, container)) + val sparkPod = KubernetesUtils.loadPodFromTemplate( + kubernetesClient, + file, + Constants.DRIVER_CONTAINER_NAME) + KubernetesDriverSpec.initialSpec(conf).copy(pod = sparkPod) } catch { case e: Exception => - logWarning( + logError( s"Encountered exception while attempting to load initial pod spec from file", e) - KubernetesDriverSpec.initialSpec(conf) + throw new SparkException("Could not load driver pod from template file.", e) } })) .getOrElse(new KubernetesDriverBuilder()) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala index 44df7f05ccb1e..412b66ea0e8af 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala @@ -18,9 +18,7 @@ package org.apache.spark.scheduler.cluster.k8s import java.io.File -import io.fabric8.kubernetes.api.model.ContainerBuilder import io.fabric8.kubernetes.client.KubernetesClient -import scala.collection.JavaConverters._ import org.apache.spark.SparkConf import org.apache.spark.deploy.k8s._ @@ -75,12 +73,10 @@ private[spark] object KubernetesExecutorBuilder { conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) .map(new File(_)) .map(file => new KubernetesExecutorBuilder(provideInitialPod = () => { - val pod = kubernetesClient.pods().load(file).get() - val container = pod.getSpec.getContainers.asScala - .filter(_.getName == Constants.EXECUTOR_CONTAINER_NAME) - .headOption - .getOrElse(new ContainerBuilder().build()) - SparkPod(pod, container) + KubernetesUtils.loadPodFromTemplate( + kubernetesClient, + file, + Constants.EXECUTOR_CONTAINER_NAME) })) .getOrElse(new KubernetesExecutorBuilder()) } diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala index 617cc185f3743..2bafe99e9d3cd 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala @@ -24,7 +24,7 @@ import io.fabric8.kubernetes.client.dsl.{MixedOperation, PodResource} import org.mockito.Matchers._ import org.mockito.Mockito._ -import org.apache.spark.{SparkConf, SparkFunSuite} +import org.apache.spark.{SparkConf, SparkException, SparkFunSuite} import org.apache.spark.deploy.k8s._ import org.apache.spark.deploy.k8s.Config.CONTAINER_IMAGE import org.apache.spark.deploy.k8s.features._ @@ -292,14 +292,15 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { } test("Starts with empty pod if bad template") { - val spec = getSpecWithPodTemplate( - new PodBuilder() - .withNewMetadata() - .addToLabels("test-label-key", "test-label-value") - .endMetadata() - .build()) - - assert(!spec.pod.pod.getMetadata.getLabels.containsKey("test-label-key")) + val exception = intercept[SparkException] { + getSpecWithPodTemplate( + new PodBuilder() + .withNewMetadata() + .addToLabels("test-label-key", "test-label-value") + .endMetadata() + .build()) + } + assert(exception.getMessage.contains("Could not load driver pod from template file.")) } private def getSpecWithPodTemplate(pod: Pod) : KubernetesDriverSpec = { From 7d0d928a5c9038e733c5951a8e0ae8b58b766a1f Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Wed, 22 Aug 2018 01:21:19 +0100 Subject: [PATCH 17/53] validate that the executor pod template is good in the driver --- .../scheduler/cluster/k8s/KubernetesClusterManager.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index 1fdfd01177417..7be8675c3c245 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -24,7 +24,7 @@ import io.fabric8.kubernetes.api.model.ContainerBuilder import io.fabric8.kubernetes.client.Config import org.apache.spark.SparkContext -import org.apache.spark.deploy.k8s.{KubernetesUtils, SparkKubernetesClientFactory, SparkPod} +import org.apache.spark.deploy.k8s.{Constants, KubernetesUtils, SparkKubernetesClientFactory, SparkPod} import org.apache.spark.deploy.k8s.Config._ import org.apache.spark.deploy.k8s.Constants._ import org.apache.spark.internal.Logging @@ -70,6 +70,13 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit defaultServiceAccountToken, defaultServiceAccountCaCrt) + if (sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) { + KubernetesUtils.loadPodFromTemplate( + kubernetesClient, + new File(sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get), + Constants.EXECUTOR_CONTAINER_NAME) + } + val requestExecutorsService = ThreadUtils.newDaemonCachedThreadPool( "kubernetes-executor-requests") From 1da79a823a5c34565f4030920927c02acd7b3d17 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Wed, 22 Aug 2018 01:22:20 +0100 Subject: [PATCH 18/53] cleaning --- .../spark/scheduler/cluster/k8s/KubernetesClusterManager.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index 7be8675c3c245..b4453ffed5bb4 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -20,7 +20,6 @@ import java.io.File import java.util.concurrent.TimeUnit import com.google.common.cache.CacheBuilder -import io.fabric8.kubernetes.api.model.ContainerBuilder import io.fabric8.kubernetes.client.Config import org.apache.spark.SparkContext From 7f3cb04623626d85555276e99100c3391bab4ff1 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Thu, 23 Aug 2018 14:01:25 +0100 Subject: [PATCH 19/53] redo mounting file --- .../apache/spark/deploy/k8s/Constants.scala | 7 +-- .../spark/deploy/k8s/KubernetesUtils.scala | 8 ++-- .../k8s/features/TemplateVolumeStep.scala | 48 +++++++++++++------ .../k8s/submit/KubernetesDriverBuilder.scala | 4 +- .../features/TemplateVolumeStepSuite.scala | 44 ++++++++++++++--- 5 files changed, 81 insertions(+), 30 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala index a95994600bb4d..ac4925783691e 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala @@ -75,10 +75,11 @@ private[spark] object Constants { val ENV_R_ARGS = "R_APP_ARGS" // Pod spec templates - val EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME = "podSpecTemplate.yml" - val EXECUTOR_POD_SPEC_TEMPLATE_FILE = - s"$SPARK_CONF_DIR_INTERNAL/$EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME" + val EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME = "pod-spec-template.yml" + val EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH = "/opt/spark/pod-template" val POD_TEMPLATE_VOLUME = "podspec-volume" + val POD_TEMPLATE_CONFIGMAP = "podspec-configmap" + val POD_TEMPLATE_KEY = "podspec-configmap-key" // Miscellaneous val KUBERNETES_MASTER_INTERNAL_URL = "https://kubernetes.default.svc" diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 1719c14e46bf4..1497e73cade21 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -70,11 +70,9 @@ private[spark] object KubernetesUtils extends Logging { containerName: String): SparkPod = { try { val pod = kubernetesClient.pods().load(templateFile).get() - val container = pod.getSpec.getContainers.asScala - .filter(_.getName == containerName) - .headOption - require(container.isDefined) - SparkPod(pod, container.get) + val containers = pod.getSpec.getContainers.asScala + require(containers.map(_.getName).contains(containerName)) + SparkPod(pod, containers.filter(_.getName == containerName).head) } catch { case e: Exception => logError( diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala index cfdb0537aee07..b954fcae5a9e6 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala @@ -16,6 +16,10 @@ */ package org.apache.spark.deploy.k8s.features +import java.io.File +import java.nio.charset.StandardCharsets + +import com.google.common.io.Files import io.fabric8.kubernetes.api.model.{Config => _, _} import org.apache.spark.deploy.k8s._ @@ -24,28 +28,44 @@ private[spark] class TemplateVolumeStep( conf: KubernetesConf[_ <: KubernetesRoleSpecificConf]) extends KubernetesFeatureConfigStep { def configurePod(pod: SparkPod): SparkPod = { - require(conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) - val podTemplateFile = conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get val podWithVolume = new PodBuilder(pod.pod) - .editSpec() - .addNewVolume() - .withName(Constants.POD_TEMPLATE_VOLUME) - .withHostPath(new HostPathVolumeSource(podTemplateFile)) - .endVolume() - .endSpec() + .editSpec() + .addNewVolume() + .withName(Constants.POD_TEMPLATE_VOLUME) + .withNewConfigMap() + .withName(Constants.POD_TEMPLATE_CONFIGMAP) + .addNewItem() + .withKey(Constants.POD_TEMPLATE_KEY) + .withPath(Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME) + .endItem() + .endConfigMap() + .endVolume() + .endSpec() .build() val containerWithVolume = new ContainerBuilder(pod.container) - .withVolumeMounts(new VolumeMountBuilder() + .addNewVolumeMount() .withName(Constants.POD_TEMPLATE_VOLUME) - .withMountPath(Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE) - .build()) - .build() + .withMountPath(Constants.EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH) + .endVolumeMount() + .build() SparkPod(podWithVolume, containerWithVolume) } def getAdditionalPodSystemProperties(): Map[String, String] = Map[String, String]( - Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE.key -> Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE) + Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE.key -> + (Constants.EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH + "/" + + Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME)) - def getAdditionalKubernetesResources(): Seq[HasMetadata] = Seq.empty + def getAdditionalKubernetesResources(): Seq[HasMetadata] = { + require(conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) + val podTemplateFile = conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get + val podTemplateString = Files.toString(new File(podTemplateFile), StandardCharsets.UTF_8) + Seq(new ConfigMapBuilder() + .withNewMetadata() + .withName(Constants.POD_TEMPLATE_CONFIGMAP) + .endMetadata() + .addToData(Constants.POD_TEMPLATE_KEY, podTemplateString) + .build()) + } } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index 0c4d5a8f17a09..c3a1c05a688e1 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -21,8 +21,8 @@ import java.io.File import io.fabric8.kubernetes.client.KubernetesClient import org.apache.spark.{SparkConf, SparkException} -import org.apache.spark.deploy.k8s._ -import org.apache.spark.deploy.k8s.features._ +import org.apache.spark.deploy.k8s.{Config, Constants, KubernetesConf, KubernetesDriverSpec, KubernetesDriverSpecificConf, KubernetesRoleSpecificConf, KubernetesUtils} +import org.apache.spark.deploy.k8s.features.{BasicDriverFeatureStep, DriverKubernetesCredentialsFeatureStep, DriverServiceFeatureStep, EnvSecretsFeatureStep, LocalDirsFeatureStep, MountSecretsFeatureStep, MountVolumesFeatureStep, TemplateVolumeStep} import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep, RDriverFeatureStep} import org.apache.spark.internal.Logging diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStepSuite.scala index 6ed6723c343fc..935bab803d406 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStepSuite.scala @@ -16,6 +16,9 @@ */ package org.apache.spark.deploy.k8s.features +import java.io.{File, PrintWriter} + +import io.fabric8.kubernetes.api.model.ConfigMap import org.mockito.Mockito import org.scalatest.BeforeAndAfter @@ -23,7 +26,6 @@ import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.deploy.k8s._ class TemplateVolumeStepSuite extends SparkFunSuite with BeforeAndAfter { - private val podTemplateLocalFile = "/path/to/file" private var sparkConf: SparkConf = _ private var kubernetesConf : KubernetesConf[_ <: KubernetesRoleSpecificConf] = _ @@ -48,17 +50,47 @@ class TemplateVolumeStepSuite extends SparkFunSuite with BeforeAndAfter { } test("Mounts executor template volume if config specified") { + val templateFile = new File("pod-template.yaml") + val writer = new PrintWriter(templateFile) + writer.write("pod-template-contents") + writer.close() + val podTemplateLocalFile = templateFile.getAbsolutePath Mockito.doReturn(Option(podTemplateLocalFile)).when(sparkConf) .get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) + val step = new TemplateVolumeStep(kubernetesConf) val configuredPod = step.configurePod(SparkPod.initialPod()) assert(configuredPod.pod.getSpec.getVolumes.size() === 1) - assert(configuredPod.pod.getSpec.getVolumes.get(0).getName === Constants.POD_TEMPLATE_VOLUME) - assert(configuredPod.pod.getSpec.getVolumes.get(0).getHostPath.getPath === "/path/to/file") + val volume = configuredPod.pod.getSpec.getVolumes.get(0) + assert(volume.getName === Constants.POD_TEMPLATE_VOLUME) + assert(volume.getConfigMap.getName === Constants.POD_TEMPLATE_CONFIGMAP) + assert(volume.getConfigMap.getItems.size() === 1) + assert(volume.getConfigMap.getItems.get(0).getKey === Constants.POD_TEMPLATE_KEY) + assert(volume.getConfigMap.getItems.get(0).getPath === + Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME) + assert(configuredPod.container.getVolumeMounts.size() === 1) - assert(configuredPod.container.getVolumeMounts.get(0).getName === Constants.POD_TEMPLATE_VOLUME) - assert(configuredPod.container.getVolumeMounts.get(0).getMountPath === - Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE) + val volumeMount = configuredPod.container.getVolumeMounts.get(0) + assert(volumeMount.getMountPath === Constants.EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH) + assert(volumeMount.getName === Constants.POD_TEMPLATE_VOLUME) + + val resources = step.getAdditionalKubernetesResources() + assert(resources.size === 1) + assert(resources.head.getMetadata.getName === Constants.POD_TEMPLATE_CONFIGMAP) + assert(resources.head.isInstanceOf[ConfigMap]) + val configMap = resources.head.asInstanceOf[ConfigMap] + assert(configMap.getData.size() === 1) + assert(configMap.getData.containsKey(Constants.POD_TEMPLATE_KEY)) + assert(configMap.getData.containsValue("pod-template-contents")) + + val systemProperties = step.getAdditionalPodSystemProperties() + assert(systemProperties.size === 1) + assert(systemProperties.contains(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE.key)) + assert(systemProperties.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE.key).get === + (Constants.EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH + "/" + + Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME)) + + templateFile.delete() } } From cc8d3f8d306a2866eef4098d47b25fd3eea81d3a Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Thu, 23 Aug 2018 14:08:37 +0100 Subject: [PATCH 20/53] rename to TemplateConfigMapStep --- ...lumeStep.scala => PodTemplateConfigMapStep.scala} | 2 +- .../deploy/k8s/submit/KubernetesDriverBuilder.scala | 12 ++++++------ ...ite.scala => PodTemplateConfigMapStepSuite.scala} | 4 ++-- .../k8s/submit/KubernetesDriverBuilderSuite.scala | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/{TemplateVolumeStep.scala => PodTemplateConfigMapStep.scala} (98%) rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/{TemplateVolumeStepSuite.scala => PodTemplateConfigMapStepSuite.scala} (96%) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala similarity index 98% rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala index b954fcae5a9e6..f4ed6d8783cac 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala @@ -24,7 +24,7 @@ import io.fabric8.kubernetes.api.model.{Config => _, _} import org.apache.spark.deploy.k8s._ -private[spark] class TemplateVolumeStep( +private[spark] class PodTemplateConfigMapStep( conf: KubernetesConf[_ <: KubernetesRoleSpecificConf]) extends KubernetesFeatureConfigStep { def configurePod(pod: SparkPod): SparkPod = { diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index c3a1c05a688e1..a1607fcf3e6ff 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -22,11 +22,11 @@ import io.fabric8.kubernetes.client.KubernetesClient import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.deploy.k8s.{Config, Constants, KubernetesConf, KubernetesDriverSpec, KubernetesDriverSpecificConf, KubernetesRoleSpecificConf, KubernetesUtils} -import org.apache.spark.deploy.k8s.features.{BasicDriverFeatureStep, DriverKubernetesCredentialsFeatureStep, DriverServiceFeatureStep, EnvSecretsFeatureStep, LocalDirsFeatureStep, MountSecretsFeatureStep, MountVolumesFeatureStep, TemplateVolumeStep} +import org.apache.spark.deploy.k8s.features.{BasicDriverFeatureStep, DriverKubernetesCredentialsFeatureStep, DriverServiceFeatureStep, EnvSecretsFeatureStep, LocalDirsFeatureStep, MountSecretsFeatureStep, MountVolumesFeatureStep, PodTemplateConfigMapStep, TemplateVolumeStep} import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep, RDriverFeatureStep} import org.apache.spark.internal.Logging -private[spark] class KubernetesDriverBuilder ( +private[spark] class KubernetesDriverBuilder( provideBasicStep: (KubernetesConf[KubernetesDriverSpecificConf]) => BasicDriverFeatureStep = new BasicDriverFeatureStep(_), provideCredentialsStep: (KubernetesConf[KubernetesDriverSpecificConf]) @@ -58,9 +58,9 @@ private[spark] class KubernetesDriverBuilder ( KubernetesConf[KubernetesDriverSpecificConf] => JavaDriverFeatureStep) = new JavaDriverFeatureStep(_), - provideTemplateVolumeStep: (KubernetesConf[_ <: KubernetesRoleSpecificConf] - => TemplateVolumeStep) = - new TemplateVolumeStep(_), + podTemplateConfigMapStep: (KubernetesConf[_ <: KubernetesRoleSpecificConf] + => PodTemplateConfigMapStep) = + new PodTemplateConfigMapStep()(_), provideInitialSpec: KubernetesConf[KubernetesDriverSpecificConf] => KubernetesDriverSpec = KubernetesDriverSpec.initialSpec) { @@ -84,7 +84,7 @@ private[spark] class KubernetesDriverBuilder ( } else Nil val templateVolumeFeature = if ( kubernetesConf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) { - Seq(provideTemplateVolumeStep(kubernetesConf)) + Seq(podTemplateConfigMapStep(kubernetesConf)) } else Nil val bindingsStep = kubernetesConf.roleSpecificConf.mainAppResource.map { diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala similarity index 96% rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStepSuite.scala rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala index 935bab803d406..b06455e8833b7 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/TemplateVolumeStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala @@ -25,7 +25,7 @@ import org.scalatest.BeforeAndAfter import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.deploy.k8s._ -class TemplateVolumeStepSuite extends SparkFunSuite with BeforeAndAfter { +class PodTemplateConfigMapStepSuite extends SparkFunSuite with BeforeAndAfter { private var sparkConf: SparkConf = _ private var kubernetesConf : KubernetesConf[_ <: KubernetesRoleSpecificConf] = _ @@ -58,7 +58,7 @@ class TemplateVolumeStepSuite extends SparkFunSuite with BeforeAndAfter { Mockito.doReturn(Option(podTemplateLocalFile)).when(sparkConf) .get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) - val step = new TemplateVolumeStep(kubernetesConf) + val step = new PodTemplateConfigMapStep(kubernetesConf) val configuredPod = step.configurePod(SparkPod.initialPod()) assert(configuredPod.pod.getSpec.getVolumes.size() === 1) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala index bc6ca354da936..b0516d3226f08 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala @@ -76,7 +76,7 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { MOUNT_VOLUMES_STEP_TYPE, classOf[MountVolumesFeatureStep]) private val templateVolumeStep = KubernetesFeaturesTestUtils.getMockConfigStepForStepType( - TEMPLATE_VOLUME_STEP_TYPE, classOf[TemplateVolumeStep] + TEMPLATE_VOLUME_STEP_TYPE, classOf[PodTemplateConfigMapStep] ) private val builderUnderTest: KubernetesDriverBuilder = @@ -322,7 +322,7 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { assert(spec.pod.container.getName === Constants.DRIVER_CONTAINER_NAME) } - test("Starts with empty pod if bad template") { + test("Throws on misconfigured pod template") { val exception = intercept[SparkException] { getSpecWithPodTemplate( new PodBuilder() From 4119899a9d3e2759dc1b5ab33cbd32ee72fea5af Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Thu, 23 Aug 2018 14:16:36 +0100 Subject: [PATCH 21/53] Pass initialPod constructor instead of Spec constructor --- .../k8s/submit/KubernetesDriverBuilder.scala | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index a1607fcf3e6ff..ea2d65827b6ed 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -21,8 +21,8 @@ import java.io.File import io.fabric8.kubernetes.client.KubernetesClient import org.apache.spark.{SparkConf, SparkException} -import org.apache.spark.deploy.k8s.{Config, Constants, KubernetesConf, KubernetesDriverSpec, KubernetesDriverSpecificConf, KubernetesRoleSpecificConf, KubernetesUtils} -import org.apache.spark.deploy.k8s.features.{BasicDriverFeatureStep, DriverKubernetesCredentialsFeatureStep, DriverServiceFeatureStep, EnvSecretsFeatureStep, LocalDirsFeatureStep, MountSecretsFeatureStep, MountVolumesFeatureStep, PodTemplateConfigMapStep, TemplateVolumeStep} +import org.apache.spark.deploy.k8s.{Config, Constants, KubernetesConf, KubernetesDriverSpec, KubernetesDriverSpecificConf, KubernetesRoleSpecificConf, KubernetesUtils, SparkPod} +import org.apache.spark.deploy.k8s.features.{BasicDriverFeatureStep, DriverKubernetesCredentialsFeatureStep, DriverServiceFeatureStep, EnvSecretsFeatureStep, LocalDirsFeatureStep, MountSecretsFeatureStep, MountVolumesFeatureStep, PodTemplateConfigMapStep} import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep, RDriverFeatureStep} import org.apache.spark.internal.Logging @@ -60,10 +60,8 @@ private[spark] class KubernetesDriverBuilder( new JavaDriverFeatureStep(_), podTemplateConfigMapStep: (KubernetesConf[_ <: KubernetesRoleSpecificConf] => PodTemplateConfigMapStep) = - new PodTemplateConfigMapStep()(_), - provideInitialSpec: KubernetesConf[KubernetesDriverSpecificConf] - => KubernetesDriverSpec = - KubernetesDriverSpec.initialSpec) { + new PodTemplateConfigMapStep(_), + provideInitialPod: () => SparkPod = SparkPod.initialPod) { def buildFromFeatures( kubernetesConf: KubernetesConf[KubernetesDriverSpecificConf]): KubernetesDriverSpec = { @@ -99,7 +97,10 @@ private[spark] class KubernetesDriverBuilder( val allFeatures = (baseFeatures :+ bindingsStep) ++ secretFeature ++ envSecretFeature ++ volumesFeature ++ templateVolumeFeature - var spec = provideInitialSpec(kubernetesConf) + var spec = KubernetesDriverSpec( + provideInitialPod(), + Seq.empty, + kubernetesConf.sparkConf.getAll.toMap) for (feature <- allFeatures) { val configuredPod = feature.configurePod(spec.pod) val addedSystemProperties = feature.getAdditionalPodSystemProperties() @@ -117,13 +118,12 @@ private[spark] object KubernetesDriverBuilder extends Logging { def apply(kubernetesClient: KubernetesClient, conf: SparkConf): KubernetesDriverBuilder = { conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE) .map(new File(_)) - .map(file => new KubernetesDriverBuilder(provideInitialSpec = conf => { + .map(file => new KubernetesDriverBuilder(provideInitialPod = () => { try { - val sparkPod = KubernetesUtils.loadPodFromTemplate( + KubernetesUtils.loadPodFromTemplate( kubernetesClient, file, Constants.DRIVER_CONTAINER_NAME) - KubernetesDriverSpec.initialSpec(conf).copy(pod = sparkPod) } catch { case e: Exception => logError( From 1d0a8fa5203c273e529e23296256a143be295465 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Thu, 23 Aug 2018 14:35:13 +0100 Subject: [PATCH 22/53] make driver and executor container names configurable --- .../scala/org/apache/spark/deploy/k8s/Config.scala | 13 ++++++++++++- .../org/apache/spark/deploy/k8s/Constants.scala | 2 -- .../k8s/features/BasicDriverFeatureStep.scala | 2 +- .../k8s/features/BasicExecutorFeatureStep.scala | 2 +- .../deploy/k8s/submit/KubernetesDriverBuilder.scala | 2 +- .../cluster/k8s/KubernetesClusterManager.scala | 2 +- .../cluster/k8s/KubernetesExecutorBuilder.scala | 2 +- .../k8s/features/BasicDriverFeatureStepSuite.scala | 3 ++- .../k8s/submit/KubernetesDriverBuilderSuite.scala | 5 +++-- 9 files changed, 22 insertions(+), 11 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index c7e392a470603..4c5176a946368 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -225,6 +225,18 @@ private[spark] object Config extends Logging { "Ensure that major Python version is either Python2 or Python3") .createWithDefault("2") + val KUBERNETES_DRIVER_CONTAINER_NAME = + ConfigBuilder("spark.kubernetes.driver.containerName") + .doc("The name of the driver container within the driver pod template file") + .stringConf + .createWithDefault("spark-kubernetes-driver") + + val KUBERNETES_EXECUTOR_CONTAINER_NAME = + ConfigBuilder("spark.kubernetes.executor.containerName") + .doc("The name of the executor container within the executor pod template file") + .stringConf + .createWithDefault("spark-kubernetes-executor") + val KUBERNETES_DRIVER_PODTEMPLATE_FILE = ConfigBuilder("spark.kubernetes.driver.podTemplateFile") .doc("File containing a template pod spec for the driver") @@ -237,7 +249,6 @@ private[spark] object Config extends Logging { .stringConf .createOptional - val KUBERNETES_AUTH_SUBMISSION_CONF_PREFIX = "spark.kubernetes.authenticate.submission" diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala index ac4925783691e..557047d72b9cd 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala @@ -83,7 +83,5 @@ private[spark] object Constants { // Miscellaneous val KUBERNETES_MASTER_INTERNAL_URL = "https://kubernetes.default.svc" - val DRIVER_CONTAINER_NAME = "spark-kubernetes-driver" - val EXECUTOR_CONTAINER_NAME = "executor" val MEMORY_OVERHEAD_MIN_MIB = 384L } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala index 575bc54ffe2bb..5d4dcd8a612d9 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala @@ -80,7 +80,7 @@ private[spark] class BasicDriverFeatureStep( ) val driverUIPort = SparkUI.getUIPort(conf.sparkConf) val driverContainer = new ContainerBuilder(pod.container) - .withName(DRIVER_CONTAINER_NAME) + .withName(conf.get(KUBERNETES_DRIVER_CONTAINER_NAME)) .withImage(driverContainerImage) .withImagePullPolicy(conf.imagePullPolicy()) .addNewPort() diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala index d413db4000bed..b5e0e8adf4e4e 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala @@ -129,7 +129,7 @@ private[spark] class BasicExecutorFeatureStep( } val executorContainer = new ContainerBuilder(pod.container) - .withName(Constants.EXECUTOR_CONTAINER_NAME) + .withName(kubernetesConf.get(KUBERNETES_EXECUTOR_CONTAINER_NAME)) .withImage(executorContainerImage) .withImagePullPolicy(kubernetesConf.imagePullPolicy()) .withNewResources() diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index ea2d65827b6ed..9ac0fdb12042d 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -123,7 +123,7 @@ private[spark] object KubernetesDriverBuilder extends Logging { KubernetesUtils.loadPodFromTemplate( kubernetesClient, file, - Constants.DRIVER_CONTAINER_NAME) + conf.get(Config.KUBERNETES_DRIVER_CONTAINER_NAME)) } catch { case e: Exception => logError( diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index b4453ffed5bb4..04a1f327fcdce 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -73,7 +73,7 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit KubernetesUtils.loadPodFromTemplate( kubernetesClient, new File(sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get), - Constants.EXECUTOR_CONTAINER_NAME) + sc.conf.get(KUBERNETES_EXECUTOR_CONTAINER_NAME)) } val requestExecutorsService = ThreadUtils.newDaemonCachedThreadPool( diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala index 412b66ea0e8af..e617ccc553047 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala @@ -76,7 +76,7 @@ private[spark] object KubernetesExecutorBuilder { KubernetesUtils.loadPodFromTemplate( kubernetesClient, file, - Constants.EXECUTOR_CONTAINER_NAME) + conf.get(Config.KUBERNETES_EXECUTOR_CONTAINER_NAME)) })) .getOrElse(new KubernetesExecutorBuilder()) } diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala index d98e113554648..4d1b51c92f690 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala @@ -67,6 +67,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite { .set(org.apache.spark.internal.config.DRIVER_MEMORY_OVERHEAD, 200L) .set(CONTAINER_IMAGE, "spark-driver:latest") .set(IMAGE_PULL_SECRETS, TEST_IMAGE_PULL_SECRETS.mkString(",")) + .set(KUBERNETES_DRIVER_CONTAINER_NAME, "driver-container") val kubernetesConf = KubernetesConf( sparkConf, emptyDriverSpecificConf, @@ -84,7 +85,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite { val basePod = SparkPod.initialPod() val configuredPod = featureStep.configurePod(basePod) - assert(configuredPod.container.getName === DRIVER_CONTAINER_NAME) + assert(configuredPod.container.getName === "driver-container") assert(configuredPod.container.getImage === "spark-driver:latest") assert(configuredPod.container.getImagePullPolicy === CONTAINER_IMAGE_PULL_POLICY) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala index b0516d3226f08..a7d291d111873 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala @@ -312,14 +312,15 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { .endMetadata() .withNewSpec() .addNewContainer() - .withName(Constants.DRIVER_CONTAINER_NAME) + .withName(Config.KUBERNETES_DRIVER_CONTAINER_NAME.defaultValueString) .endContainer() .endSpec() .build()) assert(spec.pod.pod.getMetadata.getLabels.containsKey("test-label-key")) assert(spec.pod.pod.getMetadata.getLabels.get("test-label-key") === "test-label-value") - assert(spec.pod.container.getName === Constants.DRIVER_CONTAINER_NAME) + assert(spec.pod.container.getName === + Config.KUBERNETES_DRIVER_CONTAINER_NAME.defaultValueString) } test("Throws on misconfigured pod template") { From 81e5a66ca5af663255001bea8df66b7690683856 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Thu, 23 Aug 2018 14:43:27 +0100 Subject: [PATCH 23/53] create temp file correctly? --- .../k8s/features/PodTemplateConfigMapStepSuite.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala index b06455e8833b7..fce88e51fb442 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.deploy.k8s.features import java.io.{File, PrintWriter} +import java.nio.file.Files import io.fabric8.kubernetes.api.model.ConfigMap import org.mockito.Mockito @@ -28,6 +29,7 @@ import org.apache.spark.deploy.k8s._ class PodTemplateConfigMapStepSuite extends SparkFunSuite with BeforeAndAfter { private var sparkConf: SparkConf = _ private var kubernetesConf : KubernetesConf[_ <: KubernetesRoleSpecificConf] = _ + private var templateFile: File = _ before { sparkConf = Mockito.mock(classOf[SparkConf]) @@ -47,16 +49,16 @@ class PodTemplateConfigMapStepSuite extends SparkFunSuite with BeforeAndAfter { Map.empty, Nil, Seq.empty[String]) + templateFile = Files.createTempFile("pod-template", "yml").toFile + templateFile.deleteOnExit() + Mockito.doReturn(Option(templateFile.getAbsolutePath)).when(sparkConf) + .get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) } test("Mounts executor template volume if config specified") { - val templateFile = new File("pod-template.yaml") val writer = new PrintWriter(templateFile) writer.write("pod-template-contents") writer.close() - val podTemplateLocalFile = templateFile.getAbsolutePath - Mockito.doReturn(Option(podTemplateLocalFile)).when(sparkConf) - .get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) val step = new PodTemplateConfigMapStep(kubernetesConf) val configuredPod = step.configurePod(SparkPod.initialPod()) @@ -90,7 +92,5 @@ class PodTemplateConfigMapStepSuite extends SparkFunSuite with BeforeAndAfter { assert(systemProperties.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE.key).get === (Constants.EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH + "/" + Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME)) - - templateFile.delete() } } From 7f4ff5a10df2de6d60481f1816051e9cd7c22a34 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Thu, 23 Aug 2018 15:30:48 +0100 Subject: [PATCH 24/53] executor initial pod test --- .../features/BasicExecutorFeatureStep.scala | 4 +- .../submit/KubernetesDriverBuilderSuite.scala | 19 +++--- .../k8s/KubernetesExecutorBuilderSuite.scala | 68 ++++++++++++++++++- 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala index b5e0e8adf4e4e..8e56e1d250d90 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala @@ -163,8 +163,8 @@ private[spark] class BasicExecutorFeatureStep( val executorPod = new PodBuilder(pod.pod) .editOrNewMetadata() .withName(name) - .withLabels(kubernetesConf.roleLabels.asJava) - .withAnnotations(kubernetesConf.roleAnnotations.asJava) + .addToLabels(kubernetesConf.roleLabels.asJava) + .addToAnnotations(kubernetesConf.roleAnnotations.asJava) .addToOwnerReferences(ownerReference.toSeq: _*) .endMetadata() .editOrNewSpec() diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala index a7d291d111873..1be37d7ab2268 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala @@ -26,9 +26,8 @@ import org.mockito.Mockito._ import org.apache.spark.{SparkConf, SparkException, SparkFunSuite} import org.apache.spark.deploy.k8s._ -import org.apache.spark.deploy.k8s.Config.CONTAINER_IMAGE -import org.apache.spark.deploy.k8s.features._ -import org.apache.spark.deploy.k8s.features.{BasicDriverFeatureStep, DriverKubernetesCredentialsFeatureStep, DriverServiceFeatureStep, EnvSecretsFeatureStep, KubernetesFeaturesTestUtils, LocalDirsFeatureStep, MountSecretsFeatureStep} +import org.apache.spark.deploy.k8s.Config.{CONTAINER_IMAGE, KUBERNETES_DRIVER_CONTAINER_NAME, KUBERNETES_DRIVER_PODTEMPLATE_FILE, KUBERNETES_EXECUTOR_PODTEMPLATE_FILE} +import org.apache.spark.deploy.k8s.features.{BasicDriverFeatureStep, DriverKubernetesCredentialsFeatureStep, DriverServiceFeatureStep, EnvSecretsFeatureStep, KubernetesFeaturesTestUtils, LocalDirsFeatureStep, MountSecretsFeatureStep, _} import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep, RDriverFeatureStep} class KubernetesDriverBuilderSuite extends SparkFunSuite { @@ -260,7 +259,7 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { test("Apply template volume step if executor template is present.") { val sparkConf = spy(new SparkConf(false)) doReturn(Option("filename")).when(sparkConf) - .get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) + .get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) val conf = KubernetesConf( sparkConf, KubernetesDriverSpecificConf( @@ -305,14 +304,14 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { } test("Starts with template if specified") { - val spec = getSpecWithPodTemplate( + val spec = constructSpecWithPodTemplate( new PodBuilder() .withNewMetadata() .addToLabels("test-label-key", "test-label-value") .endMetadata() .withNewSpec() .addNewContainer() - .withName(Config.KUBERNETES_DRIVER_CONTAINER_NAME.defaultValueString) + .withName(KUBERNETES_DRIVER_CONTAINER_NAME.defaultValueString) .endContainer() .endSpec() .build()) @@ -320,12 +319,12 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { assert(spec.pod.pod.getMetadata.getLabels.containsKey("test-label-key")) assert(spec.pod.pod.getMetadata.getLabels.get("test-label-key") === "test-label-value") assert(spec.pod.container.getName === - Config.KUBERNETES_DRIVER_CONTAINER_NAME.defaultValueString) + KUBERNETES_DRIVER_CONTAINER_NAME.defaultValueString) } test("Throws on misconfigured pod template") { val exception = intercept[SparkException] { - getSpecWithPodTemplate( + constructSpecWithPodTemplate( new PodBuilder() .withNewMetadata() .addToLabels("test-label-key", "test-label-value") @@ -335,7 +334,7 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { assert(exception.getMessage.contains("Could not load driver pod from template file.")) } - private def getSpecWithPodTemplate(pod: Pod) : KubernetesDriverSpec = { + private def constructSpecWithPodTemplate(pod: Pod) : KubernetesDriverSpec = { val kubernetesClient = mock(classOf[KubernetesClient]) val pods = mock(classOf[MixedOperation[Pod, PodList, DoneablePod, PodResource[Pod, DoneablePod]]]) @@ -346,7 +345,7 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { val sparkConf = new SparkConf(false) .set(CONTAINER_IMAGE, "spark-driver:latest") - .set(Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE, "template-file.yaml") + .set(KUBERNETES_DRIVER_PODTEMPLATE_FILE, "template-file.yaml") val kubernetesConf = new KubernetesConf( sparkConf, diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala index 44fe4a24e1102..60a7f2eea3a15 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala @@ -16,7 +16,13 @@ */ package org.apache.spark.scheduler.cluster.k8s -import io.fabric8.kubernetes.api.model.PodBuilder +import java.io.File + +import io.fabric8.kubernetes.api.model.{DoneablePod, Pod, PodBuilder, PodList} +import io.fabric8.kubernetes.client.KubernetesClient +import io.fabric8.kubernetes.client.dsl.{MixedOperation, PodResource} +import org.mockito.Matchers.any +import org.mockito.Mockito.{mock, never, verify, when} import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.deploy.k8s._ @@ -119,4 +125,64 @@ class KubernetesExecutorBuilderSuite extends SparkFunSuite { assert(resolvedPod.pod.getMetadata.getLabels.get(stepType) === stepType) } } + + test("Starts with empty executor pod if template is not specified") { + val kubernetesClient = mock(classOf[KubernetesClient]) + val executorBuilder = KubernetesExecutorBuilder.apply(kubernetesClient, new SparkConf()) + verify(kubernetesClient, never()).pods() + } + + test("Starts with executor template if specified") { + val pod = constructPodWithPodTemplate( + new PodBuilder() + .withNewMetadata() + .addToLabels("test-label-key", "test-label-value") + .endMetadata() + .withNewSpec() + .addNewContainer() + .withName(Config.KUBERNETES_EXECUTOR_CONTAINER_NAME.defaultValueString) + .endContainer() + .endSpec() + .build()) + + assert(pod.pod.getMetadata.getLabels.containsKey("test-label-key")) + assert(pod.pod.getMetadata.getLabels.get("test-label-key") === "test-label-value") + assert(pod.container.getName === + Config.KUBERNETES_EXECUTOR_CONTAINER_NAME.defaultValueString) + } + + private def constructPodWithPodTemplate(pod: Pod) : SparkPod = { + val kubernetesClient = mock(classOf[KubernetesClient]) + val pods = + mock(classOf[MixedOperation[Pod, PodList, DoneablePod, PodResource[Pod, DoneablePod]]]) + val podResource = mock(classOf[PodResource[Pod, DoneablePod]]) + when(kubernetesClient.pods()).thenReturn(pods) + when(pods.load(any(classOf[File]))).thenReturn(podResource) + when(podResource.get()).thenReturn(pod) + + val sparkConf = new SparkConf(false) + .set("spark.driver.host", "https://driver.host.com") + .set(Config.CONTAINER_IMAGE, "spark-executor:latest") + .set(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE, "template-file.yaml") + + val kubernetesConf = KubernetesConf( + sparkConf, + KubernetesExecutorSpecificConf( + "executor-id", Some(new PodBuilder() + .withNewMetadata() + .withName("driver") + .endMetadata() + .build())), + "prefix", + "appId", + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Nil, + Seq.empty[String]) + + KubernetesExecutorBuilder.apply(kubernetesClient, sparkConf).buildFromFeatures(kubernetesConf) + } } From 3097aefb89921bddf1d48b2b1f2fe8dbd7dcb230 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Thu, 23 Aug 2018 19:34:44 +0100 Subject: [PATCH 25/53] add docs --- docs/running-on-kubernetes.md | 191 ++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index 8f84ca044e163..dd9483c1a407e 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -185,6 +185,18 @@ To use a secret through an environment variable use the following options to the --conf spark.kubernetes.executor.secretKeyRef.ENV_NAME=name:key ``` +## Pod Template +Kubernetes allows defining pods from [template files](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/#pod-templates). +Spark users can similarly use template files to define the driver or executor pod configurations that Spark configurations do not support. +To do so, specify the spark properties `spark.kubernetes.driver.podTemplateFile` and `spark.kubernetes.executor.podTemplateFile` +to point to local files accessible to the `spark-submit` process. To allow the driver pod access the executor pod template +file, the file will be automatically mounted onto a volume in the driver pod when it's created. + +It is important to note that Spark is opinionated about certain pod configurations so there are values in the +pod template that will always be overwritten by Spark. Therefore, users of this feature should note that specifying +the pod template file only lets Spark start with a template pod instead of an empty pod during the pod-building process. +For details, see the [full list](#pod-template-properties) of pod template values that will be overwritten by spark. + ## Introspection and Debugging These are the different ways in which you can investigate a running/completed Spark application, monitor progress, and @@ -775,4 +787,183 @@ specific to Spark on Kubernetes. This sets the major Python version of the docker image used to run the driver and executor containers. Can either be 2 or 3. + + spark.kubernetes.driver.containerName + "spark-kubernetes-driver" + + This sets the driver container name. If you are specifying a driver [pod template](#pod-template), you can match this name to the + driver container name set in the template. + + + + spark.kubernetes.executor.containerName + "spark-kubernetes-executor" + + This sets the executor container name. If you are specifying a an executor [pod template](#pod-template), you can match this name to the + driver container name set in the template. + + + + spark.kubernetes.driver.podTemplateFile + (none) + + Specify the local file that contains the driver [pod template](#pod-template). For example + spark.kubernetes.driver.podTemplateFile=/path/to/driver-pod-template.yaml` + + + + spark.kubernetes.executor.podTemplateFile + (none) + + Specify the local file that contains the executor [pod template](#pod-template). For example + spark.kubernetes.executor.podTemplateFile=/path/to/executor-pod-template.yaml` + + + + +#### Pod template properties + +See the below table for the full list of pod specifications that will be overwritten by spark. + +### Pod Metadata + + + + + + + + + + + + + + + + + + + + + + + +
Pod metadata keyModified valueDescription
nameValue of spark.kubernetes.driver.pod.name + The driver pod name will be overwritten with either the configured or default value of + spark.kubernetes.driver.pod.name. The executor pod names will be unaffected. +
namespaceValue of spark.kubernetes.namespace + Spark makes strong assumptions about the driver and executor namespaces. Both driver and executor namespaces will + be replaced by this spark conf value. +
labelsAdds the labels from spark.kubernetes.{driver,executor}.label.* + Spark will add additional labels specified by the spark configuration. +
annotationsAdds the annotations from spark.kubernetes.{driver,executor}.annotation.* + Spark will add additional labels specified by the spark configuration. +
+ +### Pod Spec + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Pod spec keyModified valueDescription
imagePullSecretsAdds image pull secrets from spark.kubernetes.container.image.pullSecrets + Additional pull secrets will be added from the spark configuration to both executor pods. +
nodeSelectorAdds node selectors from spark.kubernetes.node.selector.* + Additional node selectors will be added from the spark configuration to both executor pods. +
restartPolicy"never" + Spark assumes that both drivers and executors never restart. +
serviceAccountValue of spark.kubernetes.authenticate.driver.serviceAccountName + Spark will override serviceAccount with the value of the spark configuration for only + driver pods. Executor pods will remain unaffected. +
serviceAccountNameValue of spark.kubernetes.authenticate.driver.serviceAccountName + Spark will override serviceAccountName with the value of the spark configuration for only + driver pods. Executor pods will remain unaffected. +
volumesAdds volumes from spark.kubernetes.{driver,executor}.volumes.[VolumeType].[VolumeName].mount.path + Spark will add volumes as specified by the spark conf, as well as additional volumes necessary for passing + spark conf and pod template files. +
+ +### Container spec + +The following affect the driver and executor containers. All other containers in the pod spec will be unaffected. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Container spec keyModified valueDescription
envAdds env variables from spark.kubernetes.driverEnv.[EnvironmentVariableName] + Spark will add driver env variables from spark.kubernetes.driverEnv.[EnvironmentVariableName], and + executor env variables from spark.executorEnv.[EnvironmentVariableName]. +
imageValue of spark.kubernetes.{driver,executor}.container.image + The image will be defined by the spark configurations. +
imagePullPolicyValue of spark.kubernetes.container.image.pullPolicy + Spark will override the pull policy for both driver and executors. +
nameSee description. + The driver and executor container names are defined by spark.kubernetes.{driver,executor}.containerName. + If they don't exist in the pod template file, then an error will be thrown. +
resourcesSee description + The cpu limits are set by spark.kubernetes.{driver,executor}.limit.cores. The cpu is set by + spark.{driver,executor}.cores. The memory request and limit are set by summing the values of + spark.{driver,executor}.memory and spark.{driver,executor}.memoryOverhead. + +
volumeMountsAdd volumes from spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.{path,readOnly} + Spark will add volumes as specified by the spark conf, as well as additional volumes necessary for passing + spark conf and pod template files. +
\ No newline at end of file From 9b1418aaee1e1457a7e807d7996e9719e3827fa4 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Thu, 23 Aug 2018 19:46:22 +0100 Subject: [PATCH 26/53] addressing some comments --- .../spark/deploy/k8s/KubernetesUtils.scala | 7 +++-- .../features/PodTemplateConfigMapStep.scala | 31 ++++++++++--------- .../k8s/ExecutorPodsLifecycleManager.scala | 1 - .../k8s/KubernetesClusterManager.scala | 1 - .../ExecutorPodsLifecycleManagerSuite.scala | 4 --- 5 files changed, 20 insertions(+), 24 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 1497e73cade21..a6e88e598f5e2 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -65,9 +65,10 @@ private[spark] object KubernetesUtils extends Logging { } } - def loadPodFromTemplate(kubernetesClient: KubernetesClient, - templateFile: File, - containerName: String): SparkPod = { + def loadPodFromTemplate( + kubernetesClient: KubernetesClient, + templateFile: File, + containerName: String): SparkPod = { try { val pod = kubernetesClient.pods().load(templateFile).get() val containers = pod.getSpec.getContainers.asScala diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala index f4ed6d8783cac..52baed636c3f4 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala @@ -20,9 +20,11 @@ import java.io.File import java.nio.charset.StandardCharsets import com.google.common.io.Files -import io.fabric8.kubernetes.api.model.{Config => _, _} +import io.fabric8.kubernetes.api.model.{ConfigMapBuilder, ContainerBuilder, HasMetadata, PodBuilder} -import org.apache.spark.deploy.k8s._ +import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesRoleSpecificConf, SparkPod} +import org.apache.spark.deploy.k8s.Config._ +import org.apache.spark.deploy.k8s.Constants._ private[spark] class PodTemplateConfigMapStep( conf: KubernetesConf[_ <: KubernetesRoleSpecificConf]) @@ -31,12 +33,12 @@ private[spark] class PodTemplateConfigMapStep( val podWithVolume = new PodBuilder(pod.pod) .editSpec() .addNewVolume() - .withName(Constants.POD_TEMPLATE_VOLUME) + .withName(POD_TEMPLATE_VOLUME) .withNewConfigMap() - .withName(Constants.POD_TEMPLATE_CONFIGMAP) + .withName(POD_TEMPLATE_CONFIGMAP) .addNewItem() - .withKey(Constants.POD_TEMPLATE_KEY) - .withPath(Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME) + .withKey(POD_TEMPLATE_KEY) + .withPath(EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME) .endItem() .endConfigMap() .endVolume() @@ -45,27 +47,26 @@ private[spark] class PodTemplateConfigMapStep( val containerWithVolume = new ContainerBuilder(pod.container) .addNewVolumeMount() - .withName(Constants.POD_TEMPLATE_VOLUME) - .withMountPath(Constants.EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH) + .withName(POD_TEMPLATE_VOLUME) + .withMountPath(EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH) .endVolumeMount() .build() SparkPod(podWithVolume, containerWithVolume) } def getAdditionalPodSystemProperties(): Map[String, String] = Map[String, String]( - Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE.key -> - (Constants.EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH + "/" + - Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME)) + KUBERNETES_EXECUTOR_PODTEMPLATE_FILE.key -> + (EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH + "/" + EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME)) def getAdditionalKubernetesResources(): Seq[HasMetadata] = { - require(conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) - val podTemplateFile = conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get + require(conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) + val podTemplateFile = conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get val podTemplateString = Files.toString(new File(podTemplateFile), StandardCharsets.UTF_8) Seq(new ConfigMapBuilder() .withNewMetadata() - .withName(Constants.POD_TEMPLATE_CONFIGMAP) + .withName(POD_TEMPLATE_CONFIGMAP) .endMetadata() - .addToData(Constants.POD_TEMPLATE_KEY, podTemplateString) + .addToData(POD_TEMPLATE_KEY, podTemplateString) .build()) } } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala index b28d93990313e..44f49f55b0983 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala @@ -30,7 +30,6 @@ import org.apache.spark.util.Utils private[spark] class ExecutorPodsLifecycleManager( conf: SparkConf, - executorBuilder: KubernetesExecutorBuilder, kubernetesClient: KubernetesClient, snapshotsStore: ExecutorPodsSnapshotsStore, // Use a best-effort to track which executors have been removed already. It's not generally diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index 04a1f327fcdce..ca4c3f96dcb3b 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -88,7 +88,6 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit .build[java.lang.Long, java.lang.Long]() val executorPodsLifecycleEventHandler = new ExecutorPodsLifecycleManager( sc.conf, - KubernetesExecutorBuilder(kubernetesClient, sc.conf), kubernetesClient, snapshotsStore, removedExecutorsCache) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala index 562ace9f49d4d..8206e25fa7c63 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala @@ -44,9 +44,6 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte @Mock private var podOperations: PODS = _ - @Mock - private var executorBuilder: KubernetesExecutorBuilder = _ - @Mock private var schedulerBackend: KubernetesClusterSchedulerBackend = _ @@ -63,7 +60,6 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte when(podOperations.withName(any(classOf[String]))).thenAnswer(namedPodsAnswer()) eventHandlerUnderTest = new ExecutorPodsLifecycleManager( new SparkConf(), - executorBuilder, kubernetesClient, snapshotsStore, removedExecutorsCache) From ebacc96a6f73eca3a3c48d79c458cf021c17e9c8 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Fri, 24 Aug 2018 14:37:57 +0100 Subject: [PATCH 27/53] integration tests attempt 1 --- .../integrationtest/PodTemplateSuite.scala | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala new file mode 100644 index 0000000000000..0b376a2b61729 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.nio.file.Files + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import io.fabric8.kubernetes.api.model.{Pod, PodBuilder} + +private[spark] trait PodTemplateSuite { k8sSuite: KubernetesSuite => + + import PodTemplateSuite._ + + test("Start pod creation from template") { + sparkAppConf + .set("spark.kubernetes.driver.podTemplateFile", DRIVER_TEMPLATE_FILE.getAbsolutePath) + .set("spark.kubernetes.executor.podTemplateFile", EXECUTOR_TEMPLATE_FILE.getAbsolutePath) + .set("spark.kubernetes.driver.containerName", DRIVER_CONTAINER_NAME) + .set("spark.kubernetes.executor.containerName", EXECUTOR_LABEL_VALUE) + runSparkPiAndVerifyCompletion( + driverPodChecker = (driverPod: Pod) => { + checkDriverPod(driverPod) + }, + executorPodChecker = (executorPod: Pod) => { + checkExecutorPod(executorPod) + } + ) + } + + private def checkDriverPod(pod: Pod): Unit = { + assert(pod.getMetadata.getName === driverPodName) + assert(pod.getSpec.getContainers.get(0).getImage === image) + assert(pod.getSpec.getContainers.get(0).getName === DRIVER_CONTAINER_NAME) + assert(pod.getMetadata.getLabels.containsKey(LABEL_KEY)) + assert(pod.getMetadata.getLabels.get(LABEL_KEY) === DRIVER_LABEL_VALUE) + } + + private def checkExecutorPod(pod: Pod): Unit = { + assert(pod.getMetadata.getName === "template-pod") + assert(pod.getSpec.getContainers.get(0).getImage === image) + assert(pod.getSpec.getContainers.get(0).getName === EXECUTOR_CONTAINER_NAME) + assert(pod.getMetadata.getLabels.containsKey(LABEL_KEY)) + assert(pod.getMetadata.getLabels.get(LABEL_KEY) === EXECUTOR_LABEL_VALUE) + } + + private def createPodTemplateFiles(): Unit = { + val objectMapper = new ObjectMapper(new YAMLFactory()) + val driverTemplatePod = new PodBuilder() + .withApiVersion("1") + .withKind("Pod") + .withNewMetadata() + .addToLabels(LABEL_KEY, DRIVER_LABEL_VALUE) + .endMetadata() + .withNewSpec() + .addNewContainer() + .withName(DRIVER_CONTAINER_NAME) + .withImage("will-be-overwritten") + .endContainer() + .endSpec() + .build() + + val executorTemplatePod = new PodBuilder() + .withApiVersion("1") + .withKind("Pod") + .withNewMetadata() + .withName("template-pod") + .addToLabels(LABEL_KEY, EXECUTOR_LABEL_VALUE) + .endMetadata() + .withNewSpec() + .addNewContainer() + .withName(EXECUTOR_CONTAINER_NAME) + .withImage("will-be-overwritten") + .endContainer() + .endSpec() + .build() + + objectMapper.writeValue(DRIVER_TEMPLATE_FILE, driverTemplatePod) + objectMapper.writeValue(EXECUTOR_TEMPLATE_FILE, executorTemplatePod) + } +} + +private[spark] object PodTemplateSuite { + val DRIVER_CONTAINER_NAME = "test-driver-container" + val EXECUTOR_CONTAINER_NAME = "test-executor-container" + val LABEL_KEY = "template-label-key" + val DRIVER_LABEL_VALUE = "driver-template-label-value" + val EXECUTOR_LABEL_VALUE = "executor-template-label-value" + + val DRIVER_TEMPLATE_FILE = Files.createTempFile("driver-pod-template", "yml").toFile + DRIVER_TEMPLATE_FILE.deleteOnExit() + + val EXECUTOR_TEMPLATE_FILE = Files.createTempFile("executor-pod-template", "yml").toFile + EXECUTOR_TEMPLATE_FILE.deleteOnExit() +} \ No newline at end of file From 98acd29d2b4c16e03aa62ead9fa2f914d0c0d244 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Fri, 24 Aug 2018 14:54:11 +0100 Subject: [PATCH 28/53] fix up docs --- docs/running-on-kubernetes.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index dd9483c1a407e..1100fd53822e6 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -842,7 +842,7 @@ See the below table for the full list of pod specifications that will be overwri Value of spark.kubernetes.namespace Spark makes strong assumptions about the driver and executor namespaces. Both driver and executor namespaces will - be replaced by this spark conf value. + be replaced by either the configured or default spark conf value. @@ -881,25 +881,25 @@ See the below table for the full list of pod specifications that will be overwri restartPolicy - "never" + "never" Spark assumes that both drivers and executors never restart. serviceAccount - Value of spark.kubernetes.authenticate.driver.serviceAccountName + Value of spark.kubernetes.authenticate.driver.serviceAccountName Spark will override serviceAccount with the value of the spark configuration for only - driver pods. Executor pods will remain unaffected. + driver pods, and only if the spark configuration is specified. Executor pods will remain unaffected. serviceAccountName - Value of spark.kubernetes.authenticate.driver.serviceAccountName + Value of spark.kubernetes.authenticate.driver.serviceAccountName Spark will override serviceAccountName with the value of the spark configuration for only - driver pods. Executor pods will remain unaffected. + driver pods, and only if the spark configuration is specified. Executor pods will remain unaffected. @@ -950,7 +950,7 @@ The following affect the driver and executor containers. All other containers in resources - See description + See description The cpu limits are set by spark.kubernetes.{driver,executor}.limit.cores. The cpu is set by spark.{driver,executor}.cores. The memory request and limit are set by summing the values of From 95f8b8b8f3260f4fad2fa78a933c88c6aadf28c2 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Fri, 24 Aug 2018 14:55:22 +0100 Subject: [PATCH 29/53] rename a variable --- .../spark/deploy/k8s/submit/KubernetesDriverBuilder.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index 9ac0fdb12042d..fc823adf20f90 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -80,7 +80,7 @@ private[spark] class KubernetesDriverBuilder( val volumesFeature = if (kubernetesConf.roleVolumes.nonEmpty) { Seq(provideVolumesStep(kubernetesConf)) } else Nil - val templateVolumeFeature = if ( + val podTemplateFeature = if ( kubernetesConf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) { Seq(podTemplateConfigMapStep(kubernetesConf)) } else Nil @@ -95,7 +95,7 @@ private[spark] class KubernetesDriverBuilder( .getOrElse(provideJavaStep(kubernetesConf)) val allFeatures = (baseFeatures :+ bindingsStep) ++ - secretFeature ++ envSecretFeature ++ volumesFeature ++ templateVolumeFeature + secretFeature ++ envSecretFeature ++ volumesFeature ++ podTemplateFeature var spec = KubernetesDriverSpec( provideInitialPod(), From da5dff5f61c28193b20d085f5b38c3dc722b6355 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Fri, 24 Aug 2018 15:08:57 +0100 Subject: [PATCH 30/53] fix style? --- .../spark/deploy/k8s/integrationtest/PodTemplateSuite.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala index 0b376a2b61729..9fadc24f9117e 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala @@ -27,6 +27,7 @@ private[spark] trait PodTemplateSuite { k8sSuite: KubernetesSuite => import PodTemplateSuite._ test("Start pod creation from template") { + createPodTemplateFiles() sparkAppConf .set("spark.kubernetes.driver.podTemplateFile", DRIVER_TEMPLATE_FILE.getAbsolutePath) .set("spark.kubernetes.executor.podTemplateFile", EXECUTOR_TEMPLATE_FILE.getAbsolutePath) @@ -106,4 +107,4 @@ private[spark] object PodTemplateSuite { val EXECUTOR_TEMPLATE_FILE = Files.createTempFile("executor-pod-template", "yml").toFile EXECUTOR_TEMPLATE_FILE.deleteOnExit() -} \ No newline at end of file +} From 7fb76c76001e696176ad709f92584878c42cc86e Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Sat, 25 Aug 2018 16:44:01 +0100 Subject: [PATCH 31/53] fix docs to remove container name conf and further clarify --- docs/running-on-kubernetes.md | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index 1100fd53822e6..930e4fc078aa2 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -197,6 +197,9 @@ pod template that will always be overwritten by Spark. Therefore, users of this the pod template file only lets Spark start with a template pod instead of an empty pod during the pod-building process. For details, see the [full list](#pod-template-properties) of pod template values that will be overwritten by spark. +Pod template files can also define multiple containers. In such cases, Spark will always assume that the first container in +the list will be the driver or executor container. + ## Introspection and Debugging These are the different ways in which you can investigate a running/completed Spark application, monitor progress, and @@ -787,22 +790,6 @@ specific to Spark on Kubernetes. This sets the major Python version of the docker image used to run the driver and executor containers. Can either be 2 or 3. - - spark.kubernetes.driver.containerName - "spark-kubernetes-driver" - - This sets the driver container name. If you are specifying a driver [pod template](#pod-template), you can match this name to the - driver container name set in the template. - - - - spark.kubernetes.executor.containerName - "spark-kubernetes-executor" - - This sets the executor container name. If you are specifying a an executor [pod template](#pod-template), you can match this name to the - driver container name set in the template. - - spark.kubernetes.driver.podTemplateFile (none) @@ -942,10 +929,11 @@ The following affect the driver and executor containers. All other containers in name - See description. + See description. - The driver and executor container names are defined by spark.kubernetes.{driver,executor}.containerName. - If they don't exist in the pod template file, then an error will be thrown. + The container name will be assigned by spark ("spark-kubernetes-driver" for the driver container, and + "executor" for each executor container) if not defined by the pod template. If the container is defined by the + template, the template's name will be used. From d86bc752a631a1bd96161c05503cc05f468adaa5 Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Sat, 25 Aug 2018 16:44:19 +0100 Subject: [PATCH 32/53] actually add the pod template test --- .../spark/deploy/k8s/integrationtest/KubernetesSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 896a83a5badbb..320ee0b7a9a1f 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -34,7 +34,7 @@ import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBacke private[spark] class KubernetesSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfter with BasicTestsSuite with SecretsTestsSuite - with PythonTestsSuite with ClientModeTestsSuite { + with PythonTestsSuite with ClientModeTestsSuite with PodTemplateSuite { import KubernetesSuite._ From f2720a583389cf712093d5cf7b174c3afc343ace Mon Sep 17 00:00:00 2001 From: Yifei Huang Date: Sat, 25 Aug 2018 17:43:40 +0100 Subject: [PATCH 33/53] remove containerName confs --- .../scala/org/apache/spark/deploy/k8s/Config.scala | 12 ------------ .../org/apache/spark/deploy/k8s/Constants.scala | 2 ++ .../apache/spark/deploy/k8s/KubernetesUtils.scala | 10 ++++++---- .../scala/org/apache/spark/deploy/k8s/SparkPod.scala | 11 +++++++++++ .../deploy/k8s/features/BasicDriverFeatureStep.scala | 4 +++- .../k8s/features/BasicExecutorFeatureStep.scala | 4 +++- .../k8s/submit/KubernetesClientApplication.scala | 2 +- .../deploy/k8s/submit/KubernetesDriverBuilder.scala | 5 +---- .../cluster/k8s/KubernetesClusterManager.scala | 3 +-- .../cluster/k8s/KubernetesExecutorBuilder.scala | 5 +---- .../k8s/features/BasicDriverFeatureStepSuite.scala | 3 +-- .../k8s/submit/KubernetesDriverBuilderSuite.scala | 7 +++---- .../cluster/k8s/KubernetesExecutorBuilderSuite.scala | 5 ++--- .../k8s/integrationtest/PodTemplateSuite.scala | 2 -- 14 files changed, 35 insertions(+), 40 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index 4c5176a946368..22e873b533320 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -225,18 +225,6 @@ private[spark] object Config extends Logging { "Ensure that major Python version is either Python2 or Python3") .createWithDefault("2") - val KUBERNETES_DRIVER_CONTAINER_NAME = - ConfigBuilder("spark.kubernetes.driver.containerName") - .doc("The name of the driver container within the driver pod template file") - .stringConf - .createWithDefault("spark-kubernetes-driver") - - val KUBERNETES_EXECUTOR_CONTAINER_NAME = - ConfigBuilder("spark.kubernetes.executor.containerName") - .doc("The name of the executor container within the executor pod template file") - .stringConf - .createWithDefault("spark-kubernetes-executor") - val KUBERNETES_DRIVER_PODTEMPLATE_FILE = ConfigBuilder("spark.kubernetes.driver.podTemplateFile") .doc("File containing a template pod spec for the driver") diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala index 557047d72b9cd..64eba3f50c7f8 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala @@ -83,5 +83,7 @@ private[spark] object Constants { // Miscellaneous val KUBERNETES_MASTER_INTERNAL_URL = "https://kubernetes.default.svc" + val DEFAULT_DRIVER_CONTAINER_NAME = "spark-kubernetes-driver" + val DEFAULT_EXECUTOR_CONTAINER_NAME = "executor" val MEMORY_OVERHEAD_MIN_MIB = 384L } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index a6e88e598f5e2..bf5dbfe9a5c37 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -18,6 +18,7 @@ package org.apache.spark.deploy.k8s import java.io.File +import io.fabric8.kubernetes.api.model.ContainerBuilder import io.fabric8.kubernetes.client.KubernetesClient import scala.collection.JavaConverters._ @@ -67,13 +68,14 @@ private[spark] object KubernetesUtils extends Logging { def loadPodFromTemplate( kubernetesClient: KubernetesClient, - templateFile: File, - containerName: String): SparkPod = { + templateFile: File): SparkPod = { try { val pod = kubernetesClient.pods().load(templateFile).get() val containers = pod.getSpec.getContainers.asScala - require(containers.map(_.getName).contains(containerName)) - SparkPod(pod, containers.filter(_.getName == containerName).head) + if (containers.nonEmpty) { + SparkPod(pod, new ContainerBuilder().build()) + } + SparkPod(pod, containers.head) } catch { case e: Exception => logError( diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala index 345dd117fd35f..61ab3b8c91875 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala @@ -31,4 +31,15 @@ private[spark] object SparkPod { .build(), new ContainerBuilder().build()) } + + def initialPodWithContainerName(name: String): SparkPod = { + SparkPod( + new PodBuilder() + .withNewMetadata() + .endMetadata() + .withNewSpec() + .endSpec() + .build(), + new ContainerBuilder().withName(name).build()) + } } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala index 5d4dcd8a612d9..ca082c54aab6f 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala @@ -79,8 +79,10 @@ private[spark] class BasicDriverFeatureStep( DEFAULT_BLOCKMANAGER_PORT ) val driverUIPort = SparkUI.getUIPort(conf.sparkConf) + val driverContainerName = if (pod.container.getName == null) DEFAULT_DRIVER_CONTAINER_NAME + else pod.container.getName val driverContainer = new ContainerBuilder(pod.container) - .withName(conf.get(KUBERNETES_DRIVER_CONTAINER_NAME)) + .withName(driverContainerName) .withImage(driverContainerImage) .withImagePullPolicy(conf.imagePullPolicy()) .addNewPort() diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala index 8e56e1d250d90..27ea429644b55 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala @@ -128,8 +128,10 @@ private[spark] class BasicExecutorFeatureStep( .build() } + val executorContainerName = if (pod.container.getName == null) DEFAULT_EXECUTOR_CONTAINER_NAME + else pod.container.getName val executorContainer = new ContainerBuilder(pod.container) - .withName(kubernetesConf.get(KUBERNETES_EXECUTOR_CONTAINER_NAME)) + .withName(executorContainerName) .withImage(executorContainerImage) .withImagePullPolicy(kubernetesConf.imagePullPolicy()) .withNewResources() diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala index c772d824869b2..0cf3524bdc0f0 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala @@ -26,7 +26,7 @@ import scala.util.control.NonFatal import org.apache.spark.SparkConf import org.apache.spark.deploy.SparkApplication -import org.apache.spark.deploy.k8s._ +import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverSpecificConf, KubernetesUtils, SparkKubernetesClientFactory} import org.apache.spark.deploy.k8s.Config._ import org.apache.spark.deploy.k8s.Constants._ import org.apache.spark.internal.Logging diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index fc823adf20f90..a1414215288f9 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -120,10 +120,7 @@ private[spark] object KubernetesDriverBuilder extends Logging { .map(new File(_)) .map(file => new KubernetesDriverBuilder(provideInitialPod = () => { try { - KubernetesUtils.loadPodFromTemplate( - kubernetesClient, - file, - conf.get(Config.KUBERNETES_DRIVER_CONTAINER_NAME)) + KubernetesUtils.loadPodFromTemplate(kubernetesClient, file) } catch { case e: Exception => logError( diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index ca4c3f96dcb3b..6c22280b3eb1d 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -72,8 +72,7 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit if (sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) { KubernetesUtils.loadPodFromTemplate( kubernetesClient, - new File(sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get), - sc.conf.get(KUBERNETES_EXECUTOR_CONTAINER_NAME)) + new File(sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get)) } val requestExecutorsService = ThreadUtils.newDaemonCachedThreadPool( diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala index e617ccc553047..6530621b85690 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala @@ -73,10 +73,7 @@ private[spark] object KubernetesExecutorBuilder { conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) .map(new File(_)) .map(file => new KubernetesExecutorBuilder(provideInitialPod = () => { - KubernetesUtils.loadPodFromTemplate( - kubernetesClient, - file, - conf.get(Config.KUBERNETES_EXECUTOR_CONTAINER_NAME)) + KubernetesUtils.loadPodFromTemplate(kubernetesClient, file) })) .getOrElse(new KubernetesExecutorBuilder()) } diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala index 4d1b51c92f690..459199c71b739 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala @@ -67,7 +67,6 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite { .set(org.apache.spark.internal.config.DRIVER_MEMORY_OVERHEAD, 200L) .set(CONTAINER_IMAGE, "spark-driver:latest") .set(IMAGE_PULL_SECRETS, TEST_IMAGE_PULL_SECRETS.mkString(",")) - .set(KUBERNETES_DRIVER_CONTAINER_NAME, "driver-container") val kubernetesConf = KubernetesConf( sparkConf, emptyDriverSpecificConf, @@ -85,7 +84,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite { val basePod = SparkPod.initialPod() val configuredPod = featureStep.configurePod(basePod) - assert(configuredPod.container.getName === "driver-container") + assert(configuredPod.container.getName === DEFAULT_DRIVER_CONTAINER_NAME) assert(configuredPod.container.getImage === "spark-driver:latest") assert(configuredPod.container.getImagePullPolicy === CONTAINER_IMAGE_PULL_POLICY) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala index 1be37d7ab2268..d30c27b7d01f6 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala @@ -26,7 +26,7 @@ import org.mockito.Mockito._ import org.apache.spark.{SparkConf, SparkException, SparkFunSuite} import org.apache.spark.deploy.k8s._ -import org.apache.spark.deploy.k8s.Config.{CONTAINER_IMAGE, KUBERNETES_DRIVER_CONTAINER_NAME, KUBERNETES_DRIVER_PODTEMPLATE_FILE, KUBERNETES_EXECUTOR_PODTEMPLATE_FILE} +import org.apache.spark.deploy.k8s.Config.{CONTAINER_IMAGE, KUBERNETES_DRIVER_PODTEMPLATE_FILE, KUBERNETES_EXECUTOR_PODTEMPLATE_FILE} import org.apache.spark.deploy.k8s.features.{BasicDriverFeatureStep, DriverKubernetesCredentialsFeatureStep, DriverServiceFeatureStep, EnvSecretsFeatureStep, KubernetesFeaturesTestUtils, LocalDirsFeatureStep, MountSecretsFeatureStep, _} import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep, RDriverFeatureStep} @@ -311,15 +311,14 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { .endMetadata() .withNewSpec() .addNewContainer() - .withName(KUBERNETES_DRIVER_CONTAINER_NAME.defaultValueString) + .withName("test-driver-container") .endContainer() .endSpec() .build()) assert(spec.pod.pod.getMetadata.getLabels.containsKey("test-label-key")) assert(spec.pod.pod.getMetadata.getLabels.get("test-label-key") === "test-label-value") - assert(spec.pod.container.getName === - KUBERNETES_DRIVER_CONTAINER_NAME.defaultValueString) + assert(spec.pod.container.getName === "test-driver-container") } test("Throws on misconfigured pod template") { diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala index 60a7f2eea3a15..f68a85b01ff48 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala @@ -140,15 +140,14 @@ class KubernetesExecutorBuilderSuite extends SparkFunSuite { .endMetadata() .withNewSpec() .addNewContainer() - .withName(Config.KUBERNETES_EXECUTOR_CONTAINER_NAME.defaultValueString) + .withName("driver-container") .endContainer() .endSpec() .build()) assert(pod.pod.getMetadata.getLabels.containsKey("test-label-key")) assert(pod.pod.getMetadata.getLabels.get("test-label-key") === "test-label-value") - assert(pod.container.getName === - Config.KUBERNETES_EXECUTOR_CONTAINER_NAME.defaultValueString) + assert(pod.container.getName === "driver-container") } private def constructPodWithPodTemplate(pod: Pod) : SparkPod = { diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala index 9fadc24f9117e..9a213c1bf1da5 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala @@ -31,8 +31,6 @@ private[spark] trait PodTemplateSuite { k8sSuite: KubernetesSuite => sparkAppConf .set("spark.kubernetes.driver.podTemplateFile", DRIVER_TEMPLATE_FILE.getAbsolutePath) .set("spark.kubernetes.executor.podTemplateFile", EXECUTOR_TEMPLATE_FILE.getAbsolutePath) - .set("spark.kubernetes.driver.containerName", DRIVER_CONTAINER_NAME) - .set("spark.kubernetes.executor.containerName", EXECUTOR_LABEL_VALUE) runSparkPiAndVerifyCompletion( driverPodChecker = (driverPod: Pod) => { checkDriverPod(driverPod) From 4b3950d394b803f30c8fd6525c58554f334579e5 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Mon, 27 Aug 2018 21:00:26 +0100 Subject: [PATCH 34/53] test tag and indent --- .../integrationtest/PodTemplateSuite.scala | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala index 9a213c1bf1da5..0efde87ebb17f 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala @@ -22,11 +22,13 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import io.fabric8.kubernetes.api.model.{Pod, PodBuilder} +import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.k8sTestTag + private[spark] trait PodTemplateSuite { k8sSuite: KubernetesSuite => import PodTemplateSuite._ - test("Start pod creation from template") { + test("Start pod creation from template", k8sTestTag) { createPodTemplateFiles() sparkAppConf .set("spark.kubernetes.driver.podTemplateFile", DRIVER_TEMPLATE_FILE.getAbsolutePath) @@ -63,29 +65,29 @@ private[spark] trait PodTemplateSuite { k8sSuite: KubernetesSuite => .withApiVersion("1") .withKind("Pod") .withNewMetadata() - .addToLabels(LABEL_KEY, DRIVER_LABEL_VALUE) - .endMetadata() + .addToLabels(LABEL_KEY, DRIVER_LABEL_VALUE) + .endMetadata() .withNewSpec() - .addNewContainer() - .withName(DRIVER_CONTAINER_NAME) - .withImage("will-be-overwritten") - .endContainer() - .endSpec() + .addNewContainer() + .withName(DRIVER_CONTAINER_NAME) + .withImage("will-be-overwritten") + .endContainer() + .endSpec() .build() val executorTemplatePod = new PodBuilder() .withApiVersion("1") .withKind("Pod") .withNewMetadata() - .withName("template-pod") - .addToLabels(LABEL_KEY, EXECUTOR_LABEL_VALUE) - .endMetadata() + .withName("template-pod") + .addToLabels(LABEL_KEY, EXECUTOR_LABEL_VALUE) + .endMetadata() .withNewSpec() - .addNewContainer() - .withName(EXECUTOR_CONTAINER_NAME) - .withImage("will-be-overwritten") - .endContainer() - .endSpec() + .addNewContainer() + .withName(EXECUTOR_CONTAINER_NAME) + .withImage("will-be-overwritten") + .endContainer() + .endSpec() .build() objectMapper.writeValue(DRIVER_TEMPLATE_FILE, driverTemplatePod) From 3813fcbb3ca84e95d45fa932f0a7e9f411c3471b Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Tue, 28 Aug 2018 11:58:51 +0100 Subject: [PATCH 35/53] extension --- .../spark/deploy/k8s/integrationtest/PodTemplateSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala index 0efde87ebb17f..7dcc075b7c488 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala @@ -102,9 +102,9 @@ private[spark] object PodTemplateSuite { val DRIVER_LABEL_VALUE = "driver-template-label-value" val EXECUTOR_LABEL_VALUE = "executor-template-label-value" - val DRIVER_TEMPLATE_FILE = Files.createTempFile("driver-pod-template", "yml").toFile + val DRIVER_TEMPLATE_FILE = Files.createTempFile("driver-pod-template", ".yml").toFile DRIVER_TEMPLATE_FILE.deleteOnExit() - val EXECUTOR_TEMPLATE_FILE = Files.createTempFile("executor-pod-template", "yml").toFile + val EXECUTOR_TEMPLATE_FILE = Files.createTempFile("executor-pod-template", ".yml").toFile EXECUTOR_TEMPLATE_FILE.deleteOnExit() } From ec043234d47b63435dd82b8931c03458a08f7361 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Tue, 28 Aug 2018 23:56:05 +0100 Subject: [PATCH 36/53] use resources for integartion tests templates --- .../src/test/resources/driver-template.yml | 10 +++ .../src/test/resources/executor-template.yml | 10 +++ .../integrationtest/PodTemplateSuite.scala | 62 +++---------------- 3 files changed, 28 insertions(+), 54 deletions(-) create mode 100644 resource-managers/kubernetes/integration-tests/src/test/resources/driver-template.yml create mode 100644 resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/driver-template.yml b/resource-managers/kubernetes/integration-tests/src/test/resources/driver-template.yml new file mode 100644 index 0000000000000..6d12a7a2acd1e --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/resources/driver-template.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +Kind: Pod +metadata: + labels: + template-label-key: driver-template-label-value +spec: + containers: + - name: test-driver-container + image: will-be-overwritten + diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml b/resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml new file mode 100644 index 0000000000000..c16be34de720e --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +Kind: Pod +metadata: + name: template-pod + labels: + template-label-key: executor-template-label-value +spec: + containers: + - name: test-executor-container + image: will-be-overwritten diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala index 7dcc075b7c488..44b2a9735dd6d 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala @@ -16,11 +16,9 @@ */ package org.apache.spark.deploy.k8s.integrationtest -import java.nio.file.Files +import java.io.File -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import io.fabric8.kubernetes.api.model.{Pod, PodBuilder} +import io.fabric8.kubernetes.api.model.Pod import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.k8sTestTag @@ -29,7 +27,6 @@ private[spark] trait PodTemplateSuite { k8sSuite: KubernetesSuite => import PodTemplateSuite._ test("Start pod creation from template", k8sTestTag) { - createPodTemplateFiles() sparkAppConf .set("spark.kubernetes.driver.podTemplateFile", DRIVER_TEMPLATE_FILE.getAbsolutePath) .set("spark.kubernetes.executor.podTemplateFile", EXECUTOR_TEMPLATE_FILE.getAbsolutePath) @@ -46,65 +43,22 @@ private[spark] trait PodTemplateSuite { k8sSuite: KubernetesSuite => private def checkDriverPod(pod: Pod): Unit = { assert(pod.getMetadata.getName === driverPodName) assert(pod.getSpec.getContainers.get(0).getImage === image) - assert(pod.getSpec.getContainers.get(0).getName === DRIVER_CONTAINER_NAME) + assert(pod.getSpec.getContainers.get(0).getName === "test-driver-container") assert(pod.getMetadata.getLabels.containsKey(LABEL_KEY)) - assert(pod.getMetadata.getLabels.get(LABEL_KEY) === DRIVER_LABEL_VALUE) + assert(pod.getMetadata.getLabels.get(LABEL_KEY) === "driver-template-label-value") } private def checkExecutorPod(pod: Pod): Unit = { assert(pod.getMetadata.getName === "template-pod") assert(pod.getSpec.getContainers.get(0).getImage === image) - assert(pod.getSpec.getContainers.get(0).getName === EXECUTOR_CONTAINER_NAME) + assert(pod.getSpec.getContainers.get(0).getName === "test-executor-container") assert(pod.getMetadata.getLabels.containsKey(LABEL_KEY)) - assert(pod.getMetadata.getLabels.get(LABEL_KEY) === EXECUTOR_LABEL_VALUE) - } - - private def createPodTemplateFiles(): Unit = { - val objectMapper = new ObjectMapper(new YAMLFactory()) - val driverTemplatePod = new PodBuilder() - .withApiVersion("1") - .withKind("Pod") - .withNewMetadata() - .addToLabels(LABEL_KEY, DRIVER_LABEL_VALUE) - .endMetadata() - .withNewSpec() - .addNewContainer() - .withName(DRIVER_CONTAINER_NAME) - .withImage("will-be-overwritten") - .endContainer() - .endSpec() - .build() - - val executorTemplatePod = new PodBuilder() - .withApiVersion("1") - .withKind("Pod") - .withNewMetadata() - .withName("template-pod") - .addToLabels(LABEL_KEY, EXECUTOR_LABEL_VALUE) - .endMetadata() - .withNewSpec() - .addNewContainer() - .withName(EXECUTOR_CONTAINER_NAME) - .withImage("will-be-overwritten") - .endContainer() - .endSpec() - .build() - - objectMapper.writeValue(DRIVER_TEMPLATE_FILE, driverTemplatePod) - objectMapper.writeValue(EXECUTOR_TEMPLATE_FILE, executorTemplatePod) + assert(pod.getMetadata.getLabels.get(LABEL_KEY) === "executor-template-label-value") } } private[spark] object PodTemplateSuite { - val DRIVER_CONTAINER_NAME = "test-driver-container" - val EXECUTOR_CONTAINER_NAME = "test-executor-container" val LABEL_KEY = "template-label-key" - val DRIVER_LABEL_VALUE = "driver-template-label-value" - val EXECUTOR_LABEL_VALUE = "executor-template-label-value" - - val DRIVER_TEMPLATE_FILE = Files.createTempFile("driver-pod-template", ".yml").toFile - DRIVER_TEMPLATE_FILE.deleteOnExit() - - val EXECUTOR_TEMPLATE_FILE = Files.createTempFile("executor-pod-template", ".yml").toFile - EXECUTOR_TEMPLATE_FILE.deleteOnExit() + val DRIVER_TEMPLATE_FILE = new File(getClass.getResource("driver-template.yml").getFile) + val EXECUTOR_TEMPLATE_FILE = new File(getClass.getResource("executor-template.yml").getFile) } From f3b60822e688a6a16404f5e983e953e3da99ffba Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Wed, 29 Aug 2018 00:03:32 +0100 Subject: [PATCH 37/53] rat --- .../src/test/resources/driver-template.yml | 16 ++++++++++++++++ .../src/test/resources/executor-template.yml | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/driver-template.yml b/resource-managers/kubernetes/integration-tests/src/test/resources/driver-template.yml index 6d12a7a2acd1e..0c185be81d59e 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/resources/driver-template.yml +++ b/resource-managers/kubernetes/integration-tests/src/test/resources/driver-template.yml @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# apiVersion: v1 Kind: Pod metadata: diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml b/resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml index c16be34de720e..b2087b72963ae 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml +++ b/resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# apiVersion: v1 Kind: Pod metadata: From fd503dbbcc313f64bec6ce6468406e0014f82968 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Wed, 29 Aug 2018 11:05:19 +0100 Subject: [PATCH 38/53] fix path --- .../spark/deploy/k8s/integrationtest/PodTemplateSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala index 44b2a9735dd6d..d68b2fd023ce4 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala @@ -59,6 +59,6 @@ private[spark] trait PodTemplateSuite { k8sSuite: KubernetesSuite => private[spark] object PodTemplateSuite { val LABEL_KEY = "template-label-key" - val DRIVER_TEMPLATE_FILE = new File(getClass.getResource("driver-template.yml").getFile) - val EXECUTOR_TEMPLATE_FILE = new File(getClass.getResource("executor-template.yml").getFile) + val DRIVER_TEMPLATE_FILE = new File(getClass.getResource("/driver-template.yml").getFile) + val EXECUTOR_TEMPLATE_FILE = new File(getClass.getResource("/executor-template.yml").getFile) } From eeb2492e9721475518c8b861d07611fd6ab70f0a Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Wed, 29 Aug 2018 13:36:49 +0100 Subject: [PATCH 39/53] prevent having duplicate containers --- .../apache/spark/deploy/k8s/KubernetesUtils.scala | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index bf5dbfe9a5c37..4942296fbe890 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s import java.io.File -import io.fabric8.kubernetes.api.model.ContainerBuilder +import io.fabric8.kubernetes.api.model.{ContainerBuilder, PodBuilder} import io.fabric8.kubernetes.client.KubernetesClient import scala.collection.JavaConverters._ @@ -72,10 +72,14 @@ private[spark] object KubernetesUtils extends Logging { try { val pod = kubernetesClient.pods().load(templateFile).get() val containers = pod.getSpec.getContainers.asScala - if (containers.nonEmpty) { - SparkPod(pod, new ContainerBuilder().build()) - } - SparkPod(pod, containers.head) + containers.headOption.map(firstContainer => { + val podWithoutFirstContainer = new PodBuilder(pod) + .editSpec() + .removeFromContainers(firstContainer) + .endSpec() + .build() + SparkPod(podWithoutFirstContainer, containers.head) + }).getOrElse(SparkPod(pod, new ContainerBuilder().build())) } catch { case e: Exception => logError( From 36a70ad1584ec8a08135759f2d4eaeacb9cce162 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Wed, 29 Aug 2018 16:52:17 +0100 Subject: [PATCH 40/53] do not use broken removeContainer --- .../spark/deploy/k8s/KubernetesUtils.scala | 19 ++++++++++--------- .../k8s/KubernetesExecutorBuilderSuite.scala | 6 ++++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 4942296fbe890..48c2fc20b8323 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -71,15 +71,16 @@ private[spark] object KubernetesUtils extends Logging { templateFile: File): SparkPod = { try { val pod = kubernetesClient.pods().load(templateFile).get() - val containers = pod.getSpec.getContainers.asScala - containers.headOption.map(firstContainer => { - val podWithoutFirstContainer = new PodBuilder(pod) - .editSpec() - .removeFromContainers(firstContainer) - .endSpec() - .build() - SparkPod(podWithoutFirstContainer, containers.head) - }).getOrElse(SparkPod(pod, new ContainerBuilder().build())) + pod.getSpec.getContainers.asScala.toList match { + case first :: rest => SparkPod( + new PodBuilder(pod) + .editSpec() + .withContainers(rest.asJava) + .endSpec() + .build(), + first) + case Nil => SparkPod(pod, new ContainerBuilder().build()) + } } catch { case e: Exception => logError( diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala index f68a85b01ff48..c5b4ecffff14a 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala @@ -23,6 +23,7 @@ import io.fabric8.kubernetes.client.KubernetesClient import io.fabric8.kubernetes.client.dsl.{MixedOperation, PodResource} import org.mockito.Matchers.any import org.mockito.Mockito.{mock, never, verify, when} +import scala.collection.JavaConverters._ import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.deploy.k8s._ @@ -140,14 +141,15 @@ class KubernetesExecutorBuilderSuite extends SparkFunSuite { .endMetadata() .withNewSpec() .addNewContainer() - .withName("driver-container") + .withName("executor-container") .endContainer() .endSpec() .build()) assert(pod.pod.getMetadata.getLabels.containsKey("test-label-key")) + assert(!pod.pod.getSpec.getContainers.asScala.exists(_.getName == "executor-container")) assert(pod.pod.getMetadata.getLabels.get("test-label-key") === "test-label-value") - assert(pod.container.getName === "driver-container") + assert(pod.container.getName === "executor-container") } private def constructPodWithPodTemplate(pod: Pod) : SparkPod = { From ece7a7ca6accd9a5828fadd90523f60cc4ee2c4c Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Wed, 29 Aug 2018 21:32:03 +0100 Subject: [PATCH 41/53] nits --- .../scala/org/apache/spark/deploy/k8s/Constants.scala | 2 +- .../scala/org/apache/spark/deploy/k8s/SparkPod.scala | 11 ----------- .../deploy/k8s/features/BasicDriverFeatureStep.scala | 4 +--- .../k8s/features/BasicExecutorFeatureStep.scala | 4 +--- .../deploy/k8s/submit/KubernetesDriverBuilder.scala | 6 +++--- .../cluster/k8s/KubernetesClusterManager.scala | 2 +- 6 files changed, 7 insertions(+), 22 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala index 64eba3f50c7f8..983af14591151 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala @@ -84,6 +84,6 @@ private[spark] object Constants { // Miscellaneous val KUBERNETES_MASTER_INTERNAL_URL = "https://kubernetes.default.svc" val DEFAULT_DRIVER_CONTAINER_NAME = "spark-kubernetes-driver" - val DEFAULT_EXECUTOR_CONTAINER_NAME = "executor" + val DEFAULT_EXECUTOR_CONTAINER_NAME = "spark-kubernetes-executor" val MEMORY_OVERHEAD_MIN_MIB = 384L } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala index 61ab3b8c91875..345dd117fd35f 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkPod.scala @@ -31,15 +31,4 @@ private[spark] object SparkPod { .build(), new ContainerBuilder().build()) } - - def initialPodWithContainerName(name: String): SparkPod = { - SparkPod( - new PodBuilder() - .withNewMetadata() - .endMetadata() - .withNewSpec() - .endSpec() - .build(), - new ContainerBuilder().withName(name).build()) - } } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala index ca082c54aab6f..9ad1c64ff148a 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala @@ -79,10 +79,8 @@ private[spark] class BasicDriverFeatureStep( DEFAULT_BLOCKMANAGER_PORT ) val driverUIPort = SparkUI.getUIPort(conf.sparkConf) - val driverContainerName = if (pod.container.getName == null) DEFAULT_DRIVER_CONTAINER_NAME - else pod.container.getName val driverContainer = new ContainerBuilder(pod.container) - .withName(driverContainerName) + .withName(Option(pod.container.getName).getOrElse(DEFAULT_DRIVER_CONTAINER_NAME)) .withImage(driverContainerImage) .withImagePullPolicy(conf.imagePullPolicy()) .addNewPort() diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala index 27ea429644b55..07e0c20588bd1 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala @@ -128,10 +128,8 @@ private[spark] class BasicExecutorFeatureStep( .build() } - val executorContainerName = if (pod.container.getName == null) DEFAULT_EXECUTOR_CONTAINER_NAME - else pod.container.getName val executorContainer = new ContainerBuilder(pod.container) - .withName(executorContainerName) + .withName(Option(pod.container.getName).getOrElse(DEFAULT_EXECUTOR_CONTAINER_NAME)) .withImage(executorContainerImage) .withImagePullPolicy(kubernetesConf.imagePullPolicy()) .withNewResources() diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index a1414215288f9..09de225f83792 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -21,7 +21,7 @@ import java.io.File import io.fabric8.kubernetes.client.KubernetesClient import org.apache.spark.{SparkConf, SparkException} -import org.apache.spark.deploy.k8s.{Config, Constants, KubernetesConf, KubernetesDriverSpec, KubernetesDriverSpecificConf, KubernetesRoleSpecificConf, KubernetesUtils, SparkPod} +import org.apache.spark.deploy.k8s.{Config, KubernetesConf, KubernetesDriverSpec, KubernetesDriverSpecificConf, KubernetesRoleSpecificConf, KubernetesUtils, SparkPod} import org.apache.spark.deploy.k8s.features.{BasicDriverFeatureStep, DriverKubernetesCredentialsFeatureStep, DriverServiceFeatureStep, EnvSecretsFeatureStep, LocalDirsFeatureStep, MountSecretsFeatureStep, MountVolumesFeatureStep, PodTemplateConfigMapStep} import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep, RDriverFeatureStep} import org.apache.spark.internal.Logging @@ -58,7 +58,7 @@ private[spark] class KubernetesDriverBuilder( KubernetesConf[KubernetesDriverSpecificConf] => JavaDriverFeatureStep) = new JavaDriverFeatureStep(_), - podTemplateConfigMapStep: (KubernetesConf[_ <: KubernetesRoleSpecificConf] + providePodTemplateConfigMapStep: (KubernetesConf[_ <: KubernetesRoleSpecificConf] => PodTemplateConfigMapStep) = new PodTemplateConfigMapStep(_), provideInitialPod: () => SparkPod = SparkPod.initialPod) { @@ -82,7 +82,7 @@ private[spark] class KubernetesDriverBuilder( } else Nil val podTemplateFeature = if ( kubernetesConf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) { - Seq(podTemplateConfigMapStep(kubernetesConf)) + Seq(providePodTemplateConfigMapStep(kubernetesConf)) } else Nil val bindingsStep = kubernetesConf.roleSpecificConf.mainAppResource.map { diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index 6c22280b3eb1d..547ee3fa83854 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -23,7 +23,7 @@ import com.google.common.cache.CacheBuilder import io.fabric8.kubernetes.client.Config import org.apache.spark.SparkContext -import org.apache.spark.deploy.k8s.{Constants, KubernetesUtils, SparkKubernetesClientFactory, SparkPod} +import org.apache.spark.deploy.k8s.{KubernetesUtils, SparkKubernetesClientFactory} import org.apache.spark.deploy.k8s.Config._ import org.apache.spark.deploy.k8s.Constants._ import org.apache.spark.internal.Logging From 8b8aa48927aa35ba3683ea7eaed093a570721143 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Thu, 30 Aug 2018 10:20:12 +0100 Subject: [PATCH 42/53] inline integration test methods, add volume to executor builder unit tests --- .../k8s/KubernetesExecutorBuilder.scala | 14 ++++++++-- .../k8s/KubernetesExecutorBuilderSuite.scala | 27 +++++++++++++----- .../integrationtest/PodTemplateSuite.scala | 28 +++++++------------ 3 files changed, 41 insertions(+), 28 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala index 6530621b85690..5e7c4179c4071 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala @@ -20,10 +20,11 @@ import java.io.File import io.fabric8.kubernetes.client.KubernetesClient -import org.apache.spark.SparkConf +import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.deploy.k8s._ import org.apache.spark.deploy.k8s.features._ import org.apache.spark.deploy.k8s.features.{BasicExecutorFeatureStep, EnvSecretsFeatureStep, LocalDirsFeatureStep, MountSecretsFeatureStep} +import org.apache.spark.internal.Logging private[spark] class KubernetesExecutorBuilder( provideBasicStep: (KubernetesConf [KubernetesExecutorSpecificConf]) @@ -68,12 +69,19 @@ private[spark] class KubernetesExecutorBuilder( } } -private[spark] object KubernetesExecutorBuilder { +private[spark] object KubernetesExecutorBuilder extends Logging { def apply(kubernetesClient: KubernetesClient, conf: SparkConf): KubernetesExecutorBuilder = { conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) .map(new File(_)) .map(file => new KubernetesExecutorBuilder(provideInitialPod = () => { - KubernetesUtils.loadPodFromTemplate(kubernetesClient, file) + try { + KubernetesUtils.loadPodFromTemplate(kubernetesClient, file) + } catch { + case e: Exception => + logError( + s"Encountered exception while attempting to load initial pod spec from file", e) + throw new SparkException("Could not load executor pod from template file.", e) + } })) .getOrElse(new KubernetesExecutorBuilder()) } diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala index c5b4ecffff14a..9c4b9f3bd0d6f 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala @@ -18,7 +18,7 @@ package org.apache.spark.scheduler.cluster.k8s import java.io.File -import io.fabric8.kubernetes.api.model.{DoneablePod, Pod, PodBuilder, PodList} +import io.fabric8.kubernetes.api.model.{Config => _, _} import io.fabric8.kubernetes.client.KubernetesClient import io.fabric8.kubernetes.client.dsl.{MixedOperation, PodResource} import org.mockito.Matchers.any @@ -137,19 +137,32 @@ class KubernetesExecutorBuilderSuite extends SparkFunSuite { val pod = constructPodWithPodTemplate( new PodBuilder() .withNewMetadata() - .addToLabels("test-label-key", "test-label-value") - .endMetadata() + .addToLabels("test-label-key", "test-label-value") + .endMetadata() .withNewSpec() - .addNewContainer() - .withName("executor-container") - .endContainer() - .endSpec() + .addNewContainer() + .withName("executor-container") + .addToVolumeMounts( + new VolumeMountBuilder() + .withName("test-volume") + .withMountPath("/test") + .build()) + .endContainer() + .addNewVolume() + .withNewHostPath() + .withPath("/test") + .endHostPath() + .withName("test-volume") + .endVolume() + .endSpec() .build()) assert(pod.pod.getMetadata.getLabels.containsKey("test-label-key")) assert(!pod.pod.getSpec.getContainers.asScala.exists(_.getName == "executor-container")) assert(pod.pod.getMetadata.getLabels.get("test-label-key") === "test-label-value") assert(pod.container.getName === "executor-container") + assert(pod.container.getVolumeMounts.asScala.exists(_.getName == "test-volume")) + assert(pod.pod.getSpec.getVolumes.asScala.exists(_.getName == "test-volume")) } private def constructPodWithPodTemplate(pod: Pod) : SparkPod = { diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala index d68b2fd023ce4..fde4fe055d9e7 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala @@ -32,29 +32,21 @@ private[spark] trait PodTemplateSuite { k8sSuite: KubernetesSuite => .set("spark.kubernetes.executor.podTemplateFile", EXECUTOR_TEMPLATE_FILE.getAbsolutePath) runSparkPiAndVerifyCompletion( driverPodChecker = (driverPod: Pod) => { - checkDriverPod(driverPod) + assert(driverPod.getMetadata.getName === driverPodName) + assert(driverPod.getSpec.getContainers.get(0).getImage === image) + assert(driverPod.getSpec.getContainers.get(0).getName === "test-driver-container") + assert(driverPod.getMetadata.getLabels.containsKey(LABEL_KEY)) + assert(driverPod.getMetadata.getLabels.get(LABEL_KEY) === "driver-template-label-value") }, executorPodChecker = (executorPod: Pod) => { - checkExecutorPod(executorPod) + assert(executorPod.getMetadata.getName === "template-pod") + assert(executorPod.getSpec.getContainers.get(0).getImage === image) + assert(executorPod.getSpec.getContainers.get(0).getName === "test-executor-container") + assert(executorPod.getMetadata.getLabels.containsKey(LABEL_KEY)) + assert(executorPod.getMetadata.getLabels.get(LABEL_KEY) === "executor-template-label-value") } ) } - - private def checkDriverPod(pod: Pod): Unit = { - assert(pod.getMetadata.getName === driverPodName) - assert(pod.getSpec.getContainers.get(0).getImage === image) - assert(pod.getSpec.getContainers.get(0).getName === "test-driver-container") - assert(pod.getMetadata.getLabels.containsKey(LABEL_KEY)) - assert(pod.getMetadata.getLabels.get(LABEL_KEY) === "driver-template-label-value") - } - - private def checkExecutorPod(pod: Pod): Unit = { - assert(pod.getMetadata.getName === "template-pod") - assert(pod.getSpec.getContainers.get(0).getImage === image) - assert(pod.getSpec.getContainers.get(0).getName === "test-executor-container") - assert(pod.getMetadata.getLabels.containsKey(LABEL_KEY)) - assert(pod.getMetadata.getLabels.get(LABEL_KEY) === "executor-template-label-value") - } } private[spark] object PodTemplateSuite { From 1ed95ab30eeb833af078af8a7e9beccfb0066bde Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 31 Aug 2018 13:41:46 +0100 Subject: [PATCH 43/53] do not raise twice on template parse failuer --- .../spark/deploy/k8s/KubernetesUtils.scala | 2 +- .../k8s/submit/KubernetesDriverBuilder.scala | 18 +++++------------- .../k8s/KubernetesExecutorBuilder.scala | 16 ++++------------ .../submit/KubernetesDriverBuilderSuite.scala | 2 +- 4 files changed, 11 insertions(+), 27 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 48c2fc20b8323..a327e96c23ca2 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -85,7 +85,7 @@ private[spark] object KubernetesUtils extends Logging { case e: Exception => logError( s"Encountered exception while attempting to load initial pod spec from file", e) - throw new SparkException("Could not load driver pod from template file.", e) + throw new SparkException("Could not load pod from template file.", e) } } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index 09de225f83792..a65a7722136d8 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -20,11 +20,10 @@ import java.io.File import io.fabric8.kubernetes.client.KubernetesClient -import org.apache.spark.{SparkConf, SparkException} +import org.apache.spark.SparkConf import org.apache.spark.deploy.k8s.{Config, KubernetesConf, KubernetesDriverSpec, KubernetesDriverSpecificConf, KubernetesRoleSpecificConf, KubernetesUtils, SparkPod} import org.apache.spark.deploy.k8s.features.{BasicDriverFeatureStep, DriverKubernetesCredentialsFeatureStep, DriverServiceFeatureStep, EnvSecretsFeatureStep, LocalDirsFeatureStep, MountSecretsFeatureStep, MountVolumesFeatureStep, PodTemplateConfigMapStep} import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep, RDriverFeatureStep} -import org.apache.spark.internal.Logging private[spark] class KubernetesDriverBuilder( provideBasicStep: (KubernetesConf[KubernetesDriverSpecificConf]) => BasicDriverFeatureStep = @@ -114,20 +113,13 @@ private[spark] class KubernetesDriverBuilder( } } -private[spark] object KubernetesDriverBuilder extends Logging { +private[spark] object KubernetesDriverBuilder { def apply(kubernetesClient: KubernetesClient, conf: SparkConf): KubernetesDriverBuilder = { conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE) .map(new File(_)) - .map(file => new KubernetesDriverBuilder(provideInitialPod = () => { - try { - KubernetesUtils.loadPodFromTemplate(kubernetesClient, file) - } catch { - case e: Exception => - logError( - s"Encountered exception while attempting to load initial pod spec from file", e) - throw new SparkException("Could not load driver pod from template file.", e) - } - })) + .map(file => new KubernetesDriverBuilder(provideInitialPod = () => + KubernetesUtils.loadPodFromTemplate(kubernetesClient, file) + )) .getOrElse(new KubernetesDriverBuilder()) } } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala index 5e7c4179c4071..9b6e60b7b79ab 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala @@ -20,11 +20,10 @@ import java.io.File import io.fabric8.kubernetes.client.KubernetesClient -import org.apache.spark.{SparkConf, SparkException} +import org.apache.spark.SparkConf import org.apache.spark.deploy.k8s._ import org.apache.spark.deploy.k8s.features._ import org.apache.spark.deploy.k8s.features.{BasicExecutorFeatureStep, EnvSecretsFeatureStep, LocalDirsFeatureStep, MountSecretsFeatureStep} -import org.apache.spark.internal.Logging private[spark] class KubernetesExecutorBuilder( provideBasicStep: (KubernetesConf [KubernetesExecutorSpecificConf]) @@ -69,20 +68,13 @@ private[spark] class KubernetesExecutorBuilder( } } -private[spark] object KubernetesExecutorBuilder extends Logging { +private[spark] object KubernetesExecutorBuilder { def apply(kubernetesClient: KubernetesClient, conf: SparkConf): KubernetesExecutorBuilder = { conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) .map(new File(_)) - .map(file => new KubernetesExecutorBuilder(provideInitialPod = () => { - try { + .map(file => new KubernetesExecutorBuilder(provideInitialPod = () => KubernetesUtils.loadPodFromTemplate(kubernetesClient, file) - } catch { - case e: Exception => - logError( - s"Encountered exception while attempting to load initial pod spec from file", e) - throw new SparkException("Could not load executor pod from template file.", e) - } - })) + )) .getOrElse(new KubernetesExecutorBuilder()) } } diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala index d30c27b7d01f6..ea99ea5812c45 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala @@ -330,7 +330,7 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { .endMetadata() .build()) } - assert(exception.getMessage.contains("Could not load driver pod from template file.")) + assert(exception.getMessage.contains("Could not load pod from template file.")) } private def constructSpecWithPodTemplate(pod: Pod) : KubernetesDriverSpec = { From a4fde0cdc4dc5b64fd3f888244656371eb76f837 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 31 Aug 2018 15:08:25 +0100 Subject: [PATCH 44/53] add comprehensive test for supported template features --- .../k8s/features/BasicDriverFeatureStep.scala | 4 +- .../features/BasicExecutorFeatureStep.scala | 4 +- .../k8s/KubernetesExecutorBuilderSuite.scala | 87 ++++++++++++++++--- 3 files changed, 80 insertions(+), 15 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala index 9ad1c64ff148a..31f601b55ece5 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala @@ -105,7 +105,7 @@ private[spark] class BasicDriverFeatureStep( .withNewFieldRef("v1", "status.podIP") .build()) .endEnv() - .withNewResources() + .editOrNewResources() .addToRequests("cpu", driverCpuQuantity) .addToLimits(maybeCpuLimitQuantity.toMap.asJava) .addToRequests("memory", driverMemoryQuantity) @@ -121,7 +121,7 @@ private[spark] class BasicDriverFeatureStep( .endMetadata() .withNewSpec() .withRestartPolicy("Never") - .withNodeSelector(conf.nodeSelector().asJava) + .addToNodeSelector(conf.nodeSelector().asJava) .addToImagePullSecrets(conf.imagePullSecrets(): _*) .endSpec() .build() diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala index 07e0c20588bd1..50e7db93d2615 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala @@ -132,7 +132,7 @@ private[spark] class BasicExecutorFeatureStep( .withName(Option(pod.container.getName).getOrElse(DEFAULT_EXECUTOR_CONTAINER_NAME)) .withImage(executorContainerImage) .withImagePullPolicy(kubernetesConf.imagePullPolicy()) - .withNewResources() + .editOrNewResources() .addToRequests("memory", executorMemoryQuantity) .addToLimits("memory", executorMemoryQuantity) .addToRequests("cpu", executorCpuQuantity) @@ -170,7 +170,7 @@ private[spark] class BasicExecutorFeatureStep( .editOrNewSpec() .withHostname(hostname) .withRestartPolicy("Never") - .withNodeSelector(kubernetesConf.nodeSelector().asJava) + .addToNodeSelector(kubernetesConf.nodeSelector().asJava) .addToImagePullSecrets(kubernetesConf.imagePullSecrets(): _*) .endSpec() .build() diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala index 9c4b9f3bd0d6f..4d0b8ec342e63 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala @@ -138,31 +138,96 @@ class KubernetesExecutorBuilderSuite extends SparkFunSuite { new PodBuilder() .withNewMetadata() .addToLabels("test-label-key", "test-label-value") + .addToAnnotations("test-annotation-key", "test-annotation-value") + .withNamespace("namespace") + .addNewOwnerReference() + .withController(true) + .withName("owner-reference") + .endOwnerReference() .endMetadata() .withNewSpec() + .withDnsPolicy("dns-policy") + .withHostAliases(new HostAliasBuilder().withHostnames("hostname").build()) + .withImagePullSecrets( + new LocalObjectReferenceBuilder().withName("local-reference").build()) + .withInitContainers(new ContainerBuilder().withName("init-container").build()) + .withNodeName("node-name") + .withNodeSelector(Map("node-selector-key" -> "node-selector-value").asJava) + .withSchedulerName("scheduler") + .withNewSecurityContext() + .withRunAsUser(1000L) + .endSecurityContext() + .withServiceAccount("service-account") + .withSubdomain("subdomain") + .withTolerations(new TolerationBuilder() + .withKey("toleration-key") + .withOperator("Equal") + .withEffect("NoSchedule") + .build()) + .addNewVolume() + .withNewHostPath() + .withPath("/test") + .endHostPath() + .withName("test-volume") + .endVolume() .addNewContainer() + .withArgs("arg") + .withCommand("command") + .addNewEnv() + .withName("env-key") + .withValue("env-value") + .endEnv() + .withImagePullPolicy("Always") .withName("executor-container") + .withNewResources() + .withLimits(Map("gpu" -> new QuantityBuilder().withAmount("1").build()).asJava) + .endResources() + .withNewSecurityContext() + .withRunAsNonRoot(true) + .endSecurityContext() + .withStdin(true) + .withTerminationMessagePath("termination-message-path") + .withTerminationMessagePolicy("termination-message-policy") .addToVolumeMounts( new VolumeMountBuilder() .withName("test-volume") .withMountPath("/test") .build()) .endContainer() - .addNewVolume() - .withNewHostPath() - .withPath("/test") - .endHostPath() - .withName("test-volume") - .endVolume() .endSpec() .build()) - assert(pod.pod.getMetadata.getLabels.containsKey("test-label-key")) - assert(!pod.pod.getSpec.getContainers.asScala.exists(_.getName == "executor-container")) - assert(pod.pod.getMetadata.getLabels.get("test-label-key") === "test-label-value") - assert(pod.container.getName === "executor-container") + val metadata = pod.pod.getMetadata + assert(metadata.getLabels.containsKey("test-label-key")) + assert(metadata.getAnnotations.containsKey("test-annotation-key")) + assert(metadata.getNamespace === "namespace") + assert(metadata.getOwnerReferences.asScala.exists(_.getName == "owner-reference")) + val spec = pod.pod.getSpec + assert(!spec.getContainers.asScala.exists(_.getName == "executor-container")) + assert(spec.getDnsPolicy === "dns-policy") + assert(spec.getHostAliases.asScala.exists(_.getHostnames.asScala.exists(_ == "hostname"))) + assert(spec.getImagePullSecrets.asScala.exists(_.getName == "local-reference")) + assert(spec.getInitContainers.asScala.exists(_.getName == "init-container")) + assert(spec.getNodeName == "node-name") + assert(spec.getNodeSelector.get("node-selector-key") === "node-selector-value") + assert(spec.getSchedulerName === "scheduler") + assert(spec.getSecurityContext.getRunAsUser === 1000L) + assert(spec.getServiceAccount === "service-account") + assert(spec.getSubdomain === "subdomain") + assert(spec.getTolerations.asScala.exists(_.getKey == "toleration-key")) + assert(spec.getVolumes.asScala.exists(_.getName == "test-volume")) + val container = pod.container + assert(container.getName === "executor-container") + assert(container.getArgs.contains("arg")) + assert(container.getCommand.equals(List("command").asJava)) + assert(container.getEnv.asScala.exists(_.getName == "env-key")) + assert(container.getResources.getLimits.get("gpu") === + new QuantityBuilder().withAmount("1").build()) + assert(container.getSecurityContext.getRunAsNonRoot) + assert(container.getStdin) + assert(container.getTerminationMessagePath === "termination-message-path") + assert(container.getTerminationMessagePolicy === "termination-message-policy") assert(pod.container.getVolumeMounts.asScala.exists(_.getName == "test-volume")) - assert(pod.pod.getSpec.getVolumes.asScala.exists(_.getName == "test-volume")) } private def constructPodWithPodTemplate(pod: Pod) : SparkPod = { From 140e89ca8c19049f7852d7f1105bf55022900ddf Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Fri, 31 Aug 2018 17:17:00 +0100 Subject: [PATCH 45/53] generalize tests to cover both driver and executor builders --- .../deploy/k8s/KubernetesDriverSpec.scala | 8 - .../k8s/features/BasicDriverFeatureStep.scala | 2 +- .../submit/KubernetesDriverBuilderSuite.scala | 76 +++++----- .../k8s/submit/PodBuilderSuiteUtils.scala | 142 ++++++++++++++++++ .../k8s/KubernetesExecutorBuilderSuite.scala | 127 ++-------------- 5 files changed, 188 insertions(+), 167 deletions(-) create mode 100644 resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/PodBuilderSuiteUtils.scala diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala index 2deb4e1ccb110..fce8c6a4bf494 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesDriverSpec.scala @@ -22,11 +22,3 @@ private[spark] case class KubernetesDriverSpec( pod: SparkPod, driverKubernetesResources: Seq[HasMetadata], systemProperties: Map[String, String]) - -private[spark] object KubernetesDriverSpec { - def initialSpec(initialConf: KubernetesConf[KubernetesDriverSpecificConf]): KubernetesDriverSpec = - KubernetesDriverSpec( - SparkPod.initialPod(), - Seq.empty, - initialConf.sparkConf.getAll.toMap) -} diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala index 31f601b55ece5..96b14a0d82b4c 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala @@ -119,7 +119,7 @@ private[spark] class BasicDriverFeatureStep( .addToLabels(conf.roleLabels.asJava) .addToAnnotations(conf.roleAnnotations.asJava) .endMetadata() - .withNewSpec() + .editOrNewSpec() .withRestartPolicy("Never") .addToNodeSelector(conf.nodeSelector().asJava) .addToImagePullSecrets(conf.imagePullSecrets(): _*) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala index ea99ea5812c45..6d4078cac46b6 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilderSuite.scala @@ -16,12 +16,8 @@ */ package org.apache.spark.deploy.k8s.submit -import java.io.File - -import io.fabric8.kubernetes.api.model.{DoneablePod, Pod, PodBuilder, PodList} +import io.fabric8.kubernetes.api.model.PodBuilder import io.fabric8.kubernetes.client.KubernetesClient -import io.fabric8.kubernetes.client.dsl.{MixedOperation, PodResource} -import org.mockito.Matchers._ import org.mockito.Mockito._ import org.apache.spark.{SparkConf, SparkException, SparkFunSuite} @@ -304,48 +300,42 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { } test("Starts with template if specified") { - val spec = constructSpecWithPodTemplate( + val kubernetesClient = PodBuilderSuiteUtils.loadingMockKubernetesClient() + val sparkConf = new SparkConf(false) + .set(CONTAINER_IMAGE, "spark-driver:latest") + .set(KUBERNETES_DRIVER_PODTEMPLATE_FILE, "template-file.yaml") + val kubernetesConf = new KubernetesConf( + sparkConf, + KubernetesDriverSpecificConf( + Some(JavaMainAppResource("example.jar")), + "test-app", + "main", + Seq.empty), + "prefix", + "appId", + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Nil, + Seq.empty[String]) + val driverSpec = KubernetesDriverBuilder + .apply(kubernetesClient, sparkConf) + .buildFromFeatures(kubernetesConf) + PodBuilderSuiteUtils.verifyPodWithSupportedFeatures(driverSpec.pod) + } + + test("Throws on misconfigured pod template") { + val kubernetesClient = PodBuilderSuiteUtils.loadingMockKubernetesClient( new PodBuilder() .withNewMetadata() .addToLabels("test-label-key", "test-label-value") .endMetadata() - .withNewSpec() - .addNewContainer() - .withName("test-driver-container") - .endContainer() - .endSpec() .build()) - - assert(spec.pod.pod.getMetadata.getLabels.containsKey("test-label-key")) - assert(spec.pod.pod.getMetadata.getLabels.get("test-label-key") === "test-label-value") - assert(spec.pod.container.getName === "test-driver-container") - } - - test("Throws on misconfigured pod template") { - val exception = intercept[SparkException] { - constructSpecWithPodTemplate( - new PodBuilder() - .withNewMetadata() - .addToLabels("test-label-key", "test-label-value") - .endMetadata() - .build()) - } - assert(exception.getMessage.contains("Could not load pod from template file.")) - } - - private def constructSpecWithPodTemplate(pod: Pod) : KubernetesDriverSpec = { - val kubernetesClient = mock(classOf[KubernetesClient]) - val pods = - mock(classOf[MixedOperation[Pod, PodList, DoneablePod, PodResource[Pod, DoneablePod]]]) - val podResource = mock(classOf[PodResource[Pod, DoneablePod]]) - when(kubernetesClient.pods()).thenReturn(pods) - when(pods.load(any(classOf[File]))).thenReturn(podResource) - when(podResource.get()).thenReturn(pod) - val sparkConf = new SparkConf(false) .set(CONTAINER_IMAGE, "spark-driver:latest") .set(KUBERNETES_DRIVER_PODTEMPLATE_FILE, "template-file.yaml") - val kubernetesConf = new KubernetesConf( sparkConf, KubernetesDriverSpecificConf( @@ -362,7 +352,11 @@ class KubernetesDriverBuilderSuite extends SparkFunSuite { Map.empty, Nil, Seq.empty[String]) - - KubernetesDriverBuilder.apply(kubernetesClient, sparkConf).buildFromFeatures(kubernetesConf) + val exception = intercept[SparkException] { + KubernetesDriverBuilder + .apply(kubernetesClient, sparkConf) + .buildFromFeatures(kubernetesConf) + } + assert(exception.getMessage.contains("Could not load pod from template file.")) } } diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/PodBuilderSuiteUtils.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/PodBuilderSuiteUtils.scala new file mode 100644 index 0000000000000..c92e9e6e3b6b3 --- /dev/null +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/PodBuilderSuiteUtils.scala @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.submit + +import java.io.File + +import io.fabric8.kubernetes.api.model._ +import io.fabric8.kubernetes.client.KubernetesClient +import io.fabric8.kubernetes.client.dsl.{MixedOperation, PodResource} +import org.mockito.Matchers.any +import org.mockito.Mockito.{mock, when} +import org.scalatest.FlatSpec +import scala.collection.JavaConverters._ + +import org.apache.spark.deploy.k8s.SparkPod + +object PodBuilderSuiteUtils extends FlatSpec { + + def loadingMockKubernetesClient(pod: Pod = podWithSupportedFeatures()): KubernetesClient = { + val kubernetesClient = mock(classOf[KubernetesClient]) + val pods = + mock(classOf[MixedOperation[Pod, PodList, DoneablePod, PodResource[Pod, DoneablePod]]]) + val podResource = mock(classOf[PodResource[Pod, DoneablePod]]) + when(kubernetesClient.pods()).thenReturn(pods) + when(pods.load(any(classOf[File]))).thenReturn(podResource) + when(podResource.get()).thenReturn(pod) + kubernetesClient + } + + def verifyPodWithSupportedFeatures(pod: SparkPod): Unit = { + val metadata = pod.pod.getMetadata + assert(metadata.getLabels.containsKey("test-label-key")) + assert(metadata.getAnnotations.containsKey("test-annotation-key")) + assert(metadata.getNamespace === "namespace") + assert(metadata.getOwnerReferences.asScala.exists(_.getName == "owner-reference")) + val spec = pod.pod.getSpec + assert(!spec.getContainers.asScala.exists(_.getName == "executor-container")) + assert(spec.getDnsPolicy === "dns-policy") + assert(spec.getHostAliases.asScala.exists(_.getHostnames.asScala.exists(_ == "hostname"))) + assert(spec.getImagePullSecrets.asScala.exists(_.getName == "local-reference")) + assert(spec.getInitContainers.asScala.exists(_.getName == "init-container")) + assert(spec.getNodeName == "node-name") + assert(spec.getNodeSelector.get("node-selector-key") === "node-selector-value") + assert(spec.getSchedulerName === "scheduler") + assert(spec.getSecurityContext.getRunAsUser === 1000L) + assert(spec.getServiceAccount === "service-account") + assert(spec.getSubdomain === "subdomain") + assert(spec.getTolerations.asScala.exists(_.getKey == "toleration-key")) + assert(spec.getVolumes.asScala.exists(_.getName == "test-volume")) + val container = pod.container + assert(container.getName === "executor-container") + assert(container.getArgs.contains("arg")) + assert(container.getCommand.equals(List("command").asJava)) + assert(container.getEnv.asScala.exists(_.getName == "env-key")) + assert(container.getResources.getLimits.get("gpu") === + new QuantityBuilder().withAmount("1").build()) + assert(container.getSecurityContext.getRunAsNonRoot) + assert(container.getStdin) + assert(container.getTerminationMessagePath === "termination-message-path") + assert(container.getTerminationMessagePolicy === "termination-message-policy") + assert(pod.container.getVolumeMounts.asScala.exists(_.getName == "test-volume")) + + } + + + def podWithSupportedFeatures(): Pod = new PodBuilder() + .withNewMetadata() + .addToLabels("test-label-key", "test-label-value") + .addToAnnotations("test-annotation-key", "test-annotation-value") + .withNamespace("namespace") + .addNewOwnerReference() + .withController(true) + .withName("owner-reference") + .endOwnerReference() + .endMetadata() + .withNewSpec() + .withDnsPolicy("dns-policy") + .withHostAliases(new HostAliasBuilder().withHostnames("hostname").build()) + .withImagePullSecrets( + new LocalObjectReferenceBuilder().withName("local-reference").build()) + .withInitContainers(new ContainerBuilder().withName("init-container").build()) + .withNodeName("node-name") + .withNodeSelector(Map("node-selector-key" -> "node-selector-value").asJava) + .withSchedulerName("scheduler") + .withNewSecurityContext() + .withRunAsUser(1000L) + .endSecurityContext() + .withServiceAccount("service-account") + .withSubdomain("subdomain") + .withTolerations(new TolerationBuilder() + .withKey("toleration-key") + .withOperator("Equal") + .withEffect("NoSchedule") + .build()) + .addNewVolume() + .withNewHostPath() + .withPath("/test") + .endHostPath() + .withName("test-volume") + .endVolume() + .addNewContainer() + .withArgs("arg") + .withCommand("command") + .addNewEnv() + .withName("env-key") + .withValue("env-value") + .endEnv() + .withImagePullPolicy("Always") + .withName("executor-container") + .withNewResources() + .withLimits(Map("gpu" -> new QuantityBuilder().withAmount("1").build()).asJava) + .endResources() + .withNewSecurityContext() + .withRunAsNonRoot(true) + .endSecurityContext() + .withStdin(true) + .withTerminationMessagePath("termination-message-path") + .withTerminationMessagePolicy("termination-message-policy") + .addToVolumeMounts( + new VolumeMountBuilder() + .withName("test-volume") + .withMountPath("/test") + .build()) + .endContainer() + .endSpec() + .build() + +} diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala index 4d0b8ec342e63..f832716d5581f 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilderSuite.scala @@ -16,18 +16,14 @@ */ package org.apache.spark.scheduler.cluster.k8s -import java.io.File - import io.fabric8.kubernetes.api.model.{Config => _, _} import io.fabric8.kubernetes.client.KubernetesClient -import io.fabric8.kubernetes.client.dsl.{MixedOperation, PodResource} -import org.mockito.Matchers.any -import org.mockito.Mockito.{mock, never, verify, when} -import scala.collection.JavaConverters._ +import org.mockito.Mockito.{mock, never, verify} import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.deploy.k8s._ import org.apache.spark.deploy.k8s.features._ +import org.apache.spark.deploy.k8s.submit.PodBuilderSuiteUtils class KubernetesExecutorBuilderSuite extends SparkFunSuite { private val BASIC_STEP_TYPE = "basic" @@ -134,123 +130,18 @@ class KubernetesExecutorBuilderSuite extends SparkFunSuite { } test("Starts with executor template if specified") { - val pod = constructPodWithPodTemplate( - new PodBuilder() - .withNewMetadata() - .addToLabels("test-label-key", "test-label-value") - .addToAnnotations("test-annotation-key", "test-annotation-value") - .withNamespace("namespace") - .addNewOwnerReference() - .withController(true) - .withName("owner-reference") - .endOwnerReference() - .endMetadata() - .withNewSpec() - .withDnsPolicy("dns-policy") - .withHostAliases(new HostAliasBuilder().withHostnames("hostname").build()) - .withImagePullSecrets( - new LocalObjectReferenceBuilder().withName("local-reference").build()) - .withInitContainers(new ContainerBuilder().withName("init-container").build()) - .withNodeName("node-name") - .withNodeSelector(Map("node-selector-key" -> "node-selector-value").asJava) - .withSchedulerName("scheduler") - .withNewSecurityContext() - .withRunAsUser(1000L) - .endSecurityContext() - .withServiceAccount("service-account") - .withSubdomain("subdomain") - .withTolerations(new TolerationBuilder() - .withKey("toleration-key") - .withOperator("Equal") - .withEffect("NoSchedule") - .build()) - .addNewVolume() - .withNewHostPath() - .withPath("/test") - .endHostPath() - .withName("test-volume") - .endVolume() - .addNewContainer() - .withArgs("arg") - .withCommand("command") - .addNewEnv() - .withName("env-key") - .withValue("env-value") - .endEnv() - .withImagePullPolicy("Always") - .withName("executor-container") - .withNewResources() - .withLimits(Map("gpu" -> new QuantityBuilder().withAmount("1").build()).asJava) - .endResources() - .withNewSecurityContext() - .withRunAsNonRoot(true) - .endSecurityContext() - .withStdin(true) - .withTerminationMessagePath("termination-message-path") - .withTerminationMessagePolicy("termination-message-policy") - .addToVolumeMounts( - new VolumeMountBuilder() - .withName("test-volume") - .withMountPath("/test") - .build()) - .endContainer() - .endSpec() - .build()) - - val metadata = pod.pod.getMetadata - assert(metadata.getLabels.containsKey("test-label-key")) - assert(metadata.getAnnotations.containsKey("test-annotation-key")) - assert(metadata.getNamespace === "namespace") - assert(metadata.getOwnerReferences.asScala.exists(_.getName == "owner-reference")) - val spec = pod.pod.getSpec - assert(!spec.getContainers.asScala.exists(_.getName == "executor-container")) - assert(spec.getDnsPolicy === "dns-policy") - assert(spec.getHostAliases.asScala.exists(_.getHostnames.asScala.exists(_ == "hostname"))) - assert(spec.getImagePullSecrets.asScala.exists(_.getName == "local-reference")) - assert(spec.getInitContainers.asScala.exists(_.getName == "init-container")) - assert(spec.getNodeName == "node-name") - assert(spec.getNodeSelector.get("node-selector-key") === "node-selector-value") - assert(spec.getSchedulerName === "scheduler") - assert(spec.getSecurityContext.getRunAsUser === 1000L) - assert(spec.getServiceAccount === "service-account") - assert(spec.getSubdomain === "subdomain") - assert(spec.getTolerations.asScala.exists(_.getKey == "toleration-key")) - assert(spec.getVolumes.asScala.exists(_.getName == "test-volume")) - val container = pod.container - assert(container.getName === "executor-container") - assert(container.getArgs.contains("arg")) - assert(container.getCommand.equals(List("command").asJava)) - assert(container.getEnv.asScala.exists(_.getName == "env-key")) - assert(container.getResources.getLimits.get("gpu") === - new QuantityBuilder().withAmount("1").build()) - assert(container.getSecurityContext.getRunAsNonRoot) - assert(container.getStdin) - assert(container.getTerminationMessagePath === "termination-message-path") - assert(container.getTerminationMessagePolicy === "termination-message-policy") - assert(pod.container.getVolumeMounts.asScala.exists(_.getName == "test-volume")) - } - - private def constructPodWithPodTemplate(pod: Pod) : SparkPod = { - val kubernetesClient = mock(classOf[KubernetesClient]) - val pods = - mock(classOf[MixedOperation[Pod, PodList, DoneablePod, PodResource[Pod, DoneablePod]]]) - val podResource = mock(classOf[PodResource[Pod, DoneablePod]]) - when(kubernetesClient.pods()).thenReturn(pods) - when(pods.load(any(classOf[File]))).thenReturn(podResource) - when(podResource.get()).thenReturn(pod) - + val kubernetesClient = PodBuilderSuiteUtils.loadingMockKubernetesClient() val sparkConf = new SparkConf(false) .set("spark.driver.host", "https://driver.host.com") .set(Config.CONTAINER_IMAGE, "spark-executor:latest") .set(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE, "template-file.yaml") - val kubernetesConf = KubernetesConf( sparkConf, KubernetesExecutorSpecificConf( "executor-id", Some(new PodBuilder() - .withNewMetadata() - .withName("driver") - .endMetadata() + .withNewMetadata() + .withName("driver") + .endMetadata() .build())), "prefix", "appId", @@ -261,7 +152,9 @@ class KubernetesExecutorBuilderSuite extends SparkFunSuite { Map.empty, Nil, Seq.empty[String]) - - KubernetesExecutorBuilder.apply(kubernetesClient, sparkConf).buildFromFeatures(kubernetesConf) + val sparkPod = KubernetesExecutorBuilder + .apply(kubernetesClient, sparkConf) + .buildFromFeatures(kubernetesConf) + PodBuilderSuiteUtils.verifyPodWithSupportedFeatures(sparkPod) } } From 838c2bdb8606a6dceec6805682ad95afd4d2b22f Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Tue, 4 Sep 2018 17:15:54 +0100 Subject: [PATCH 46/53] docs --- docs/running-on-kubernetes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index f07bdc5fb763d..2a9dc39ca43c2 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -191,6 +191,7 @@ Spark users can similarly use template files to define the driver or executor po To do so, specify the spark properties `spark.kubernetes.driver.podTemplateFile` and `spark.kubernetes.executor.podTemplateFile` to point to local files accessible to the `spark-submit` process. To allow the driver pod access the executor pod template file, the file will be automatically mounted onto a volume in the driver pod when it's created. +Spark does not do any validation after unmarshalling these template files and relies on the Kubernetes API server for validation. It is important to note that Spark is opinionated about certain pod configurations so there are values in the pod template that will always be overwritten by Spark. Therefore, users of this feature should note that specifying From 9e6a4b2db16daad02d4c51d25b5e4ae1f4582292 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Mon, 29 Oct 2018 13:45:30 +0000 Subject: [PATCH 47/53] fix tests, templates does not support changing executor pod names --- .../integration-tests/src/test/resources/executor-template.yml | 1 - .../spark/deploy/k8s/integrationtest/PodTemplateSuite.scala | 1 - 2 files changed, 2 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml b/resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml index b2087b72963ae..0282e23a39bd2 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml +++ b/resource-managers/kubernetes/integration-tests/src/test/resources/executor-template.yml @@ -17,7 +17,6 @@ apiVersion: v1 Kind: Pod metadata: - name: template-pod labels: template-label-key: executor-template-label-value spec: diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala index fde4fe055d9e7..e5a847e7210cb 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PodTemplateSuite.scala @@ -39,7 +39,6 @@ private[spark] trait PodTemplateSuite { k8sSuite: KubernetesSuite => assert(driverPod.getMetadata.getLabels.get(LABEL_KEY) === "driver-template-label-value") }, executorPodChecker = (executorPod: Pod) => { - assert(executorPod.getMetadata.getName === "template-pod") assert(executorPod.getSpec.getContainers.get(0).getImage === image) assert(executorPod.getSpec.getContainers.get(0).getName === "test-executor-container") assert(executorPod.getMetadata.getLabels.containsKey(LABEL_KEY)) From c8077dcd4e74cbfd5d50605c5174662ed23a497f Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Mon, 29 Oct 2018 15:33:04 +0000 Subject: [PATCH 48/53] config to select spark containers in pod templates --- .../org/apache/spark/deploy/k8s/Config.scala | 12 ++++ .../spark/deploy/k8s/KubernetesUtils.scala | 38 ++++++++--- .../k8s/submit/KubernetesDriverBuilder.scala | 5 +- .../k8s/KubernetesClusterManager.scala | 3 +- .../k8s/KubernetesExecutorBuilder.scala | 5 +- .../deploy/k8s/KubernetesUtilsSuite.scala | 68 +++++++++++++++++++ 6 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesUtilsSuite.scala diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index 54ff869643e8d..862f1d63ed39f 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -290,6 +290,18 @@ private[spark] object Config extends Logging { .stringConf .createOptional + val KUBERNETES_DRIVER_PODTEMPLATE_CONTAINER_NAME = + ConfigBuilder("spark.kubernetes.driver.podTemplateContainerName") + .doc("container name to be used as a basis for the driver in the given pod template") + .stringConf + .createOptional + + val KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME = + ConfigBuilder("spark.kubernetes.executor.podTemplateContainerName") + .doc("container name to be used as a basis for executors in the given pod template") + .stringConf + .createOptional + val KUBERNETES_AUTH_SUBMISSION_CONF_PREFIX = "spark.kubernetes.authenticate.submission" diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 26d8619be98e6..3f9d85df878df 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -88,19 +88,11 @@ private[spark] object KubernetesUtils extends Logging { def loadPodFromTemplate( kubernetesClient: KubernetesClient, - templateFile: File): SparkPod = { + templateFile: File, + containerName: Option[String]): SparkPod = { try { val pod = kubernetesClient.pods().load(templateFile).get() - pod.getSpec.getContainers.asScala.toList match { - case first :: rest => SparkPod( - new PodBuilder(pod) - .editSpec() - .withContainers(rest.asJava) - .endSpec() - .build(), - first) - case Nil => SparkPod(pod, new ContainerBuilder().build()) - } + selectSparkContainer(pod, containerName) } catch { case e: Exception => logError( @@ -109,6 +101,30 @@ private[spark] object KubernetesUtils extends Logging { } } + def selectSparkContainer(pod: Pod, containerName: Option[String]): SparkPod = { + val containers = pod.getSpec.getContainers.asScala.toList + containerName.flatMap(name => containers.partition(_.getName == name) match { + case (sparkContainer :: Nil, rest) => Some((sparkContainer, rest)) + case _ => + logWarning( + s"specified container ${name} not found on pod template, " + + s"falling back to taking the first container") + Option.empty + }).orElse(containers match { + case first :: rest => Some((first, rest)) + case _ => Option.empty + }).map { + case (sparkContainer: Container, rest: List[Container]) => SparkPod( + new PodBuilder(pod) + .editSpec() + .withContainers(rest.asJava) + .endSpec() + .build(), + sparkContainer) + case _ => SparkPod(pod, new ContainerBuilder().build()) + }.getOrElse(SparkPod(pod, new ContainerBuilder().build())) + } + def parseMasterUrl(url: String): String = url.substring("k8s://".length) def formatPairsBundle(pairs: Seq[(String, String)], indent: Int = 1) : String = { diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index ee823064733ec..4443067d8ec76 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -128,7 +128,10 @@ private[spark] object KubernetesDriverBuilder { conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_FILE) .map(new File(_)) .map(file => new KubernetesDriverBuilder(provideInitialPod = () => - KubernetesUtils.loadPodFromTemplate(kubernetesClient, file) + KubernetesUtils.loadPodFromTemplate( + kubernetesClient, + file, + conf.get(Config.KUBERNETES_DRIVER_PODTEMPLATE_CONTAINER_NAME)) )) .getOrElse(new KubernetesDriverBuilder()) } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index 547ee3fa83854..ce10f766334ff 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -72,7 +72,8 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit if (sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) { KubernetesUtils.loadPodFromTemplate( kubernetesClient, - new File(sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get)) + new File(sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).get), + sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME)) } val requestExecutorsService = ThreadUtils.newDaemonCachedThreadPool( diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala index 3f194f889403d..089f84dec277f 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesExecutorBuilder.scala @@ -104,7 +104,10 @@ private[spark] object KubernetesExecutorBuilder { conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE) .map(new File(_)) .map(file => new KubernetesExecutorBuilder(provideInitialPod = () => - KubernetesUtils.loadPodFromTemplate(kubernetesClient, file) + KubernetesUtils.loadPodFromTemplate( + kubernetesClient, + file, + conf.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME)) )) .getOrElse(new KubernetesExecutorBuilder()) } diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesUtilsSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesUtilsSuite.scala new file mode 100644 index 0000000000000..7c231586af935 --- /dev/null +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesUtilsSuite.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.deploy.k8s + +import scala.collection.JavaConverters._ + +import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, PodBuilder} + +import org.apache.spark.SparkFunSuite + +class KubernetesUtilsSuite extends SparkFunSuite { + private val HOST = "test-host" + private val POD = new PodBuilder() + .withNewSpec() + .withHostname(HOST) + .withContainers( + new ContainerBuilder().withName("first").build(), + new ContainerBuilder().withName("second").build()) + .endSpec() + .build() + + test("Selects the given container as spark container.") { + val sparkPod = KubernetesUtils.selectSparkContainer(POD, Some("second")) + assert(sparkPod.pod.getSpec.getHostname == HOST) + assert(sparkPod.pod.getSpec.getContainers.asScala.toList.map(_.getName) == List("first")) + assert(sparkPod.container.getName == "second") + } + + test("Selects the first container if no container name is given.") { + val sparkPod = KubernetesUtils.selectSparkContainer(POD, Option.empty) + assert(sparkPod.pod.getSpec.getHostname == HOST) + assert(sparkPod.pod.getSpec.getContainers.asScala.toList.map(_.getName) == List("second")) + assert(sparkPod.container.getName == "first") + } + + test("Falls back to the first container if given container name does not exist.") { + val sparkPod = KubernetesUtils.selectSparkContainer(POD, Some("does-not-exist")) + assert(sparkPod.pod.getSpec.getHostname == HOST) + assert(sparkPod.pod.getSpec.getContainers.asScala.toList.map(_.getName) == List("second")) + assert(sparkPod.container.getName == "first") + } + + test("constructs spark pod correctly with pod template with no containers") { + val noContainersPod = new PodBuilder(POD).editSpec().withContainers().endSpec().build() + val sparkPod = KubernetesUtils.selectSparkContainer(noContainersPod, Some("does-not-exist")) + assert(sparkPod.pod.getSpec.getHostname == HOST) + assert(sparkPod.container.getName == null) + val sparkPodWithNoContainerName = + KubernetesUtils.selectSparkContainer(noContainersPod, Option.empty) + assert(sparkPodWithNoContainerName.pod.getSpec.getHostname == HOST) + assert(sparkPodWithNoContainerName.container.getName == null) + } +} From 3d6ff3be364b1a8f2df6e3ec40b53d41eb4d802e Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Mon, 29 Oct 2018 16:18:24 +0000 Subject: [PATCH 49/53] more readable select container logic --- .../spark/deploy/k8s/KubernetesUtils.scala | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 3f9d85df878df..9ecc55fa30ac8 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -102,27 +102,29 @@ private[spark] object KubernetesUtils extends Logging { } def selectSparkContainer(pod: Pod, containerName: Option[String]): SparkPod = { + def selectNamedContainer( + containers: List[Container], name: String): Option[(Container, List[Container])] = + containers.partition(_.getName == name) match { + case (sparkContainer :: Nil, rest) => Some((sparkContainer, rest)) + case _ => + logWarning( + s"specified container ${name} not found on pod template, " + + s"falling back to taking the first container") + Option.empty + } val containers = pod.getSpec.getContainers.asScala.toList - containerName.flatMap(name => containers.partition(_.getName == name) match { - case (sparkContainer :: Nil, rest) => Some((sparkContainer, rest)) - case _ => - logWarning( - s"specified container ${name} not found on pod template, " + - s"falling back to taking the first container") - Option.empty - }).orElse(containers match { - case first :: rest => Some((first, rest)) - case _ => Option.empty - }).map { - case (sparkContainer: Container, rest: List[Container]) => SparkPod( - new PodBuilder(pod) - .editSpec() - .withContainers(rest.asJava) - .endSpec() - .build(), - sparkContainer) - case _ => SparkPod(pod, new ContainerBuilder().build()) - }.getOrElse(SparkPod(pod, new ContainerBuilder().build())) + containerName + .flatMap(selectNamedContainer(containers, _)) + .orElse(containers.headOption.map((_, containers.tail))) + .map { + case (sparkContainer: Container, rest: List[Container]) => SparkPod( + new PodBuilder(pod) + .editSpec() + .withContainers(rest.asJava) + .endSpec() + .build(), + sparkContainer) + }.getOrElse(SparkPod(pod, new ContainerBuilder().build())) } def parseMasterUrl(url: String): String = url.substring("k8s://".length) From 83087eb55dde12ecf3a16523f8802c6fd3f8cc1b Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Mon, 29 Oct 2018 16:31:40 +0000 Subject: [PATCH 50/53] fix integration tests --- .../spark/deploy/k8s/integrationtest/KubernetesSuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 5ce433b9b7b46..e2e5880255e2c 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -288,21 +288,21 @@ private[spark] class KubernetesSuite extends SparkFunSuite protected def doBasicExecutorPodCheck(executorPod: Pod): Unit = { assert(executorPod.getSpec.getContainers.get(0).getImage === image) - assert(executorPod.getSpec.getContainers.get(0).getName === "executor") + assert(executorPod.getSpec.getContainers.get(0).getName === "spark-kubernetes-executor") assert(executorPod.getSpec.getContainers.get(0).getResources.getRequests.get("memory").getAmount === baseMemory) } protected def doBasicExecutorPyPodCheck(executorPod: Pod): Unit = { assert(executorPod.getSpec.getContainers.get(0).getImage === pyImage) - assert(executorPod.getSpec.getContainers.get(0).getName === "executor") + assert(executorPod.getSpec.getContainers.get(0).getName === "spark-kubernetes-executor") assert(executorPod.getSpec.getContainers.get(0).getResources.getRequests.get("memory").getAmount === standardNonJVMMemory) } protected def doBasicExecutorRPodCheck(executorPod: Pod): Unit = { assert(executorPod.getSpec.getContainers.get(0).getImage === rImage) - assert(executorPod.getSpec.getContainers.get(0).getName === "executor") + assert(executorPod.getSpec.getContainers.get(0).getName === "spark-kubernetes-executor") assert(executorPod.getSpec.getContainers.get(0).getResources.getRequests.get("memory").getAmount === standardNonJVMMemory) } From 80b56c1200e2674afe911ba7c0cfec7d85777cd6 Mon Sep 17 00:00:00 2001 From: onursatici Date: Mon, 29 Oct 2018 20:05:34 +0000 Subject: [PATCH 51/53] address comments --- .../main/scala/org/apache/spark/deploy/k8s/Constants.scala | 2 +- .../spark/deploy/k8s/features/PodTemplateConfigMapStep.scala | 4 ++-- .../spark/deploy/k8s/submit/KubernetesDriverBuilder.scala | 2 +- .../deploy/k8s/features/PodTemplateConfigMapStepSuite.scala | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala index afb102f624353..d41deae29ad50 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala @@ -78,7 +78,7 @@ private[spark] object Constants { // Pod spec templates val EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME = "pod-spec-template.yml" - val EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH = "/opt/spark/pod-template" + val EXECUTOR_POD_SPEC_TEMPLATE_MOUNTPATH = "/opt/spark/pod-template" val POD_TEMPLATE_VOLUME = "podspec-volume" val POD_TEMPLATE_CONFIGMAP = "podspec-configmap" val POD_TEMPLATE_KEY = "podspec-configmap-key" diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala index 52baed636c3f4..96a8013246b74 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala @@ -48,7 +48,7 @@ private[spark] class PodTemplateConfigMapStep( val containerWithVolume = new ContainerBuilder(pod.container) .addNewVolumeMount() .withName(POD_TEMPLATE_VOLUME) - .withMountPath(EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH) + .withMountPath(EXECUTOR_POD_SPEC_TEMPLATE_MOUNTPATH) .endVolumeMount() .build() SparkPod(podWithVolume, containerWithVolume) @@ -56,7 +56,7 @@ private[spark] class PodTemplateConfigMapStep( def getAdditionalPodSystemProperties(): Map[String, String] = Map[String, String]( KUBERNETES_EXECUTOR_PODTEMPLATE_FILE.key -> - (EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH + "/" + EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME)) + (EXECUTOR_POD_SPEC_TEMPLATE_MOUNTPATH + "/" + EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME)) def getAdditionalKubernetesResources(): Seq[HasMetadata] = { require(conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE).isDefined) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index 4443067d8ec76..f1bfd7391bf0c 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -108,7 +108,7 @@ private[spark] class KubernetesDriverBuilder( var spec = KubernetesDriverSpec( provideInitialPod(), - Seq.empty, + driverKubernetesResources = Seq.empty, kubernetesConf.sparkConf.getAll.toMap) for (feature <- allFeatures) { val configuredPod = feature.configurePod(spec.pod) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala index 632eb65f7dcd4..d7bbbd121af72 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala @@ -75,7 +75,7 @@ class PodTemplateConfigMapStepSuite extends SparkFunSuite with BeforeAndAfter { assert(configuredPod.container.getVolumeMounts.size() === 1) val volumeMount = configuredPod.container.getVolumeMounts.get(0) - assert(volumeMount.getMountPath === Constants.EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH) + assert(volumeMount.getMountPath === Constants.EXECUTOR_POD_SPEC_TEMPLATE_MOUNTPATH) assert(volumeMount.getName === Constants.POD_TEMPLATE_VOLUME) val resources = step.getAdditionalKubernetesResources() @@ -91,7 +91,7 @@ class PodTemplateConfigMapStepSuite extends SparkFunSuite with BeforeAndAfter { assert(systemProperties.size === 1) assert(systemProperties.contains(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE.key)) assert(systemProperties.get(Config.KUBERNETES_EXECUTOR_PODTEMPLATE_FILE.key).get === - (Constants.EXECUTOR_POD_SPEC_TEMPLATE_MOUNTHPATH + "/" + + (Constants.EXECUTOR_POD_SPEC_TEMPLATE_MOUNTPATH + "/" + Constants.EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME)) } } From 8f7f57197cd17f1322e760efd0c0c8699713ede7 Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Tue, 30 Oct 2018 16:16:17 +0000 Subject: [PATCH 52/53] rename pod template volume name --- .../src/main/scala/org/apache/spark/deploy/k8s/Constants.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala index d41deae29ad50..1c6d53c16871e 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala @@ -79,7 +79,7 @@ private[spark] object Constants { // Pod spec templates val EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME = "pod-spec-template.yml" val EXECUTOR_POD_SPEC_TEMPLATE_MOUNTPATH = "/opt/spark/pod-template" - val POD_TEMPLATE_VOLUME = "podspec-volume" + val POD_TEMPLATE_VOLUME = "pod-template-volume" val POD_TEMPLATE_CONFIGMAP = "podspec-configmap" val POD_TEMPLATE_KEY = "podspec-configmap-key" From 3707e6a411cdb8410365cde8e79b8d3ec01a7fee Mon Sep 17 00:00:00 2001 From: Onur Satici Date: Tue, 30 Oct 2018 16:22:54 +0000 Subject: [PATCH 53/53] imports --- .../scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala | 2 +- .../spark/deploy/k8s/submit/KubernetesDriverBuilder.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 9ecc55fa30ac8..6fafac3ee13c9 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -20,7 +20,7 @@ import java.io.File import scala.collection.JavaConverters._ -import io.fabric8.kubernetes.api.model._ +import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, PodBuilder} import io.fabric8.kubernetes.client.KubernetesClient import org.apache.spark.{SparkConf, SparkException} diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala index f1bfd7391bf0c..5565cd74280e6 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala @@ -21,7 +21,7 @@ import java.io.File import io.fabric8.kubernetes.client.KubernetesClient import org.apache.spark.SparkConf -import org.apache.spark.deploy.k8s._ +import org.apache.spark.deploy.k8s.{Config, KubernetesConf, KubernetesDriverSpec, KubernetesDriverSpecificConf, KubernetesRoleSpecificConf, KubernetesUtils, SparkPod} import org.apache.spark.deploy.k8s.features._ import org.apache.spark.deploy.k8s.features.bindings.{JavaDriverFeatureStep, PythonDriverFeatureStep, RDriverFeatureStep}