From 98ad48fbf92e3cc22f48089b282e215c199b2792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 10 Jan 2024 18:57:40 +0100 Subject: [PATCH 1/4] ./dev/test-dependencies.sh --replace-manifest --- dev/deps/spark-deps-hadoop-3-hive-2.3 | 50 +++++++++++++-------------- pom.xml | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index d5f56bb6a4388..a064747799a53 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -148,31 +148,31 @@ jsr305/3.0.0//jsr305-3.0.0.jar jta/1.1//jta-1.1.jar jul-to-slf4j/2.0.10//jul-to-slf4j-2.0.10.jar kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar -kubernetes-client-api/6.9.1//kubernetes-client-api-6.9.1.jar -kubernetes-client/6.9.1//kubernetes-client-6.9.1.jar -kubernetes-httpclient-okhttp/6.9.1//kubernetes-httpclient-okhttp-6.9.1.jar -kubernetes-model-admissionregistration/6.9.1//kubernetes-model-admissionregistration-6.9.1.jar -kubernetes-model-apiextensions/6.9.1//kubernetes-model-apiextensions-6.9.1.jar -kubernetes-model-apps/6.9.1//kubernetes-model-apps-6.9.1.jar -kubernetes-model-autoscaling/6.9.1//kubernetes-model-autoscaling-6.9.1.jar -kubernetes-model-batch/6.9.1//kubernetes-model-batch-6.9.1.jar -kubernetes-model-certificates/6.9.1//kubernetes-model-certificates-6.9.1.jar -kubernetes-model-common/6.9.1//kubernetes-model-common-6.9.1.jar -kubernetes-model-coordination/6.9.1//kubernetes-model-coordination-6.9.1.jar -kubernetes-model-core/6.9.1//kubernetes-model-core-6.9.1.jar -kubernetes-model-discovery/6.9.1//kubernetes-model-discovery-6.9.1.jar -kubernetes-model-events/6.9.1//kubernetes-model-events-6.9.1.jar -kubernetes-model-extensions/6.9.1//kubernetes-model-extensions-6.9.1.jar -kubernetes-model-flowcontrol/6.9.1//kubernetes-model-flowcontrol-6.9.1.jar -kubernetes-model-gatewayapi/6.9.1//kubernetes-model-gatewayapi-6.9.1.jar -kubernetes-model-metrics/6.9.1//kubernetes-model-metrics-6.9.1.jar -kubernetes-model-networking/6.9.1//kubernetes-model-networking-6.9.1.jar -kubernetes-model-node/6.9.1//kubernetes-model-node-6.9.1.jar -kubernetes-model-policy/6.9.1//kubernetes-model-policy-6.9.1.jar -kubernetes-model-rbac/6.9.1//kubernetes-model-rbac-6.9.1.jar -kubernetes-model-resource/6.9.1//kubernetes-model-resource-6.9.1.jar -kubernetes-model-scheduling/6.9.1//kubernetes-model-scheduling-6.9.1.jar -kubernetes-model-storageclass/6.9.1//kubernetes-model-storageclass-6.9.1.jar +kubernetes-client-api/6.10.0//kubernetes-client-api-6.10.0.jar +kubernetes-client/6.10.0//kubernetes-client-6.10.0.jar +kubernetes-httpclient-okhttp/6.10.0//kubernetes-httpclient-okhttp-6.10.0.jar +kubernetes-model-admissionregistration/6.10.0//kubernetes-model-admissionregistration-6.10.0.jar +kubernetes-model-apiextensions/6.10.0//kubernetes-model-apiextensions-6.10.0.jar +kubernetes-model-apps/6.10.0//kubernetes-model-apps-6.10.0.jar +kubernetes-model-autoscaling/6.10.0//kubernetes-model-autoscaling-6.10.0.jar +kubernetes-model-batch/6.10.0//kubernetes-model-batch-6.10.0.jar +kubernetes-model-certificates/6.10.0//kubernetes-model-certificates-6.10.0.jar +kubernetes-model-common/6.10.0//kubernetes-model-common-6.10.0.jar +kubernetes-model-coordination/6.10.0//kubernetes-model-coordination-6.10.0.jar +kubernetes-model-core/6.10.0//kubernetes-model-core-6.10.0.jar +kubernetes-model-discovery/6.10.0//kubernetes-model-discovery-6.10.0.jar +kubernetes-model-events/6.10.0//kubernetes-model-events-6.10.0.jar +kubernetes-model-extensions/6.10.0//kubernetes-model-extensions-6.10.0.jar +kubernetes-model-flowcontrol/6.10.0//kubernetes-model-flowcontrol-6.10.0.jar +kubernetes-model-gatewayapi/6.10.0//kubernetes-model-gatewayapi-6.10.0.jar +kubernetes-model-metrics/6.10.0//kubernetes-model-metrics-6.10.0.jar +kubernetes-model-networking/6.10.0//kubernetes-model-networking-6.10.0.jar +kubernetes-model-node/6.10.0//kubernetes-model-node-6.10.0.jar +kubernetes-model-policy/6.10.0//kubernetes-model-policy-6.10.0.jar +kubernetes-model-rbac/6.10.0//kubernetes-model-rbac-6.10.0.jar +kubernetes-model-resource/6.10.0//kubernetes-model-resource-6.10.0.jar +kubernetes-model-scheduling/6.10.0//kubernetes-model-scheduling-6.10.0.jar +kubernetes-model-storageclass/6.10.0//kubernetes-model-storageclass-6.10.0.jar lapack/3.0.3//lapack-3.0.3.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar diff --git a/pom.xml b/pom.xml index 51471cd31f620..e9aeb9eeefc2e 100644 --- a/pom.xml +++ b/pom.xml @@ -235,7 +235,7 @@ org.fusesource.leveldbjni - 6.9.1 + 6.10.0 ${java.home} From 63641a9988ca2961646507cd5d0ba918df704914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 10 Jan 2024 20:15:42 +0100 Subject: [PATCH 2/4] change ResourceRequirements to VolumeResourceRequirements --- .../spark/deploy/k8s/features/MountVolumesFeatureStep.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala index eed076ca877bc..031d18545b898 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala @@ -90,7 +90,7 @@ private[spark] class MountVolumesFeatureStep(conf: KubernetesConf) .withNewSpec() .withStorageClassName(storageClass.get) .withAccessModes(PVC_ACCESS_MODE) - .withResources(new ResourceRequirementsBuilder() + .withResources(new VolumeResourceRequirementsBuilder() .withRequests(Map("storage" -> new Quantity(size.get)).asJava).build()) .endSpec() .build()) From 547ffc93057c027ae5c8f185400417526e138b27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 10 Jan 2024 20:40:18 +0100 Subject: [PATCH 3/4] more ResourceRequirements to VolumeResourceRequirements --- .../apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala index 7e61e9339f5f7..2745aa7dbe618 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala @@ -87,7 +87,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => .withNewSpec() .withAccessModes("ReadWriteOnce") .withStorageClassName("test-local-storage") - .withResources(new ResourceRequirementsBuilder() + .withResources(new VolumeResourceRequirementsBuilder() .withRequests(Map("storage" -> new Quantity("1Gi")).asJava).build()) .endSpec() From 283f253b1eaa276dd72857e638c00c6cf56113f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 10 Jan 2024 21:04:28 +0100 Subject: [PATCH 4/4] more VolumeResourceRequirements --- .../scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala index 09b3d5a230b79..299979071b5d7 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala @@ -228,7 +228,7 @@ object ExecutorLifecycleTestUtils { .withNewSpec() .withStorageClassName(storageClass) .withAccessModes("ReadWriteOnce") - .withResources(new ResourceRequirementsBuilder() + .withResources(new VolumeResourceRequirementsBuilder() .withRequests(Map("storage" -> new Quantity(size)).asJava).build()) .endSpec() .build()