Implement SparkApplication integration (without dynamic scaling)

Signed-off-by: Shingo Omura <everpeace@gmail.com>
kubernetes-sigs · Jan 21, 2025 · 799345a · 799345a
1 parent 298decd
commit 799345a
Show file tree

Hide file tree

Showing 10 changed files with 1,445 additions and 1 deletion.
diff --git a/config/components/manager/controller_manager_config.yaml b/config/components/manager/controller_manager_config.yaml
@@ -53,6 +53,8 @@ integrations:
   - "kubeflow.org/tfjob"
   - "kubeflow.org/xgboostjob"
   - "workload.codeflare.dev/appwrapper"
+  - "codeflare.dev/appwrapper"
+  - "sparkoperator.k8s.io/sparkapplication"
 #  - "pod"
 #  - "deployment" # requires enabling pod integration
 #  - "statefulset" # requires enabling pod integration

diff --git a/config/components/manager/manager.yaml b/config/components/manager/manager.yaml
@@ -24,7 +24,7 @@ spec:
         - /manager
         args:
         - "--zap-log-level=2"
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         image: controller:latest
         name: manager
         securityContext:

diff --git a/config/components/rbac/role.yaml b/config/components/rbac/role.yaml
@@ -324,6 +324,31 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - sparkoperator.k8s.io
+  resources:
+  - sparkapplications
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - sparkoperator.k8s.io
+  resources:
+  - sparkapplications/finalizers
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - sparkoperator.k8s.io
+  resources:
+  - sparkapplications/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - workload.codeflare.dev
   resources:

diff --git a/config/components/webhook/manifests.yaml b/config/components/webhook/manifests.yaml
@@ -253,6 +253,25 @@ webhooks:
     resources:
     - rayjobs
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-sparkoperator-k8s-io-v1beta2-sparkapplication
+  failurePolicy: Fail
+  name: msparkapplication.kb.io
+  rules:
+  - apiGroups:
+    - sparkoperator.k8s.io
+    apiVersions:
+    - v1beta2
+    operations:
+    - CREATE
+    resources:
+    - sparkapplications
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:
@@ -596,6 +615,26 @@ webhooks:
     resources:
     - rayjobs
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-sparkoperator-k8s-io-v1beta2-sparkapplication
+  failurePolicy: Fail
+  name: vsparkapplication.kb.io
+  rules:
+  - apiGroups:
+    - sparkoperator.k8s.io
+    apiVersions:
+    - v1beta2
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - sparkapplications
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:

diff --git a/pkg/controller/jobs/jobs.go b/pkg/controller/jobs/jobs.go
@@ -28,5 +28,6 @@ import (
 	_ "sigs.k8s.io/kueue/pkg/controller/jobs/pod"
 	_ "sigs.k8s.io/kueue/pkg/controller/jobs/raycluster"
 	_ "sigs.k8s.io/kueue/pkg/controller/jobs/rayjob"
+	_ "sigs.k8s.io/kueue/pkg/controller/jobs/sparkapplication"
 	_ "sigs.k8s.io/kueue/pkg/controller/jobs/statefulset"
 )