feat: use internal cert

Dockerfile

@@ -15,6 +15,7 @@ RUN go mod download
 COPY main.go main.go
 COPY api/ api/
 COPY pkg/controllers/ pkg/controllers/
+COPY pkg/util/cert/ pkg/util/cert/
 
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command

config/default/kustomization.yaml

@@ -20,7 +20,7 @@ bases:
 # crd/kustomization.yaml
 - ../webhook
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
-- ../certmanager
+- ../internalcert
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
 
@@ -39,34 +39,34 @@ patchesStrategicMerge:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
 # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
 # 'CERTMANAGER' needs to be enabled to use ca injection
-- webhookcainjection_patch.yaml
+#- webhookcainjection_patch.yaml
 
 # the following config is for teaching kustomize how to do var substitution
-vars:
+#vars:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
-- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
-  objref:
-    kind: Certificate
-    group: cert-manager.io
-    version: v1
-    name: serving-cert # this name should match the one in certificate.yaml
-  fieldref:
-    fieldpath: metadata.namespace
-- name: CERTIFICATE_NAME
-  objref:
-    kind: Certificate
-    group: cert-manager.io
-    version: v1
-    name: serving-cert # this name should match the one in certificate.yaml
-- name: SERVICE_NAMESPACE # namespace of the service
-  objref:
-    kind: Service
-    version: v1
-    name: webhook-service
-  fieldref:
-    fieldpath: metadata.namespace
-- name: SERVICE_NAME
-  objref:
-    kind: Service
-    version: v1
-    name: webhook-service
+#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
+#  objref:
+#    kind: Certificate
+#    group: cert-manager.io
+#    version: v1
+#    name: serving-cert # this name should match the one in certificate.yaml
+#  fieldref:
+#    fieldpath: metadata.namespace
+#- name: CERTIFICATE_NAME
+#  objref:
+#    kind: Certificate
+#    group: cert-manager.io
+#    version: v1
+#    name: serving-cert # this name should match the one in certificate.yaml
+#- name: SERVICE_NAMESPACE # namespace of the service
+#  objref:
+#    kind: Service
+#    version: v1
+#    name: webhook-service
+#  fieldref:
+#    fieldpath: metadata.namespace
+#- name: SERVICE_NAME
+#  objref:
+#    kind: Service
+#    version: v1
+#    name: webhook-service

config/internalcert/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - secret.yaml

config/internalcert/secret.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: webhook-server-cert
+  namespace: system

config/rbac/role.yaml

@@ -25,6 +25,33 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+    - ""
+  resources:
+    - secrets
+  verbs:
+    - get
+    - list
+    - update
+    - watch
+- apiGroups:
+    - admissionregistration.k8s.io
+  resources:
+    - mutatingwebhookconfigurations
+  verbs:
+    - get
+    - list
+    - update
+    - watch
+- apiGroups:
+    - admissionregistration.k8s.io
+  resources:
+    - validatingwebhookconfigurations
+  verbs:
+    - get
+    - list
+    - update
+    - watch
 - apiGroups:
   - batch
   resources:

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/onsi/ginkgo/v2 v2.9.2
 	github.com/onsi/gomega v1.27.6
+	github.com/open-policy-agent/cert-controller v0.7.0
 	k8s.io/api v0.26.4
 	k8s.io/apimachinery v0.26.4
 	k8s.io/client-go v0.26.4

go.sum

@@ -129,6 +129,9 @@ github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU=
 github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts=
 github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
 github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
+github.com/open-policy-agent/cert-controller v0.7.0 h1:5ggZjSQJ1YgkT+ngNAGBGHaOYfGuYq97IrNldchCoHI=
+github.com/open-policy-agent/cert-controller v0.7.0/go.mod h1:Dkkdcr1BeSUig/62dYLqgvx3lIN0XtEikKTGW1lKSQo=
+github.com/open-policy-agent/frameworks/constraint v0.0.0-20230201235642-777dc99a6669 h1:vKt4PhZXBxYHeLujYraNVpkoILQ/NISiifzaq1DkMXk=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -315,6 +318,7 @@ k8s.io/component-base v0.26.3 h1:oC0WMK/ggcbGDTkdcqefI4wIZRYdK3JySx9/HADpV0g=
 k8s.io/component-base v0.26.3/go.mod h1:5kj1kZYwSC6ZstHJN7oHBqcJC6yyn41eR+Sqa/mQc8E=
 k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw=
 k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-aggregator v0.23.2 h1:6CoZZqNdFc9benrgSJJ0GQGgFtKjI0y3UwlBbioXtc8=
 k8s.io/kube-openapi v0.0.0-20230327201221-f5883ff37f0c h1:EFfsozyzZ/pggw5qNx7ftTVZdp7WZl+3ih89GEjYEK8=
 k8s.io/kube-openapi v0.0.0-20230327201221-f5883ff37f0c/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg=
 k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 h1:xMMXJlJbsU8w3V5N2FLDQ8YgU8s1EoULdbQBcAeNJkY=

main.go

@@ -33,6 +33,7 @@ import (
 
 	jobset "sigs.k8s.io/jobset/api/v1alpha1"
 	"sigs.k8s.io/jobset/pkg/controllers"
+	"sigs.k8s.io/jobset/pkg/util/cert"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -88,16 +89,52 @@ func main() {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
+
+	certsReady := make(chan struct{})
+	if err = cert.CertsManager(mgr, certsReady); err != nil {
+		setupLog.Error(err, "unable to setup cert rotation")
+		os.Exit(1)
+	}
+
+	if err := controllers.SetupIndexes(mgr.GetFieldIndexer()); err != nil {
+		setupLog.Error(err, "unable to setup indexes")
+	}
+
+	// Cert won't be ready until manager starts, so start a goroutine here which
+	// will block until the cert is ready before setting up the controllers.
+	// Controllers who register after manager starts will start directly.
+	go setupControllers(mgr, certsReady)
+
+	setupHealthzAndReadyzCheck(mgr)
+
+	setupLog.Info("starting manager")
+	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+		setupLog.Error(err, "problem running manager")
+		os.Exit(1)
+	}
+}
+
+func setupControllers(mgr ctrl.Manager, certsReady chan struct{}) {
+	// The controllers won't work until the webhooks are operating,
+	// and the webhook won't work until the certs are all in places.
+	setupLog.Info("waiting for the cert generation to complete")
+	<-certsReady
+	setupLog.Info("certs ready")
+
 	jobSetController := controllers.NewJobSetReconciler(mgr.GetClient(), mgr.GetScheme(), mgr.GetEventRecorderFor("jobset"))
-	if err = jobSetController.SetupWithManager(mgr); err != nil {
+	if err := jobSetController.SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "JobSet")
 		os.Exit(1)
 	}
-	if err = (&jobset.JobSet{}).SetupWebhookWithManager(mgr); err != nil {
+	if err := (&jobset.JobSet{}).SetupWebhookWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create webhook", "webhook", "JobSet")
 		os.Exit(1)
 	}
 	//+kubebuilder:scaffold:builder
+}
+
+func setupHealthzAndReadyzCheck(mgr ctrl.Manager) {
+	defer setupLog.Info("both healthz and readyz check are finished and configured")
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
@@ -107,10 +144,4 @@ func main() {
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
 	}
-
-	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
-		setupLog.Error(err, "problem running manager")
-		os.Exit(1)
-	}
 }

pkg/controllers/jobset_controller.go

@@ -134,28 +134,25 @@ func (r *JobSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *JobSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	if err := mgr.GetFieldIndexer().IndexField(context.Background(), &batchv1.Job{}, jobOwnerKey, func(rawObj client.Object) []string {
-		// grab the job object, extract the owner...
-		job := rawObj.(*batchv1.Job)
-		owner := metav1.GetControllerOf(job)
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&jobset.JobSet{}).
+		Owns(&batchv1.Job{}).
+		Complete(r)
+}
+
+func SetupIndexes(indexer client.FieldIndexer) error {
+	return indexer.IndexField(context.Background(), &batchv1.Job{}, jobOwnerKey, func(obj client.Object) []string {
+		o := obj.(*batchv1.Job)
+		owner := metav1.GetControllerOf(o)
 		if owner == nil {
 			return nil
 		}
 		// ...make sure it's a JobSet...
 		if owner.APIVersion != apiGVStr || owner.Kind != "JobSet" {
 			return nil
 		}
-
-		// ...and if so, return it
 		return []string{owner.Name}
-	}); err != nil {
-		return err
-	}
-
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&jobset.JobSet{}).
-		Owns(&batchv1.Job{}).
-		Complete(r)
+	})
 }
 
 // getChildJobs gets jobs owned by the JobSet then categorizes them by status (active, successful, failed).

pkg/util/cert/cert.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cert
+
+import (
+	"fmt"
+
+	cert "github.com/open-policy-agent/cert-controller/pkg/rotator"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+)
+
+const (
+	serviceName             = "jobset-webhook-service"
+	secretName              = "jobset-webhook-server-cert"
+	secretNamespace         = "jobset-system"
+	certDir                 = "/tmp/k8s-webhook-server/serving-certs"
+	validateWebhookConfName = "jobset-validating-webhook-configuration"
+	mutatingWebhookConfName = "jobset-mutating-webhook-configuration"
+	caName                  = "jobset-ca"
+	caOrg                   = "jobset"
+)
+
+// dnsName is the format of <service name>.<namespace>.svc
+var dnsName = fmt.Sprintf("%s.%s.svc", serviceName, secretNamespace)
+
+//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update
+//+kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=mutatingwebhookconfigurations,verbs=get;list;watch;update
+//+kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=validatingwebhookconfigurations,verbs=get;list;watch;update
+
+// CertsManager creates certs for webhooks.
+func CertsManager(mgr ctrl.Manager, setupFinish chan struct{}) error {
+	return cert.AddRotator(mgr, &cert.CertRotator{
+		SecretKey: types.NamespacedName{
+			Namespace: secretNamespace,
+			Name:      secretName,
+		},
+		CertDir:        certDir,
+		CAName:         caName,
+		CAOrganization: caOrg,
+		DNSName:        dnsName,
+		IsReady:        setupFinish,
+		Webhooks: []cert.WebhookInfo{
+			{
+				Type: cert.Validating,
+				Name: validateWebhookConfName,
+			},
+			{
+				Type: cert.Mutating,
+				Name: mutatingWebhookConfName,
+			},
+		},
+	})
+}

test/integration/controller/suite_test.go

@@ -84,6 +84,9 @@ var _ = BeforeSuite(func() {
 	Expect(err).ToNot(HaveOccurred())
 	jobSetController := controllers.NewJobSetReconciler(k8sManager.GetClient(), k8sManager.GetScheme(), k8sManager.GetEventRecorderFor("jobset"))
 
+	err = controllers.SetupIndexes(k8sManager.GetFieldIndexer())
+	Expect(err).ToNot(HaveOccurred())
+
 	err = jobSetController.SetupWithManager(k8sManager)
 	Expect(err).ToNot(HaveOccurred())
 

test/integration/webhook/suite_test.go

@@ -37,6 +37,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	jobset "sigs.k8s.io/jobset/api/v1alpha1"
+	"sigs.k8s.io/jobset/pkg/controllers"
 )
 
 // These tests use Ginkgo (BDD-style Go testing framework). Refer to
@@ -99,6 +100,9 @@ var _ = BeforeSuite(func() {
 	})
 	Expect(err).NotTo(HaveOccurred())
 
+	err = controllers.SetupIndexes(mgr.GetFieldIndexer())
+	Expect(err).NotTo(HaveOccurred())
+
 	err = (&jobset.JobSet{}).SetupWebhookWithManager(mgr)
 	Expect(err).NotTo(HaveOccurred())