KEP-2170: Implement skeleton webhook servers

Signed-off-by: Yuki Iwai <yuki.iwai.tz@gmail.com>

Makefile

@@ -41,8 +41,9 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust
 		output:crd:artifacts:config=manifests/base/crds \
 		output:rbac:artifacts:config=manifests/base/rbac \
 		output:webhook:artifacts:config=manifests/base/webhook
-	$(CONTROLLER_GEN) "crd:generateEmbeddedObjectMeta=true" paths="./pkg/apis/kubeflow.org/v2alpha1/..." \
-		output:crd:artifacts:config=manifests/v2/base/crds
+	$(CONTROLLER_GEN) "crd:generateEmbeddedObjectMeta=true" "webhook" paths="./pkg/apis/kubeflow.org/v2alpha1/...;./pkg/webhook.v2/..." \
+		output:crd:artifacts:config=manifests/v2/base/crds \
+		output:webhook:artifacts:config=manifests/v2/base/webhook
 
 generate: controller-gen ## Generate apidoc, sdk and code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate/boilerplate.go.txt" paths="./pkg/apis/..."

manifests/v2/base/webhook/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - manifests.yaml

manifests/v2/base/webhook/manifests.yaml

@@ -0,0 +1,66 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-kubeflow-org-v2alpha1-clustertrainingruntime
+  failurePolicy: Fail
+  name: validator.clustertrainingruntime.training-operator.kubeflow.org
+  rules:
+  - apiGroups:
+    - kubeflow.org
+    apiVersions:
+    - v2alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - clustertrainingruntimes
+  sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-kubeflow-org-v2alpha1-trainingruntime
+  failurePolicy: Fail
+  name: validator.trainingruntime.training-operator.kubeflow.org
+  rules:
+  - apiGroups:
+    - kubeflow.org
+    apiVersions:
+    - v2alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - trainingruntimes
+  sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-kubeflow-org-v2alpha1-trainjob
+  failurePolicy: Fail
+  name: validator.trainjob.training-operator.kubeflow.org
+  rules:
+  - apiGroups:
+    - kubeflow.org
+    apiVersions:
+    - v2alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - trainjobs
+  sideEffects: None

pkg/webhook.v2/clustertrainingruntime_webhook.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2024 The Kubeflow Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package webhookv2
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	kubeflowv2 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v2alpha1"
+)
+
+type ClusterTrainingRuntimeWebhook struct{}
+
+func setupWebhookForClusterTrainingRuntime(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(&kubeflowv2.ClusterTrainingRuntime{}).
+		WithValidator(&ClusterTrainingRuntimeWebhook{}).
+		Complete()
+}
+
+// +kubebuilder:webhook:path=/validate-kubeflow-org-v2alpha1-clustertrainingruntime,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=clustertrainingruntimes,verbs=create;update,versions=v2alpha1,name=validator.clustertrainingruntime.training-operator.kubeflow.org,admissionReviewVersions=v1
+
+var _ webhook.CustomValidator = (*ClusterTrainingRuntimeWebhook)(nil)
+
+func (w *ClusterTrainingRuntimeWebhook) ValidateCreate(context.Context, runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}
+
+func (w *ClusterTrainingRuntimeWebhook) ValidateUpdate(context.Context, runtime.Object, runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}
+
+func (w *ClusterTrainingRuntimeWebhook) ValidateDelete(context.Context, runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}

pkg/webhook.v2/webhook.go → pkg/webhook.v2/setup.go

@@ -18,6 +18,15 @@ package webhookv2
 
 import ctrl "sigs.k8s.io/controller-runtime"
 
-func Setup(ctrl.Manager) (string, error) {
+func Setup(mgr ctrl.Manager) (string, error) {
+	if err := setupWebhookForClusterTrainingRuntime(mgr); err != nil {
+		return "ClusterTrainingRuntime", err
+	}
+	if err := setupWebhookForTrainingRuntime(mgr); err != nil {
+		return "TrainingRuntime", err
+	}
+	if err := setupWebhookForTrainJob(mgr); err != nil {
+		return "TrainJob", err
+	}
 	return "", nil
 }

pkg/webhook.v2/trainingruntime_webhook.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2024 The Kubeflow Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package webhookv2
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	kubeflowv2 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v2alpha1"
+)
+
+type TrainingRuntimeWebhook struct{}
+
+func setupWebhookForTrainingRuntime(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(&kubeflowv2.TrainingRuntime{}).
+		WithValidator(&TrainingRuntimeWebhook{}).
+		Complete()
+}
+
+// +kubebuilder:webhook:path=/validate-kubeflow-org-v2alpha1-trainingruntime,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=trainingruntimes,verbs=create;update,versions=v2alpha1,name=validator.trainingruntime.training-operator.kubeflow.org,admissionReviewVersions=v1
+
+var _ webhook.CustomValidator = (*TrainingRuntimeWebhook)(nil)
+
+func (w *TrainingRuntimeWebhook) ValidateCreate(context.Context, runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}
+
+func (w *TrainingRuntimeWebhook) ValidateUpdate(context.Context, runtime.Object, runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}
+
+func (w *TrainingRuntimeWebhook) ValidateDelete(context.Context, runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}

pkg/webhook.v2/trainjob_webhook.go

@@ -15,3 +15,39 @@ limitations under the License.
 */
 
 package webhookv2
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	kubeflowv2 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v2alpha1"
+)
+
+type TrainJobWebhook struct{}
+
+func setupWebhookForTrainJob(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(&kubeflowv2.TrainJob{}).
+		WithValidator(&TrainJobWebhook{}).
+		Complete()
+}
+
+// +kubebuilder:webhook:path=/validate-kubeflow-org-v2alpha1-trainjob,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=trainjobs,verbs=create;update,versions=v2alpha1,name=validator.trainjob.training-operator.kubeflow.org,admissionReviewVersions=v1
+
+var _ webhook.CustomValidator = (*TrainJobWebhook)(nil)
+
+func (w *TrainJobWebhook) ValidateCreate(context.Context, runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}
+
+func (w *TrainJobWebhook) ValidateUpdate(context.Context, runtime.Object, runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}
+
+func (w *TrainJobWebhook) ValidateDelete(context.Context, runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}

test/integration/framework/framework.go

@@ -18,7 +18,11 @@ package framework
 
 import (
 	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
 	"path/filepath"
+	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
@@ -31,9 +35,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	kubeflowv2 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v2alpha1"
 	controllerv2 "github.com/kubeflow/training-operator/pkg/controller.v2"
+	webhookv2 "github.com/kubeflow/training-operator/pkg/webhook.v2"
 )
 
 type Framework struct {
@@ -45,7 +51,10 @@ func (f *Framework) Init() *rest.Config {
 	log.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true)))
 	ginkgo.By("bootstrapping test environment")
 	f.testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "manifests", "v2", "base", "crds")},
+		CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "manifests", "v2", "base", "crds")},
+		WebhookInstallOptions: envtest.WebhookInstallOptions{
+			Paths: []string{filepath.Join("..", "..", "..", "manifests", "v2", "base", "webhook")},
+		},
 		ErrorIfCRDPathMissing: true,
 	}
 	cfg, err := f.testEnv.Start()
@@ -55,6 +64,7 @@ func (f *Framework) Init() *rest.Config {
 }
 
 func (f *Framework) RunManager(cfg *rest.Config) (context.Context, client.Client) {
+	webhookInstallOpts := &f.testEnv.WebhookInstallOptions
 	gomega.ExpectWithOffset(1, kubeflowv2.AddToScheme(scheme.Scheme)).NotTo(gomega.HaveOccurred())
 
 	// +kubebuilder:scaffold:scheme
@@ -70,18 +80,34 @@ func (f *Framework) RunManager(cfg *rest.Config) (context.Context, client.Client
 		Metrics: metricsserver.Options{
 			BindAddress: "0", // disable metrics to avoid conflicts between packages.
 		},
+		WebhookServer: webhook.NewServer(
+			webhook.Options{
+				Host:    webhookInstallOpts.LocalServingHost,
+				Port:    webhookInstallOpts.LocalServingPort,
+				CertDir: webhookInstallOpts.LocalServingCertDir,
+			}),
 	})
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred(), "failed to create manager")
 
 	failedCtrlName, err := controllerv2.SetupControllers(mgr)
 	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred(), "controller", failedCtrlName)
+	failedWebhookName, err := webhookv2.Setup(mgr)
+	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred(), "webhook", failedWebhookName)
 
 	go func() {
 		defer ginkgo.GinkgoRecover()
 		err = mgr.Start(ctx)
 		gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred(), "failed to run manager")
 	}()
 
+	dialer := &net.Dialer{Timeout: time.Second}
+	addrPort := fmt.Sprintf("%s:%d", webhookInstallOpts.LocalServingHost, webhookInstallOpts.LocalServingPort)
+	gomega.Eventually(func(g gomega.Gomega) {
+		var conn *tls.Conn
+		conn, err = tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true})
+		g.Expect(err).Should(gomega.Succeed())
+		g.Expect(conn.Close()).Should(gomega.Succeed())
+	}).Should(gomega.Succeed())
 	return ctx, k8sClient
 }
 

test/integration/webhook.v2/clustertrainingruntime_test.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2024 The Kubeflow Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package webhookv2
+
+import (
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/kubeflow/training-operator/test/integration/framework"
+)
+
+var _ = ginkgo.Describe("ClusterTrainingRuntime Webhook", ginkgo.Ordered, func() {
+	var ns *corev1.Namespace
+
+	ginkgo.BeforeAll(func() {
+		fwk = &framework.Framework{}
+		cfg = fwk.Init()
+		ctx, k8sClient = fwk.RunManager(cfg)
+	})
+	ginkgo.AfterAll(func() {
+		fwk.Teardown()
+	})
+
+	ginkgo.BeforeEach(func() {
+		ns = &corev1.Namespace{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: corev1.SchemeGroupVersion.String(),
+				Kind:       "Namespace",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "clustertrainingruntime-webhook-",
+			},
+		}
+		gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed())
+	})
+})