From 3538acd4e61cf86899d05ae6a275172ad3e13cd6 Mon Sep 17 00:00:00 2001 From: Yuki Iwai Date: Sat, 7 Sep 2024 01:36:38 +0900 Subject: [PATCH] KEP-2170: Implement skeleton webhook servers Signed-off-by: Yuki Iwai --- Makefile | 5 +- manifests/v2/base/webhook/kustomization.yaml | 2 + manifests/v2/base/webhook/manifests.yaml | 66 +++++++++++++++++++ .../clustertrainingruntime_webhook.go | 53 +++++++++++++++ pkg/webhook.v2/{webhook.go => setup.go} | 11 +++- pkg/webhook.v2/trainingruntime_webhook.go | 53 +++++++++++++++ pkg/webhook.v2/trainjob_webhook.go | 36 ++++++++++ test/integration/framework/framework.go | 28 +++++++- .../webhook.v2/clustertrainingruntime_test.go | 52 +++++++++++++++ test/integration/webhook.v2/suite_test.go | 42 ++++++++++++ .../webhook.v2/trainingruntime_test.go | 52 +++++++++++++++ test/integration/webhook.v2/trainjob_test.go | 52 +++++++++++++++ 12 files changed, 448 insertions(+), 4 deletions(-) create mode 100644 manifests/v2/base/webhook/kustomization.yaml create mode 100644 manifests/v2/base/webhook/manifests.yaml create mode 100644 pkg/webhook.v2/clustertrainingruntime_webhook.go rename pkg/webhook.v2/{webhook.go => setup.go} (66%) create mode 100644 pkg/webhook.v2/trainingruntime_webhook.go create mode 100644 test/integration/webhook.v2/clustertrainingruntime_test.go create mode 100644 test/integration/webhook.v2/suite_test.go create mode 100644 test/integration/webhook.v2/trainingruntime_test.go create mode 100644 test/integration/webhook.v2/trainjob_test.go diff --git a/Makefile b/Makefile index dce93e7ed2..5d8bfc2596 100644 --- a/Makefile +++ b/Makefile @@ -41,8 +41,9 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust output:crd:artifacts:config=manifests/base/crds \ output:rbac:artifacts:config=manifests/base/rbac \ output:webhook:artifacts:config=manifests/base/webhook - $(CONTROLLER_GEN) "crd:generateEmbeddedObjectMeta=true" paths="./pkg/apis/kubeflow.org/v2alpha1/..." \ - output:crd:artifacts:config=manifests/v2/base/crds + $(CONTROLLER_GEN) "crd:generateEmbeddedObjectMeta=true" "webhook" paths="./pkg/apis/kubeflow.org/v2alpha1/...;./pkg/webhook.v2/..." \ + output:crd:artifacts:config=manifests/v2/base/crds \ + output:webhook:artifacts:config=manifests/v2/base/webhook generate: controller-gen ## Generate apidoc, sdk and code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate/boilerplate.go.txt" paths="./pkg/apis/..." diff --git a/manifests/v2/base/webhook/kustomization.yaml b/manifests/v2/base/webhook/kustomization.yaml new file mode 100644 index 0000000000..62dd305281 --- /dev/null +++ b/manifests/v2/base/webhook/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - manifests.yaml diff --git a/manifests/v2/base/webhook/manifests.yaml b/manifests/v2/base/webhook/manifests.yaml new file mode 100644 index 0000000000..d4ee6aee5e --- /dev/null +++ b/manifests/v2/base/webhook/manifests.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-kubeflow-org-v2alpha1-clustertrainingruntime + failurePolicy: Fail + name: validator.clustertrainingruntime.kubeflow.org + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v2alpha1 + operations: + - CREATE + - UPDATE + resources: + - clustertrainingruntimes + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-kubeflow-org-v2alpha1-trainingruntime + failurePolicy: Fail + name: validator.trainingruntime.kubeflow.org + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v2alpha1 + operations: + - CREATE + - UPDATE + resources: + - trainingruntimes + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-kubeflow-org-v2alpha1-trainjob + failurePolicy: Fail + name: validator.trainjob.kubeflow.org + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v2alpha1 + operations: + - CREATE + - UPDATE + resources: + - trainjobs + sideEffects: None diff --git a/pkg/webhook.v2/clustertrainingruntime_webhook.go b/pkg/webhook.v2/clustertrainingruntime_webhook.go new file mode 100644 index 0000000000..0ce728f654 --- /dev/null +++ b/pkg/webhook.v2/clustertrainingruntime_webhook.go @@ -0,0 +1,53 @@ +/* +Copyright 2024 The Kubeflow Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookv2 + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + kubeflowv2 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v2alpha1" +) + +type ClusterTrainingRuntimeWebhook struct{} + +func setupWebhookForClusterTrainingRuntime(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&kubeflowv2.ClusterTrainingRuntime{}). + WithValidator(&ClusterTrainingRuntimeWebhook{}). + Complete() +} + +// +kubebuilder:webhook:path=/validate-kubeflow-org-v2alpha1-clustertrainingruntime,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=clustertrainingruntimes,verbs=create;update,versions=v2alpha1,name=validator.clustertrainingruntime.kubeflow.org,admissionReviewVersions=v1 + +var _ webhook.CustomValidator = (*ClusterTrainingRuntimeWebhook)(nil) + +func (w *ClusterTrainingRuntimeWebhook) ValidateCreate(context.Context, runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +func (w *ClusterTrainingRuntimeWebhook) ValidateUpdate(context.Context, runtime.Object, runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +func (w *ClusterTrainingRuntimeWebhook) ValidateDelete(context.Context, runtime.Object) (admission.Warnings, error) { + return nil, nil +} diff --git a/pkg/webhook.v2/webhook.go b/pkg/webhook.v2/setup.go similarity index 66% rename from pkg/webhook.v2/webhook.go rename to pkg/webhook.v2/setup.go index 3d4970ef45..f7c9436ab7 100644 --- a/pkg/webhook.v2/webhook.go +++ b/pkg/webhook.v2/setup.go @@ -18,6 +18,15 @@ package webhookv2 import ctrl "sigs.k8s.io/controller-runtime" -func Setup(ctrl.Manager) (string, error) { +func Setup(mgr ctrl.Manager) (string, error) { + if err := setupWebhookForClusterTrainingRuntime(mgr); err != nil { + return "ClusterTrainingRuntime", err + } + if err := setupWebhookForTrainingRuntime(mgr); err != nil { + return "TrainingRuntime", err + } + if err := setupWebhookForTrainJob(mgr); err != nil { + return "TrainJob", err + } return "", nil } diff --git a/pkg/webhook.v2/trainingruntime_webhook.go b/pkg/webhook.v2/trainingruntime_webhook.go new file mode 100644 index 0000000000..a9fa897dbc --- /dev/null +++ b/pkg/webhook.v2/trainingruntime_webhook.go @@ -0,0 +1,53 @@ +/* +Copyright 2024 The Kubeflow Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookv2 + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + kubeflowv2 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v2alpha1" +) + +type TrainingRuntimeWebhook struct{} + +func setupWebhookForTrainingRuntime(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&kubeflowv2.TrainingRuntime{}). + WithValidator(&TrainingRuntimeWebhook{}). + Complete() +} + +// +kubebuilder:webhook:path=/validate-kubeflow-org-v2alpha1-trainingruntime,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=trainingruntimes,verbs=create;update,versions=v2alpha1,name=validator.trainingruntime.kubeflow.org,admissionReviewVersions=v1 + +var _ webhook.CustomValidator = (*TrainingRuntimeWebhook)(nil) + +func (w *TrainingRuntimeWebhook) ValidateCreate(context.Context, runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +func (w *TrainingRuntimeWebhook) ValidateUpdate(context.Context, runtime.Object, runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +func (w *TrainingRuntimeWebhook) ValidateDelete(context.Context, runtime.Object) (admission.Warnings, error) { + return nil, nil +} diff --git a/pkg/webhook.v2/trainjob_webhook.go b/pkg/webhook.v2/trainjob_webhook.go index f228441857..231e124f3d 100644 --- a/pkg/webhook.v2/trainjob_webhook.go +++ b/pkg/webhook.v2/trainjob_webhook.go @@ -15,3 +15,39 @@ limitations under the License. */ package webhookv2 + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + kubeflowv2 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v2alpha1" +) + +type TrainJobWebhook struct{} + +func setupWebhookForTrainJob(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&kubeflowv2.TrainJob{}). + WithValidator(&TrainJobWebhook{}). + Complete() +} + +// +kubebuilder:webhook:path=/validate-kubeflow-org-v2alpha1-trainjob,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=trainjobs,verbs=create;update,versions=v2alpha1,name=validator.trainjob.kubeflow.org,admissionReviewVersions=v1 + +var _ webhook.CustomValidator = (*TrainJobWebhook)(nil) + +func (w *TrainJobWebhook) ValidateCreate(context.Context, runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +func (w *TrainJobWebhook) ValidateUpdate(context.Context, runtime.Object, runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +func (w *TrainJobWebhook) ValidateDelete(context.Context, runtime.Object) (admission.Warnings, error) { + return nil, nil +} diff --git a/test/integration/framework/framework.go b/test/integration/framework/framework.go index 97d15246dd..83d85c7e4a 100644 --- a/test/integration/framework/framework.go +++ b/test/integration/framework/framework.go @@ -18,7 +18,11 @@ package framework import ( "context" + "crypto/tls" + "fmt" + "net" "path/filepath" + "time" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -31,9 +35,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" kubeflowv2 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v2alpha1" controllerv2 "github.com/kubeflow/training-operator/pkg/controller.v2" + webhookv2 "github.com/kubeflow/training-operator/pkg/webhook.v2" ) type Framework struct { @@ -45,7 +51,10 @@ func (f *Framework) Init() *rest.Config { log.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true))) ginkgo.By("bootstrapping test environment") f.testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "manifests", "v2", "base", "crds")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "manifests", "v2", "base", "crds")}, + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "..", "manifests", "v2", "base", "webhook")}, + }, ErrorIfCRDPathMissing: true, } cfg, err := f.testEnv.Start() @@ -55,6 +64,7 @@ func (f *Framework) Init() *rest.Config { } func (f *Framework) RunManager(cfg *rest.Config) (context.Context, client.Client) { + webhookInstallOpts := &f.testEnv.WebhookInstallOptions gomega.ExpectWithOffset(1, kubeflowv2.AddToScheme(scheme.Scheme)).NotTo(gomega.HaveOccurred()) // +kubebuilder:scaffold:scheme @@ -70,11 +80,19 @@ func (f *Framework) RunManager(cfg *rest.Config) (context.Context, client.Client Metrics: metricsserver.Options{ BindAddress: "0", // disable metrics to avoid conflicts between packages. }, + WebhookServer: webhook.NewServer( + webhook.Options{ + Host: webhookInstallOpts.LocalServingHost, + Port: webhookInstallOpts.LocalServingPort, + CertDir: webhookInstallOpts.LocalServingCertDir, + }), }) gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred(), "failed to create manager") failedCtrlName, err := controllerv2.SetupControllers(mgr) gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred(), "controller", failedCtrlName) + failedWebhookName, err := webhookv2.Setup(mgr) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred(), "webhook", failedWebhookName) go func() { defer ginkgo.GinkgoRecover() @@ -82,6 +100,14 @@ func (f *Framework) RunManager(cfg *rest.Config) (context.Context, client.Client gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred(), "failed to run manager") }() + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOpts.LocalServingHost, webhookInstallOpts.LocalServingPort) + gomega.Eventually(func(g gomega.Gomega) { + var conn *tls.Conn + conn, err = tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + g.Expect(err).Should(gomega.Succeed()) + g.Expect(conn.Close()).Should(gomega.Succeed()) + }).Should(gomega.Succeed()) return ctx, k8sClient } diff --git a/test/integration/webhook.v2/clustertrainingruntime_test.go b/test/integration/webhook.v2/clustertrainingruntime_test.go new file mode 100644 index 0000000000..9ba9be69af --- /dev/null +++ b/test/integration/webhook.v2/clustertrainingruntime_test.go @@ -0,0 +1,52 @@ +/* +Copyright 2024 The Kubeflow Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookv2 + +import ( + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kubeflow/training-operator/test/integration/framework" +) + +var _ = ginkgo.Describe("ClusterTrainingRuntime Webhook", ginkgo.Ordered, func() { + var ns *corev1.Namespace + + ginkgo.BeforeAll(func() { + fwk = &framework.Framework{} + cfg = fwk.Init() + ctx, k8sClient = fwk.RunManager(cfg) + }) + ginkgo.AfterAll(func() { + fwk.Teardown() + }) + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Namespace", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "clustertrainingruntime-webhook-", + }, + } + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + }) +}) diff --git a/test/integration/webhook.v2/suite_test.go b/test/integration/webhook.v2/suite_test.go new file mode 100644 index 0000000000..addf4e5d65 --- /dev/null +++ b/test/integration/webhook.v2/suite_test.go @@ -0,0 +1,42 @@ +/* +Copyright 2024 The Kubeflow Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookv2 + +import ( + "context" + "testing" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubeflow/training-operator/test/integration/framework" +) + +var ( + cfg *rest.Config + k8sClient client.Client + ctx context.Context + fwk *framework.Framework +) + +func TestAPIs(t *testing.T) { + gomega.RegisterFailHandler(ginkgo.Fail) + + ginkgo.RunSpecs(t, "v2 Webhooks Suite") +} diff --git a/test/integration/webhook.v2/trainingruntime_test.go b/test/integration/webhook.v2/trainingruntime_test.go new file mode 100644 index 0000000000..dc2add14ce --- /dev/null +++ b/test/integration/webhook.v2/trainingruntime_test.go @@ -0,0 +1,52 @@ +/* +Copyright 2024 The Kubeflow Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookv2 + +import ( + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kubeflow/training-operator/test/integration/framework" +) + +var _ = ginkgo.Describe("TrainingRuntime Webhook", ginkgo.Ordered, func() { + var ns *corev1.Namespace + + ginkgo.BeforeAll(func() { + fwk = &framework.Framework{} + cfg = fwk.Init() + ctx, k8sClient = fwk.RunManager(cfg) + }) + ginkgo.AfterAll(func() { + fwk.Teardown() + }) + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Namespace", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "trainingruntime-webhook-", + }, + } + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + }) +}) diff --git a/test/integration/webhook.v2/trainjob_test.go b/test/integration/webhook.v2/trainjob_test.go new file mode 100644 index 0000000000..a8578f007b --- /dev/null +++ b/test/integration/webhook.v2/trainjob_test.go @@ -0,0 +1,52 @@ +/* +Copyright 2024 The Kubeflow Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhookv2 + +import ( + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kubeflow/training-operator/test/integration/framework" +) + +var _ = ginkgo.Describe("TrainJob Webhook", ginkgo.Ordered, func() { + var ns *corev1.Namespace + + ginkgo.BeforeAll(func() { + fwk = &framework.Framework{} + cfg = fwk.Init() + ctx, k8sClient = fwk.RunManager(cfg) + }) + ginkgo.AfterAll(func() { + fwk.Teardown() + }) + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Namespace", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "trainjob-webhook-", + }, + } + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + }) +})