From 285b4fb7666bfcfbf44ad08a8be1097d1928494f Mon Sep 17 00:00:00 2001 From: souravbiswassanto Date: Fri, 26 Jul 2024 18:37:11 +0600 Subject: [PATCH 1/3] Fix timing issue of webhookconfiguration apply Signed-off-by: souravbiswassanto --- pkg/cmds/webhook.go | 46 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/pkg/cmds/webhook.go b/pkg/cmds/webhook.go index 06c09b94..a343814c 100644 --- a/pkg/cmds/webhook.go +++ b/pkg/cmds/webhook.go @@ -25,11 +25,15 @@ import ( "github.com/spf13/cobra" flag "github.com/spf13/pflag" + reg "k8s.io/api/admissionregistration/v1" v1 "k8s.io/api/admissionregistration/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" clientscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" + kutil "kmodules.xyz/client-go" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -40,6 +44,10 @@ import ( var setupLog = ctrl.Log.WithName("setup") +const ( + installerApplyLabelKey = "updated-for" +) + func NewCmdWebhook(ctx context.Context) *cobra.Command { certDir := "/var/serving-cert" var webhookName string @@ -104,6 +112,12 @@ func NewCmdWebhook(ctx context.Context) *cobra.Command { if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { if mgr.GetCache().WaitForCacheSync(context.TODO()) { + kbclient := mgr.GetClient() + klog.Infoln("waiting for webhook configuration to be ready") + err := WaitUntilWebhookConfigurationApplied(ctx, webhookName, kbclient) + if err != nil { + setupLog.Error(err, "unable to wait until webhook configuration is applied") + } if err := updateMutatingWebhookCABundle(mgr, webhookName, certDir); err != nil { setupLog.Error(err, "unable to update caBundle for MutatingWebhookConfiguration") os.Exit(1) @@ -138,7 +152,7 @@ func updateMutatingWebhookCABundle(mgr ctrl.Manager, name, certDir string) error if err != nil { return err } - + delete(webhook.ObjectMeta.Labels, installerApplyLabelKey) caBundle, err := os.ReadFile(filepath.Join(certDir, "ca.crt")) if err != nil { return err @@ -157,6 +171,7 @@ func updateValidatingWebhookCABundle(mgr ctrl.Manager, name, certDir string) err if err != nil { return err } + delete(webhook.ObjectMeta.Labels, installerApplyLabelKey) caBundle, err := os.ReadFile(filepath.Join(certDir, "ca.crt")) if err != nil { @@ -167,3 +182,32 @@ func updateValidatingWebhookCABundle(mgr ctrl.Manager, name, certDir string) err } return mgr.GetClient().Update(context.TODO(), webhook, &client.UpdateOptions{}) } + +func WaitUntilWebhookConfigurationApplied(ctx context.Context, webhookName string, c client.Client) error { + return wait.PollUntilContextTimeout(ctx, kutil.RetryInterval, kutil.ReadinessTimeout, true, func(ctx context.Context) (bool, error) { + var mwc reg.MutatingWebhookConfiguration + err := c.Get(ctx, types.NamespacedName{ + Name: webhookName, + }, &mwc) + if err != nil { + return false, nil + } + var vwc reg.ValidatingWebhookConfiguration + err = c.Get(ctx, types.NamespacedName{ + Name: webhookName, + }, &vwc) + if err != nil { + return false, nil + } + _, mwcExists := mwc.ObjectMeta.Labels[installerApplyLabelKey] + _, vwcExists := vwc.ObjectMeta.Labels[installerApplyLabelKey] + + klog.Infoln("mwc exisrtssssssssssssssssssssss", mwcExists, vwcExists) + + if !mwcExists || !vwcExists { + return false, nil + } + + return true, nil + }) +} From 4518876283b459c364b341917e0f0fcb52f33f5d Mon Sep 17 00:00:00 2001 From: souravbiswassanto Date: Tue, 30 Jul 2024 11:13:27 +0600 Subject: [PATCH 2/3] fix Signed-off-by: souravbiswassanto --- pkg/cmds/webhook.go | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/pkg/cmds/webhook.go b/pkg/cmds/webhook.go index a343814c..b456a28e 100644 --- a/pkg/cmds/webhook.go +++ b/pkg/cmds/webhook.go @@ -18,11 +18,12 @@ package cmds import ( "context" + "fmt" + corev1 "k8s.io/api/core/v1" + appsv1 "kubeops.dev/petset/apis/apps/v1" "os" "path/filepath" - appsv1 "kubeops.dev/petset/apis/apps/v1" - "github.com/spf13/cobra" flag "github.com/spf13/pflag" reg "k8s.io/api/admissionregistration/v1" @@ -152,7 +153,6 @@ func updateMutatingWebhookCABundle(mgr ctrl.Manager, name, certDir string) error if err != nil { return err } - delete(webhook.ObjectMeta.Labels, installerApplyLabelKey) caBundle, err := os.ReadFile(filepath.Join(certDir, "ca.crt")) if err != nil { return err @@ -171,7 +171,6 @@ func updateValidatingWebhookCABundle(mgr ctrl.Manager, name, certDir string) err if err != nil { return err } - delete(webhook.ObjectMeta.Labels, installerApplyLabelKey) caBundle, err := os.ReadFile(filepath.Join(certDir, "ca.crt")) if err != nil { @@ -184,6 +183,23 @@ func updateValidatingWebhookCABundle(mgr ctrl.Manager, name, certDir string) err } func WaitUntilWebhookConfigurationApplied(ctx context.Context, webhookName string, c client.Client) error { + var pod corev1.Pod + // k8s. io/ api/ core/ v1 + podName := os.Getenv("POD_NAME") + podNamespace := os.Getenv("POD_NAMESPACE") + err := c.Get(ctx, types.NamespacedName{ + Name: podName, + Namespace: podNamespace, + }, &pod) + klog.Infoln("errrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr, ", err) + if err != nil { + return err + } + val, exists := pod.Labels[installerApplyLabelKey] + if !exists { + return fmt.Errorf("missing %s label", installerApplyLabelKey) + } + return wait.PollUntilContextTimeout(ctx, kutil.RetryInterval, kutil.ReadinessTimeout, true, func(ctx context.Context) (bool, error) { var mwc reg.MutatingWebhookConfiguration err := c.Get(ctx, types.NamespacedName{ @@ -199,8 +215,8 @@ func WaitUntilWebhookConfigurationApplied(ctx context.Context, webhookName strin if err != nil { return false, nil } - _, mwcExists := mwc.ObjectMeta.Labels[installerApplyLabelKey] - _, vwcExists := vwc.ObjectMeta.Labels[installerApplyLabelKey] + mwcVal, mwcExists := mwc.ObjectMeta.Labels[installerApplyLabelKey] + vwcVal, vwcExists := vwc.ObjectMeta.Labels[installerApplyLabelKey] klog.Infoln("mwc exisrtssssssssssssssssssssss", mwcExists, vwcExists) @@ -208,6 +224,10 @@ func WaitUntilWebhookConfigurationApplied(ctx context.Context, webhookName strin return false, nil } + if mwcVal != val || vwcVal != val { + return false, nil + } + return true, nil }) } From a6f8ac96fd8affde1080de8568ce0c1a4caec17c Mon Sep 17 00:00:00 2001 From: souravbiswassanto Date: Fri, 2 Aug 2024 17:58:23 +0600 Subject: [PATCH 3/3] Fix webhookconfiguration overwrite issue Signed-off-by: souravbiswassanto --- pkg/cmds/webhook.go | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/pkg/cmds/webhook.go b/pkg/cmds/webhook.go index b456a28e..768fc471 100644 --- a/pkg/cmds/webhook.go +++ b/pkg/cmds/webhook.go @@ -18,12 +18,11 @@ package cmds import ( "context" - "fmt" - corev1 "k8s.io/api/core/v1" - appsv1 "kubeops.dev/petset/apis/apps/v1" "os" "path/filepath" + appsv1 "kubeops.dev/petset/apis/apps/v1" + "github.com/spf13/cobra" flag "github.com/spf13/pflag" reg "k8s.io/api/admissionregistration/v1" @@ -46,7 +45,8 @@ import ( var setupLog = ctrl.Log.WithName("setup") const ( - installerApplyLabelKey = "updated-for" + releaseRevisionLabelKey = "release-revision" + releaseRevisionEnvKey = "RELEASE_REVISION" ) func NewCmdWebhook(ctx context.Context) *cobra.Command { @@ -183,22 +183,7 @@ func updateValidatingWebhookCABundle(mgr ctrl.Manager, name, certDir string) err } func WaitUntilWebhookConfigurationApplied(ctx context.Context, webhookName string, c client.Client) error { - var pod corev1.Pod - // k8s. io/ api/ core/ v1 - podName := os.Getenv("POD_NAME") - podNamespace := os.Getenv("POD_NAMESPACE") - err := c.Get(ctx, types.NamespacedName{ - Name: podName, - Namespace: podNamespace, - }, &pod) - klog.Infoln("errrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr, ", err) - if err != nil { - return err - } - val, exists := pod.Labels[installerApplyLabelKey] - if !exists { - return fmt.Errorf("missing %s label", installerApplyLabelKey) - } + releaseRevision := os.Getenv(releaseRevisionEnvKey) return wait.PollUntilContextTimeout(ctx, kutil.RetryInterval, kutil.ReadinessTimeout, true, func(ctx context.Context) (bool, error) { var mwc reg.MutatingWebhookConfiguration @@ -215,16 +200,14 @@ func WaitUntilWebhookConfigurationApplied(ctx context.Context, webhookName strin if err != nil { return false, nil } - mwcVal, mwcExists := mwc.ObjectMeta.Labels[installerApplyLabelKey] - vwcVal, vwcExists := vwc.ObjectMeta.Labels[installerApplyLabelKey] - - klog.Infoln("mwc exisrtssssssssssssssssssssss", mwcExists, vwcExists) + mwcReleaseRevision, mwcExists := mwc.ObjectMeta.Labels[releaseRevisionLabelKey] + vwcReleaseRevision, vwcExists := vwc.ObjectMeta.Labels[releaseRevisionLabelKey] if !mwcExists || !vwcExists { return false, nil } - if mwcVal != val || vwcVal != val { + if mwcReleaseRevision != releaseRevision || vwcReleaseRevision != releaseRevision { return false, nil }