feat: added support for controller runtime v0.15.0

.github/actions/deploy-klt-on-cluster/action.yml

@@ -5,12 +5,12 @@ inputs:
     required: false
     description: "Version of kind that should be used"
     # renovate: datasource=github-releases depName=kubernetes-sigs/kind
-    default: "v0.17.0"
+    default: "v0.18.0"
   k8s-version:
     required: false
     description: "Kubernetes that should be used"
     # renovate: datasource=github-releases depName=kubernetes/kubernetes
-    default: "v1.25.3"
+    default: "v1.27.1"
   runtime_tag:
     description: "Tag for the runner image"
     required: true

klt-cert-manager/go.mod

@@ -8,11 +8,11 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/afero v1.9.5
 	github.com/stretchr/testify v1.8.4
-	k8s.io/api v0.26.5
-	k8s.io/apiextensions-apiserver v0.26.5
-	k8s.io/apimachinery v0.26.5
-	k8s.io/client-go v0.26.5
-	sigs.k8s.io/controller-runtime v0.14.6
+	k8s.io/api v0.27.1
+	k8s.io/apiextensions-apiserver v0.27.1
+	k8s.io/apimachinery v0.27.1
+	k8s.io/client-go v0.27.1
+	sigs.k8s.io/controller-runtime v0.15.0
 )
 
 require (
@@ -72,4 +72,4 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace golang.org/x/net => golang.org/x/net v0.10.0
+replace sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.15.0

klt-cert-manager/pkg/webhook/builder_test.go

@@ -45,8 +45,8 @@ func TestBuilder_Run(t *testing.T) {
 	mockManager.GetAPIReaderFunc = func() client.Reader {
 		return newFakeClient()
 	}
-	webhookServer := &webhook.Server{}
-	mockManager.GetWebhookServerFunc = func() *webhook.Server {
+	webhookServer := webhook.NewServer(webhook.Options{})
+	mockManager.GetWebhookServerFunc = func() webhook.Server {
 		return webhookServer
 	}
 

klt-cert-manager/pkg/webhook/manager.go

@@ -4,6 +4,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
 const (
@@ -44,8 +45,8 @@ func (provider WebhookProvider) createOptions(scheme *runtime.Scheme, namespace
 
 func (provider WebhookProvider) SetupWebhookServer(mgr manager.Manager) {
 	webhookServer := mgr.GetWebhookServer()
-	webhookServer.CertDir = provider.certificateDirectory
-	webhookServer.KeyName = provider.keyFileName
-	webhookServer.CertName = provider.certificateFileName
+	webhookServer.(*webhook.DefaultServer).Options.CertDir = provider.certificateDirectory
+	webhookServer.(*webhook.DefaultServer).Options.KeyName = provider.keyFileName
+	webhookServer.(*webhook.DefaultServer).Options.CertName = provider.certificateFileName
 
 }

klt-cert-manager/pkg/webhook/manager_test.go

@@ -31,23 +31,23 @@ func TestCreateOptions(t *testing.T) {
 	})
 	t.Run("configures webhooks server", func(t *testing.T) {
 		provider := NewWebhookManagerProvider("certs-dir", "key-file", "cert-file")
-		expectedWebhookServer := &webhook.Server{}
+		expectedWebhookServer := webhook.NewServer(webhook.Options{})
 
 		mgr := &fake.MockManager{
-			GetWebhookServerFunc: func() *webhook.Server {
+			GetWebhookServerFunc: func() webhook.Server {
 				return expectedWebhookServer
 			},
 		}
 
 		provider.SetupWebhookServer(mgr)
 
-		assert.Equal(t, "certs-dir", expectedWebhookServer.CertDir)
-		assert.Equal(t, "key-file", expectedWebhookServer.KeyName)
-		assert.Equal(t, "cert-file", expectedWebhookServer.CertName)
+		assert.Equal(t, "certs-dir", expectedWebhookServer.(*webhook.DefaultServer).Options.CertDir)
+		assert.Equal(t, "key-file", expectedWebhookServer.(*webhook.DefaultServer).Options.KeyName)
+		assert.Equal(t, "cert-file", expectedWebhookServer.(*webhook.DefaultServer).Options.CertName)
 
 		mgrWebhookServer := mgr.GetWebhookServer()
-		assert.Equal(t, "certs-dir", mgrWebhookServer.CertDir)
-		assert.Equal(t, "key-file", mgrWebhookServer.KeyName)
-		assert.Equal(t, "cert-file", mgrWebhookServer.CertName)
+		assert.Equal(t, "certs-dir", mgrWebhookServer.(*webhook.DefaultServer).Options.CertDir)
+		assert.Equal(t, "key-file", mgrWebhookServer.(*webhook.DefaultServer).Options.KeyName)
+		assert.Equal(t, "cert-file", mgrWebhookServer.(*webhook.DefaultServer).Options.CertName)
 	})
 }

metrics-operator/go.mod

@@ -14,15 +14,15 @@ require (
 	github.com/prometheus/client_golang v1.15.1
 	github.com/prometheus/common v0.44.0
 	github.com/stretchr/testify v1.8.4
-	k8s.io/api v0.26.5
-	k8s.io/apiextensions-apiserver v0.26.5
-	k8s.io/apimachinery v0.26.5
-	k8s.io/apiserver v0.26.5
-	k8s.io/client-go v0.26.5
-	k8s.io/component-base v0.26.5
+	k8s.io/api v0.27.1
+	k8s.io/apiextensions-apiserver v0.27.1
+	k8s.io/apimachinery v0.27.1
+	k8s.io/apiserver v0.27.1
+	k8s.io/client-go v0.27.1
+	k8s.io/component-base v0.27.1
 	k8s.io/klog/v2 v2.100.1
 	k8s.io/metrics v0.26.5
-	sigs.k8s.io/controller-runtime v0.14.6
+	sigs.k8s.io/controller-runtime v0.15.0
 	sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230116101851-63817c8ac8f2
 )
 

metrics-operator/main.go

@@ -45,6 +45,7 @@ import (
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	ctrlWebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
 var (
@@ -157,12 +158,12 @@ func main() {
 			SetPodName(env.PodName).
 			SetManagerProvider(
 				webhook.NewWebhookManagerProvider(
-					mgr.GetWebhookServer().CertDir, "tls.key", "tls.crt"),
+					mgr.GetWebhookServer().(*ctrlWebhook.DefaultServer).Options.CertDir, "tls.key", "tls.crt"),
 			).
 			SetCertificateWatcher(
 				certificates.NewCertificateWatcher(
 					mgr.GetAPIReader(),
-					mgr.GetWebhookServer().CertDir,
+					mgr.GetWebhookServer().(*ctrlWebhook.DefaultServer).Options.CertDir,
 					env.PodNamespace,
 					certCommon.SecretName,
 					setupLog,

operator/apis/lifecycle/v1alpha3/common/common.go

@@ -22,6 +22,7 @@ const K8sRecommendedAppAnnotations = "app.kubernetes.io/part-of"
 const K8sRecommendedManagedByAnnotations = "app.kubernetes.io/managed-by"
 const PreDeploymentEvaluationAnnotation = "keptn.sh/pre-deployment-evaluations"
 const PostDeploymentEvaluationAnnotation = "keptn.sh/post-deployment-evaluations"
+const SchedullingGateRemoved = "keptn.sh/schedulling-gate-removed"
 const TaskNameAnnotation = "keptn.sh/task-name"
 const NamespaceEnabledAnnotation = "keptn.sh/lifecycle-toolkit"
 const CreateAppTaskSpanName = "create_%s_app_task"

operator/config/default/webhooknamespaces_patch.yaml

@@ -25,3 +25,21 @@ webhooks:
             - "keptn-lifecycle-toolkit-system"
             - "observability"
             - "monitoring"
+  - name: gpod.keptn.sh
+    namespaceSelector:
+      matchExpressions:
+        - key: control-plane
+          operator: NotIn
+          values:
+            - "lifecycle-operator"
+        - key: kubernetes.io/metadata.name
+          operator: NotIn
+          values:
+            - "kube-system"
+            - "kube-public"
+            - "kube-node-lease"
+            - "cert-manager"
+            - "keptn-lifecycle-toolkit-system"
+            - "observability"
+            - "monitoring"
+

operator/config/rbac/role.yaml

@@ -85,6 +85,7 @@ rules:
   verbs:
   - get
   - list
+  - update
   - watch
 - apiGroups:
   - ""

operator/config/webhook/manifests.yaml

@@ -27,6 +27,25 @@ webhooks:
         resources:
           - pods
     sideEffects: None
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: lifecycle-webhook-service
+        namespace: system
+        path: /gate-v1-pod
+    failurePolicy: Fail
+    name: gpod.keptn.sh
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+        resources:
+          - pods
+    sideEffects: None
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration

operator/controllers/common/helperfunctions.go

@@ -7,7 +7,9 @@ import (
 	"github.com/go-logr/logr"
 	klcv1alpha3 "github.com/keptn/lifecycle-toolkit/operator/apis/lifecycle/v1alpha3"
 	apicommon "github.com/keptn/lifecycle-toolkit/operator/apis/lifecycle/v1alpha3/common"
+	controllererrors "github.com/keptn/lifecycle-toolkit/operator/controllers/errors"
 	"github.com/keptn/lifecycle-toolkit/operator/controllers/lifecycle/interfaces"
+	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
@@ -114,3 +116,64 @@ func getObject(k8sclient client.Client, log logr.Logger, ctx context.Context, de
 	}
 	return nil
 }
+
+func RemoveGates(ctx context.Context, c client.Client, log logr.Logger, workloadInstance *klcv1alpha3.KeptnWorkloadInstance) error {
+	switch workloadInstance.Spec.ResourceReference.Kind {
+	case "Pod":
+		return removePodGates(ctx, c, log, workloadInstance.Spec.ResourceReference.Name, workloadInstance.Namespace)
+	case "ReplicaSet", "StatefulSet", "DaemonSet":
+		podList, err := getPodsOfOwner(ctx, c, log, workloadInstance.Spec.ResourceReference.UID, workloadInstance.Spec.ResourceReference.Kind, workloadInstance.Namespace)
+		if err != nil {
+			log.Error(err, "cannot get pods")
+			return err
+		}
+		for _, pod := range podList {
+			err := removePodGates(ctx, c, log, pod, workloadInstance.Namespace)
+			if err != nil {
+				log.Error(err, "cannot remove gates from pod")
+				return err
+			}
+		}
+	default:
+		return controllererrors.ErrUnsupportedWorkloadInstanceResourceReference
+	}
+
+	return nil
+}
+
+func removePodGates(ctx context.Context, c client.Client, log logr.Logger, podName string, podNamespace string) error {
+	pod := &v1.Pod{}
+	err := c.Get(ctx, types.NamespacedName{Namespace: podNamespace, Name: podName}, pod)
+	if err != nil {
+		log.Error(err, "cannot remove gates from pod - inner")
+		return err
+	}
+	if len(pod.Annotations) == 0 {
+		pod.Annotations = make(map[string]string)
+	}
+	pod.Annotations[apicommon.SchedullingGateRemoved] = "true"
+	pod.Spec.SchedulingGates = nil
+	return c.Update(ctx, pod)
+}
+
+func getPodsOfOwner(ctx context.Context, c client.Client, log logr.Logger, ownerUID types.UID, ownerKind string, namespace string) ([]string, error) {
+	pods := &v1.PodList{}
+	err := c.List(ctx, pods, client.InNamespace(namespace))
+	if err != nil {
+		log.Error(err, "cannot list pods - inner")
+		return nil, err
+	}
+
+	var resultPods []string
+
+	for _, pod := range pods.Items {
+		for _, owner := range pod.OwnerReferences {
+			if owner.Kind == ownerKind && owner.UID == ownerUID {
+				resultPods = append(resultPods, pod.Name)
+				break
+			}
+		}
+	}
+
+	return resultPods, nil
+}

operator/controllers/lifecycle/keptnworkloadinstance/controller.go

@@ -61,7 +61,7 @@ type KeptnWorkloadInstanceReconciler struct {
 // +kubebuilder:rbac:groups=lifecycle.keptn.sh,resources=keptntasks/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=lifecycle.keptn.sh,resources=keptntasks/finalizers,verbs=update
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;watch;patch
-// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
+// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;update
 // +kubebuilder:rbac:groups=apps,resources=replicasets;deployments;statefulsets;daemonsets,verbs=get;list;watch
 // +kubebuilder:rbac:groups=argoproj.io,resources=rollouts,verbs=get;list;watch
 
@@ -139,6 +139,12 @@ func (r *KeptnWorkloadInstanceReconciler) Reconcile(ctx context.Context, req ctr
 		}
 	}
 
+	// pre-evaluation checks done at this moment, we can remove the gate
+	if err := controllercommon.RemoveGates(ctx, r.Client, r.Log, workloadInstance); err != nil {
+		r.Log.Error(err, "could not remove SchedullingGates")
+		return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil
+	}
+
 	// Wait for deployment of Workload
 	phase = apicommon.PhaseWorkloadDeployment
 	if !workloadInstance.IsDeploymentSucceeded() {

operator/controllers/options/keptnconfig_controller_test.go

@@ -7,24 +7,18 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
+	"github.com/go-logr/logr/testr"
 	optionsv1alpha1 "github.com/keptn/lifecycle-toolkit/operator/apis/options/v1alpha1"
 	"github.com/keptn/lifecycle-toolkit/operator/controllers/common/fake"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
 func TestKeptnConfigReconciler_Reconcile(t *testing.T) {
-	reconciler := setupReconciler()
-
-	// set up logger
-	opts := zap.Options{
-		Development: true,
-	}
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+	reconciler := setupReconciler(t)
 
 	type args struct {
 		ctx context.Context
@@ -147,11 +141,6 @@ func TestKeptnConfigReconciler_initConfig(t *testing.T) {
 }
 
 func TestKeptnConfigReconciler_reconcileOtelCollectorUrl(t *testing.T) {
-	// set up logger
-	opts := zap.Options{
-		Development: true,
-	}
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
 	type fields struct {
 		Client          client.Client
@@ -174,7 +163,7 @@ func TestKeptnConfigReconciler_reconcileOtelCollectorUrl(t *testing.T) {
 			fields: fields{
 				Client: nil,
 				Scheme: nil,
-				Log:    ctrl.Log.WithName("test-keptn-config-controller"),
+				Log:    testr.New(t),
 				LastAppliedSpec: &optionsv1alpha1.KeptnConfigSpec{
 					OTelCollectorUrl: "",
 				},
@@ -197,7 +186,7 @@ func TestKeptnConfigReconciler_reconcileOtelCollectorUrl(t *testing.T) {
 			fields: fields{
 				Client: nil,
 				Scheme: nil,
-				Log:    ctrl.Log.WithName("test-keptn-config-controller"),
+				Log:    testr.New(t),
 			},
 			args: args{
 				config: &optionsv1alpha1.KeptnConfig{
@@ -233,7 +222,7 @@ func TestKeptnConfigReconciler_reconcileOtelCollectorUrl(t *testing.T) {
 	}
 }
 
-func setupReconciler() *KeptnConfigReconciler {
+func setupReconciler(t *testing.T) *KeptnConfigReconciler {
 	emptyConfig := &optionsv1alpha1.KeptnConfig{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "empty-config",
@@ -262,18 +251,12 @@ func setupReconciler() *KeptnConfigReconciler {
 		},
 	}
 
-	//setup logger
-	opts := zap.Options{
-		Development: true,
-	}
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
-
 	fakeClient := fake.NewClient(emptyConfig, config1, config2)
 
 	r := &KeptnConfigReconciler{
 		Client: fakeClient,
 		Scheme: fakeClient.Scheme(),
-		Log:    ctrl.Log.WithName("test-keptnconfig-controller"),
+		Log:    testr.New(t),
 	}
 	return r
 }