implemented cleanup especially for sidecars

internal/controller/basic_authenticator/basicauthenticator_controller.go

@@ -19,10 +19,12 @@ package basic_authenticator
 import (
 	"context"
 	"github.com/go-logr/logr"
+	"github.com/opdev/subreconciler"
 	authenticatorv1alpha1 "github.com/snapp-incubator/simple-authenticator/api/v1alpha1"
 	"github.com/snapp-incubator/simple-authenticator/internal/config"
 	appv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -59,6 +61,19 @@ func (r *BasicAuthenticatorReconciler) Reconcile(ctx context.Context, req ctrl.R
 	r.logger.Info("reconcile triggered")
 	r.logger.Info(req.String())
 	r.initVars(req)
+
+	basicAuthenticator := &authenticatorv1alpha1.BasicAuthenticator{}
+	switch err := r.Get(ctx, req.NamespacedName, basicAuthenticator); {
+	case errors.IsNotFound(err):
+		return r.Cleanup(ctx, req)
+	case err != nil:
+		r.logger.Error(err, "failed to fetch object")
+		return subreconciler.Evaluate(subreconciler.Requeue())
+	default:
+		if basicAuthenticator.ObjectMeta.DeletionTimestamp != nil {
+			return r.Cleanup(ctx, req)
+		}
+	}
 	return r.Provision(ctx, req)
 }
 

internal/controller/basic_authenticator/cleanup.go

@@ -0,0 +1,184 @@
+package basic_authenticator
+
+import (
+	"context"
+	"errors"
+	"github.com/opdev/subreconciler"
+	"github.com/snapp-incubator/simple-authenticator/api/v1alpha1"
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (r *BasicAuthenticatorReconciler) Cleanup(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// Do the actual reconcile work
+	subRecs := []subreconciler.FnWithRequest{
+		r.setDeletionStatus,
+		r.removeInjectedContainers,
+		r.removeCleanupFinalizer,
+	}
+	for _, rec := range subRecs {
+		result, err := rec(ctx, req)
+		if subreconciler.ShouldHaltOrRequeue(result, err) {
+			return subreconciler.Evaluate(result, err)
+		}
+	}
+
+	return subreconciler.Evaluate(subreconciler.DoNotRequeue())
+}
+func (r *BasicAuthenticatorReconciler) setDeletionStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
+	basicAuthenticator := &v1alpha1.BasicAuthenticator{}
+	if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
+		return r, err
+	}
+	basicAuthenticator.Status.State = StatusDeleting
+
+	if err := r.Update(ctx, basicAuthenticator); err != nil {
+		r.logger.Error(err, "Failed to update status while cleaning")
+		return subreconciler.RequeueWithError(err)
+	}
+	return subreconciler.ContinueReconciling()
+}
+func (r *BasicAuthenticatorReconciler) removeInjectedContainers(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
+	basicAuthenticator := &v1alpha1.BasicAuthenticator{}
+
+	if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
+		return r, err
+	}
+
+	if basicAuthenticator.Spec.Type != "sidecar" {
+		return subreconciler.ContinueReconciling()
+	}
+	basicAuthLabel := map[string]string{
+		basicAuthenticatorNameLabel: basicAuthenticator.Name,
+	}
+	deployments, err := getTargetDeployment(ctx, basicAuthenticator, r.Client, basicAuthLabel)
+	if err != nil {
+		r.logger.Error(err, "failed to get target deployments to clean up")
+		return subreconciler.RequeueWithError(err)
+	}
+	configmaps, err := getTargetConfigmapNames(ctx, basicAuthenticator, r.Client, basicAuthLabel)
+	if err != nil {
+		r.logger.Error(err, "failed to get target configmap to clean up")
+		return subreconciler.RequeueWithError(err)
+	}
+	secrets, err := getTargetSecretName(ctx, basicAuthenticator, r.Client, basicAuthLabel)
+	if err != nil {
+		r.logger.Error(err, "failed to get target secret to clean up")
+		return subreconciler.RequeueWithError(err)
+	}
+	r.logger.Info("debug", "configmap", configmaps, "secret", secrets)
+
+	cleanupDeployments := removeInjectedResources(deployments, secrets, configmaps)
+	for _, deploy := range cleanupDeployments {
+		if controllerutil.ContainsFinalizer(deploy, basicAuthenticatorFinalizer) {
+			if objUpdated := controllerutil.RemoveFinalizer(deploy, basicAuthenticatorFinalizer); objUpdated {
+				if err := r.Update(ctx, deploy); err != nil {
+					r.logger.Error(err, "failed to add basicAuthenticator to injected deployment")
+					return subreconciler.RequeueWithError(err)
+				}
+			}
+		}
+	}
+	return subreconciler.ContinueReconciling()
+}
+func (r *BasicAuthenticatorReconciler) removeCleanupFinalizer(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
+	basicAuthenticator := &v1alpha1.BasicAuthenticator{}
+	if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
+		return r, err
+	}
+	if controllerutil.ContainsFinalizer(basicAuthenticator, basicAuthenticatorFinalizer) {
+		if ok := controllerutil.RemoveFinalizer(basicAuthenticator, basicAuthenticatorFinalizer); !ok {
+			r.logger.Error(errors.New("finalizer not updated"), "Failed to remove finalizer for BasicAuthenticator")
+			return subreconciler.Requeue()
+		}
+	}
+
+	if err := r.Update(ctx, basicAuthenticator); err != nil {
+		r.logger.Error(err, "Failed to remove finalizer for BasicAuthenticator")
+		return subreconciler.RequeueWithError(err)
+	}
+	return subreconciler.ContinueReconciling()
+}
+
+func getTargetDeployment(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]*appsv1.Deployment, error) {
+	var deploymentList appsv1.DeploymentList
+	if err := k8Client.List(
+		ctx,
+		&deploymentList,
+		client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
+		client.InNamespace(basicAuthenticator.Namespace)); err != nil {
+		return nil, err
+	}
+
+	resultDeployments := make([]*appsv1.Deployment, 0)
+	for _, deploy := range deploymentList.Items {
+		resultDeployments = append(resultDeployments, &deploy)
+	}
+	return resultDeployments, nil
+}
+func getTargetConfigmapNames(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]string, error) {
+	var configMapList v1.ConfigMapList
+	if err := k8Client.List(
+		ctx,
+		&configMapList,
+		client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
+		client.InNamespace(basicAuthenticator.Namespace)); err != nil {
+		return nil, err
+	}
+	resultConfigMaps := make([]string, 0)
+	for _, cm := range configMapList.Items {
+		resultConfigMaps = append(resultConfigMaps, cm.Name)
+	}
+	return resultConfigMaps, nil
+}
+func getTargetSecretName(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]string, error) {
+	var secretList v1.SecretList
+	if err := k8Client.List(
+		ctx,
+		&secretList,
+		client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
+		client.InNamespace(basicAuthenticator.Namespace)); err != nil {
+		return nil, err
+	}
+
+	resultSecrets := make([]string, 0)
+	for _, sec := range secretList.Items {
+		resultSecrets = append(resultSecrets, sec.Name)
+	}
+	return resultSecrets, nil
+}
+func removeInjectedResources(deployments []*appsv1.Deployment, secrets []string, configmap []string) []*appsv1.Deployment {
+	for _, deploy := range deployments {
+		containers := make([]v1.Container, 0)
+		for _, container := range deploy.Spec.Template.Spec.Containers {
+			if container.Name != nginxDefaultContainerName {
+				containers = append(containers, container)
+			}
+		}
+		deploy.Spec.Template.Spec.Containers = containers
+		volumes := make([]v1.Volume, 0)
+		for _, vol := range deploy.Spec.Template.Spec.Volumes {
+			if !existsInList(secrets, vol.Name) && !existsInList(configmap, vol.Name) {
+				volumes = append(volumes, vol)
+			}
+		}
+		deploy.Spec.Template.Spec.Volumes = volumes
+		if deploy.Labels != nil {
+			delete(deploy.Labels, basicAuthenticatorNameLabel)
+		}
+	}
+	return deployments
+}
+
+func existsInList(strList []string, targetStr string) bool {
+	for _, val := range strList {
+		if val == targetStr {
+			return true
+		}
+	}
+	return false
+}

internal/controller/basic_authenticator/const.go

@@ -3,8 +3,7 @@ package basic_authenticator
 const (
 	nginxDefaultImageAddress    = "nginx:1.25.3"
 	nginxDefaultContainerName   = "nginx"
-	SecretAnnotation            = "authenticator.snappcloud.io/secret.name"
-	ConfigmapAnnotation         = "authenticator.snappcloud.io/configmap.name"
+	basicAuthenticatorNameLabel = "basicauthenticator.snappcloud.io/name"
 	basicAuthenticatorFinalizer = "basicauthenticator.snappcloud.io/finalizer"
 	ExternallyManaged           = "basicauthenticator.snappcloud.io/externally.managed"
 	ConfigMountPath             = "/etc/nginx/conf.d"
@@ -15,7 +14,6 @@ const (
 	template = `server {
 	listen AUTHENTICATOR_PORT;
 	location / {
-		resolver    8.8.8.8;
 		auth_basic	"basic authentication area";
 		auth_basic_user_file "FILE_PATH";
 		proxy_pass http://APP_SERVICE:APP_PORT;
@@ -25,4 +23,7 @@ const (
 		proxy_set_header X-Forwarded-Proto $scheme;
 	}
 }`
+	StatusAvailable   = "Available"
+	StatusReconciling = "Reconciling"
+	StatusDeleting    = "Deleting"
 )

internal/controller/basic_authenticator/provision.go

@@ -13,16 +13,20 @@ import (
 	"reflect"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
 // Provision provisions the required resources for the basicAuthenticator object
 func (r *BasicAuthenticatorReconciler) Provision(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	// Do the actual reconcile work
 	subProvisioner := []subreconciler.FnWithRequest{
+		r.setReconcilingStatus,
+		r.addCleanupFinalizer,
 		r.ensureSecret,
 		r.ensureConfigmap,
 		r.ensureDeployment,
 		r.ensureService,
+		r.setAvailableStatus,
 	}
 	for _, provisioner := range subProvisioner {
 		result, err := provisioner(ctx, req)
@@ -33,6 +37,37 @@ func (r *BasicAuthenticatorReconciler) Provision(ctx context.Context, req ctrl.R
 
 	return subreconciler.Evaluate(subreconciler.DoNotRequeue())
 }
+func (r *BasicAuthenticatorReconciler) setReconcilingStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
+	basicAuthenticator := &v1alpha1.BasicAuthenticator{}
+
+	if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
+		return r, err
+	}
+
+	basicAuthenticator.Status.State = StatusReconciling
+	if err := r.Update(ctx, basicAuthenticator); err != nil {
+		r.logger.Error(err, "failed to update status")
+		return subreconciler.Requeue()
+	}
+	return subreconciler.ContinueReconciling()
+}
+
+func (r *BasicAuthenticatorReconciler) addCleanupFinalizer(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
+	basicAuthenticator := &v1alpha1.BasicAuthenticator{}
+
+	if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
+		return r, err
+	}
+	if !controllerutil.ContainsFinalizer(basicAuthenticator, basicAuthenticatorFinalizer) {
+		if objUpdated := controllerutil.AddFinalizer(basicAuthenticator, basicAuthenticatorFinalizer); objUpdated {
+			if err := r.Update(ctx, basicAuthenticator); err != nil {
+				r.logger.Error(err, "failed to add basicAuthenticator finalizer")
+				return subreconciler.Requeue()
+			}
+		}
+	}
+	return subreconciler.ContinueReconciling()
+}
 
 func (r *BasicAuthenticatorReconciler) getLatestBasicAuthenticator(ctx context.Context, req ctrl.Request, basicAuthenticator *v1alpha1.BasicAuthenticator) (*ctrl.Result, error) {
 	err := r.Get(ctx, req.NamespacedName, basicAuthenticator)
@@ -217,6 +252,22 @@ func (r *BasicAuthenticatorReconciler) ensureService(ctx context.Context, req ct
 	}
 	return subreconciler.ContinueReconciling()
 }
+
+func (r *BasicAuthenticatorReconciler) setAvailableStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
+	basicAuthenticator := &v1alpha1.BasicAuthenticator{}
+
+	if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
+		return r, err
+	}
+
+	basicAuthenticator.Status.State = StatusAvailable
+	if err := r.Update(ctx, basicAuthenticator); err != nil {
+		r.logger.Error(err, "failed to update status")
+		return subreconciler.Requeue()
+	}
+	return subreconciler.ContinueReconciling()
+}
+
 func (r *BasicAuthenticatorReconciler) createDeploymentAuthenticator(ctx context.Context, req ctrl.Request, basicAuthenticator *v1alpha1.BasicAuthenticator, authenticatorConfigName, secretName string) (*ctrl.Result, error) {
 
 	newDeployment := createNginxDeployment(basicAuthenticator, authenticatorConfigName, secretName, r.CustomConfig)
@@ -287,10 +338,12 @@ func (r *BasicAuthenticatorReconciler) createSidecarAuthenticator(ctx context.Co
 		return subreconciler.RequeueWithError(err)
 	}
 	for _, deploy := range deploymentsToUpdate {
-		if err := ctrl.SetControllerReference(basicAuthenticator, deploy, r.Scheme); err != nil {
-			r.logger.Error(err, "failed to set injected deployment owner")
-			return subreconciler.RequeueWithError(err)
+		if !controllerutil.ContainsFinalizer(deploy, basicAuthenticatorFinalizer) {
+			if objUpdated := controllerutil.AddFinalizer(deploy, basicAuthenticatorFinalizer); !objUpdated {
+				return subreconciler.Requeue()
+			}
 		}
+
 		err := r.Update(ctx, deploy)
 		if err != nil {
 			r.logger.Error(err, "failed to update injected deployments")