Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implemented cleanup especially for sidecars #16

Merged
merged 8 commits into from
Dec 12, 2023
Merged

implemented cleanup especially for sidecars #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d052723
implemented cleanup especially for sidecars
sinamna Dec 11, 2023
a43a3f2
updated tests
sinamna Dec 11, 2023
d211511
refactored and fixed typos
sinamna Dec 12, 2023
834bce4
changed the name of subprovisioner to subRec
sinamna Dec 12, 2023
71c9964
updating state in status
sinamna Dec 12, 2023
7186084
use subreconciler method to return error
sinamna Dec 12, 2023
a8b7e9f
removed finalizer from sidecar deployments
sinamna Dec 12, 2023
f69090f
comparing parts effecting adaptive scale
sinamna Dec 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions internal/controller/basic_authenticator/basicauthenticator_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@ package basic_authenticator
import (
"context"
"github.com/go-logr/logr"
"github.com/opdev/subreconciler"
authenticatorv1alpha1 "github.com/snapp-incubator/simple-authenticator/api/v1alpha1"
"github.com/snapp-incubator/simple-authenticator/internal/config"
appv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
Expand Down Expand Up @@ -59,6 +61,19 @@ func (r *BasicAuthenticatorReconciler) Reconcile(ctx context.Context, req ctrl.R
r.logger.Info("reconcile triggered")
r.logger.Info(req.String())
r.initVars(req)

basicAuthenticator := &authenticatorv1alpha1.BasicAuthenticator{}
switch err := r.Get(ctx, req.NamespacedName, basicAuthenticator); {
case errors.IsNotFound(err):
return r.Cleanup(ctx, req)
case err != nil:
r.logger.Error(err, "failed to fetch object")
return subreconciler.Evaluate(subreconciler.Requeue())
default:
if basicAuthenticator.ObjectMeta.DeletionTimestamp != nil {
return r.Cleanup(ctx, req)
}
}
return r.Provision(ctx, req)
}

Expand Down
180 changes: 180 additions & 0 deletions internal/controller/basic_authenticator/cleanup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
package basic_authenticator

import (
"context"
"errors"
"github.com/opdev/subreconciler"
"github.com/snapp-incubator/simple-authenticator/api/v1alpha1"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

func (r *BasicAuthenticatorReconciler) Cleanup(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
// Do the actual reconcile work
subRecs := []subreconciler.FnWithRequest{
r.setDeletionStatus,
r.removeInjectedContainers,
r.removeCleanupFinalizer,
}
for _, rec := range subRecs {
result, err := rec(ctx, req)
if subreconciler.ShouldHaltOrRequeue(result, err) {
return subreconciler.Evaluate(result, err)
}
}

return subreconciler.Evaluate(subreconciler.DoNotRequeue())
}
func (r *BasicAuthenticatorReconciler) setDeletionStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}
basicAuthenticator.Status.State = StatusDeleting

if err := r.Update(ctx, basicAuthenticator); err != nil {
r.logger.Error(err, "Failed to update status while cleaning")
return subreconciler.RequeueWithError(err)
}
return subreconciler.ContinueReconciling()
}
func (r *BasicAuthenticatorReconciler) removeInjectedContainers(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}

if basicAuthenticator.Spec.Type != "sidecar" {
return subreconciler.ContinueReconciling()
}
basicAuthLabel := map[string]string{
basicAuthenticatorNameLabel: basicAuthenticator.Name,
}
deployments, err := getTargetDeployment(ctx, basicAuthenticator, r.Client, basicAuthLabel)
if err != nil {
r.logger.Error(err, "failed to get target deployments to clean up")
return subreconciler.RequeueWithError(err)
}
configmaps, err := getTargetConfigmapNames(ctx, basicAuthenticator, r.Client, basicAuthLabel)
if err != nil {
r.logger.Error(err, "failed to get target configmap to clean up")
return subreconciler.RequeueWithError(err)
}
secrets, err := getTargetSecretName(ctx, basicAuthenticator, r.Client, basicAuthLabel)
if err != nil {
r.logger.Error(err, "failed to get target secret to clean up")
return subreconciler.RequeueWithError(err)
}
r.logger.Info("debug", "configmap", configmaps, "secret", secrets)

cleanupDeployments := removeInjectedResources(deployments, secrets, configmaps)
for _, deploy := range cleanupDeployments {
if err := r.Update(ctx, deploy); err != nil {
r.logger.Error(err, "failed to add update cleaned up deployments")
return subreconciler.RequeueWithError(err)
}
}
return subreconciler.ContinueReconciling()
}
func (r *BasicAuthenticatorReconciler) removeCleanupFinalizer(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}
if controllerutil.ContainsFinalizer(basicAuthenticator, basicAuthenticatorFinalizer) {
if ok := controllerutil.RemoveFinalizer(basicAuthenticator, basicAuthenticatorFinalizer); !ok {
r.logger.Error(errors.New("finalizer not updated"), "Failed to remove finalizer for BasicAuthenticator")
return subreconciler.Requeue()
}
}

if err := r.Update(ctx, basicAuthenticator); err != nil {
r.logger.Error(err, "Failed to remove finalizer for BasicAuthenticator")
return subreconciler.RequeueWithError(err)
}
return subreconciler.ContinueReconciling()
}

func getTargetDeployment(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]*appsv1.Deployment, error) {
var deploymentList appsv1.DeploymentList
if err := k8Client.List(
ctx,
&deploymentList,
client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
client.InNamespace(basicAuthenticator.Namespace)); err != nil {
return nil, err
}

resultDeployments := make([]*appsv1.Deployment, 0)
for _, deploy := range deploymentList.Items {
resultDeployments = append(resultDeployments, &deploy)
}
return resultDeployments, nil
}
func getTargetConfigmapNames(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]string, error) {
var configMapList v1.ConfigMapList
if err := k8Client.List(
ctx,
&configMapList,
client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
client.InNamespace(basicAuthenticator.Namespace)); err != nil {
return nil, err
}
resultConfigMaps := make([]string, 0)
for _, cm := range configMapList.Items {
resultConfigMaps = append(resultConfigMaps, cm.Name)
}
return resultConfigMaps, nil
}
func getTargetSecretName(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]string, error) {
var secretList v1.SecretList
if err := k8Client.List(
ctx,
&secretList,
client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
client.InNamespace(basicAuthenticator.Namespace)); err != nil {
return nil, err
}

resultSecrets := make([]string, 0)
for _, sec := range secretList.Items {
resultSecrets = append(resultSecrets, sec.Name)
}
return resultSecrets, nil
}
func removeInjectedResources(deployments []*appsv1.Deployment, secrets []string, configmap []string) []*appsv1.Deployment {
for _, deploy := range deployments {
containers := make([]v1.Container, 0)
for _, container := range deploy.Spec.Template.Spec.Containers {
if container.Name != nginxDefaultContainerName {
containers = append(containers, container)
}
}
deploy.Spec.Template.Spec.Containers = containers
volumes := make([]v1.Volume, 0)
for _, vol := range deploy.Spec.Template.Spec.Volumes {
if !existsInList(secrets, vol.Name) && !existsInList(configmap, vol.Name) {
volumes = append(volumes, vol)
}
}
deploy.Spec.Template.Spec.Volumes = volumes
if deploy.Labels != nil {
delete(deploy.Labels, basicAuthenticatorNameLabel)
}
}
return deployments
hoptical marked this conversation as resolved.
Show resolved Hide resolved
}

func existsInList(strList []string, targetStr string) bool {
for _, val := range strList {
if val == targetStr {
return true
}
}
return false
hoptical marked this conversation as resolved.
Show resolved Hide resolved
}
7 changes: 4 additions & 3 deletions internal/controller/basic_authenticator/const.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ package basic_authenticator
const (
nginxDefaultImageAddress = "nginx:1.25.3"
nginxDefaultContainerName = "nginx"
SecretAnnotation = "authenticator.snappcloud.io/secret.name"
ConfigmapAnnotation = "authenticator.snappcloud.io/configmap.name"
basicAuthenticatorNameLabel = "basicauthenticator.snappcloud.io/name"
basicAuthenticatorFinalizer = "basicauthenticator.snappcloud.io/finalizer"
ExternallyManaged = "basicauthenticator.snappcloud.io/externally.managed"
ConfigMountPath = "/etc/nginx/conf.d"
Expand All @@ -15,7 +14,6 @@ const (
template = `server {
listen AUTHENTICATOR_PORT;
location / {
resolver 8.8.8.8;
auth_basic "basic authentication area";
auth_basic_user_file "FILE_PATH";
proxy_pass http://APP_SERVICE:APP_PORT;
Expand All @@ -25,4 +23,7 @@ const (
proxy_set_header X-Forwarded-Proto $scheme;
}
}`
StatusAvailable = "Available"
StatusReconciling = "Reconciling"
StatusDeleting = "Deleting"
hoptical marked this conversation as resolved.
Show resolved Hide resolved
)
69 changes: 58 additions & 11 deletions internal/controller/basic_authenticator/provision.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,20 @@ import (
"reflect"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

// Provision provisions the required resources for the basicAuthenticator object
func (r *BasicAuthenticatorReconciler) Provision(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
// Do the actual reconcile work
subProvisioner := []subreconciler.FnWithRequest{
r.setReconcilingStatus,
r.addCleanupFinalizer,
r.ensureSecret,
r.ensureConfigmap,
r.ensureDeployment,
r.ensureService,
r.setAvailableStatus,
}
for _, provisioner := range subProvisioner {
result, err := provisioner(ctx, req)
Expand All @@ -33,6 +37,37 @@ func (r *BasicAuthenticatorReconciler) Provision(ctx context.Context, req ctrl.R

return subreconciler.Evaluate(subreconciler.DoNotRequeue())
}
func (r *BasicAuthenticatorReconciler) setReconcilingStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}

basicAuthenticator.Status.State = StatusReconciling
if err := r.Update(ctx, basicAuthenticator); err != nil {
r.logger.Error(err, "failed to update status")
return subreconciler.Requeue()
}
return subreconciler.ContinueReconciling()
}

func (r *BasicAuthenticatorReconciler) addCleanupFinalizer(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}
if !controllerutil.ContainsFinalizer(basicAuthenticator, basicAuthenticatorFinalizer) {
if objUpdated := controllerutil.AddFinalizer(basicAuthenticator, basicAuthenticatorFinalizer); objUpdated {
if err := r.Update(ctx, basicAuthenticator); err != nil {
r.logger.Error(err, "failed to add basicAuthenticator finalizer")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the error message appropriate?

return subreconciler.Requeue()
}
}
}
return subreconciler.ContinueReconciling()
}

func (r *BasicAuthenticatorReconciler) getLatestBasicAuthenticator(ctx context.Context, req ctrl.Request, basicAuthenticator *v1alpha1.BasicAuthenticator) (*ctrl.Result, error) {
err := r.Get(ctx, req.NamespacedName, basicAuthenticator)
Expand All @@ -51,7 +86,7 @@ func (r *BasicAuthenticatorReconciler) ensureSecret(ctx context.Context, req ctr
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return r, err
return subreconciler.RequeueWithError(err)
}
r.credentialName = basicAuthenticator.Spec.CredentialsSecretRef
var credentialSecret corev1.Secret
Expand Down Expand Up @@ -118,7 +153,7 @@ func (r *BasicAuthenticatorReconciler) ensureConfigmap(ctx context.Context, req
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return r, err
return subreconciler.RequeueWithError(err)
}

authenticatorConfig := createNginxConfigmap(basicAuthenticator)
Expand Down Expand Up @@ -160,7 +195,7 @@ func (r *BasicAuthenticatorReconciler) ensureDeployment(ctx context.Context, req
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return r, err
return subreconciler.RequeueWithError(err)
}

if r.configMapName == "" {
Expand All @@ -182,7 +217,7 @@ func (r *BasicAuthenticatorReconciler) ensureService(ctx context.Context, req ct
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return r, err
return subreconciler.RequeueWithError(err)
}
if r.deploymentLabel == nil {
return subreconciler.ContinueReconciling()
Expand Down Expand Up @@ -217,6 +252,22 @@ func (r *BasicAuthenticatorReconciler) ensureService(ctx context.Context, req ct
}
return subreconciler.ContinueReconciling()
}

func (r *BasicAuthenticatorReconciler) setAvailableStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}

basicAuthenticator.Status.State = StatusAvailable
if err := r.Update(ctx, basicAuthenticator); err != nil {
r.logger.Error(err, "failed to update status")
return subreconciler.Requeue()
}
return subreconciler.ContinueReconciling()
}

func (r *BasicAuthenticatorReconciler) createDeploymentAuthenticator(ctx context.Context, req ctrl.Request, basicAuthenticator *v1alpha1.BasicAuthenticator, authenticatorConfigName, secretName string) (*ctrl.Result, error) {

newDeployment := createNginxDeployment(basicAuthenticator, authenticatorConfigName, secretName, r.CustomConfig)
Expand Down Expand Up @@ -253,6 +304,7 @@ func (r *BasicAuthenticatorReconciler) createDeploymentAuthenticator(ctx context
replica, err := r.acquireTargetReplica(ctx, basicAuthenticator)
if err != nil {
r.logger.Error(err, "failed to acquire target replica using adaptiveScale")
return subreconciler.RequeueWithError(err)
}
targetReplica = &replica
}
Expand All @@ -262,16 +314,15 @@ func (r *BasicAuthenticatorReconciler) createDeploymentAuthenticator(ctx context

foundDeployment.Spec = newDeployment.Spec
foundDeployment.Spec.Replicas = targetReplica

err := r.Update(ctx, foundDeployment)
err = r.Update(ctx, foundDeployment)
if err != nil {
r.logger.Error(err, "failed to update deployment")
return subreconciler.RequeueWithError(err)
}
}
r.logger.Info("updating ready replicas")
basicAuthenticator.Status.ReadyReplicas = int(foundDeployment.Status.ReadyReplicas)
err := r.Status().Update(ctx, basicAuthenticator)
err = r.Status().Update(ctx, basicAuthenticator)
if err != nil {
r.logger.Error(err, "failed to update basic authenticator status")
return subreconciler.RequeueWithError(err)
Expand All @@ -287,10 +338,6 @@ func (r *BasicAuthenticatorReconciler) createSidecarAuthenticator(ctx context.Co
return subreconciler.RequeueWithError(err)
}
for _, deploy := range deploymentsToUpdate {
if err := ctrl.SetControllerReference(basicAuthenticator, deploy, r.Scheme); err != nil {
r.logger.Error(err, "failed to set injected deployment owner")
return subreconciler.RequeueWithError(err)
}
err := r.Update(ctx, deploy)
if err != nil {
r.logger.Error(err, "failed to update injected deployments")
Expand Down
Loading