Finalizes Gateway Dependent Resources (#219)

Signed-off-by: Daneyon Hansen <daneyonhansen@gmail.com>

hack/test-example.sh

@@ -50,10 +50,8 @@ kubectl::apply -f examples/gateway/gateway-nodeport.yaml
 kubectl::apply -f examples/gateway/kuard/kuard.yaml
 waitForHttpResponse http://local.projectcontour.io 1 100
 kubectl::delete -f examples/gateway/kuard/kuard.yaml
-# TODO [danehans]: Uncomment the following when contour-operator/issues/213 is fixed.
-# kubectl::delete -f examples/gateway/gateway-nodeport.yaml
-# kubectl::delete -f examples/operator/operator.yaml
-# kubectl::delete ns projectcontour
+kubectl::delete -f examples/gateway/gateway-nodeport.yaml
+kubectl::delete -f examples/operator/operator.yaml
 
 if ${RESP} == false ; then
   echo "examples test passed"

internal/objects/gateway/gateway.go

@@ -105,3 +105,23 @@ func ContourForGateway(ctx context.Context, cli client.Client, gw *gatewayv1alph
 	}
 	return cntr, nil
 }
+
+// OtherGatewaysRefGatewayClass returns true if other gateways have the same
+// gatewayClassName as gw.
+func OtherGatewaysRefGatewayClass(ctx context.Context, cli client.Client, gw *gatewayv1alpha1.Gateway) (bool, error) {
+	gwList, err := OtherGatewaysExist(ctx, cli, gw)
+	if err != nil {
+		return false, fmt.Errorf("failed to verify if gateways other than %s/%s exist: %v", gw.Namespace, gw.Name, err)
+	}
+	if gwList != nil {
+		for _, g := range gwList.Items {
+			switch {
+			case g.Namespace == gw.Namespace && g.Name == gw.Name:
+				continue
+			case g.Spec.GatewayClassName == gw.Spec.GatewayClassName:
+				return true, nil
+			}
+		}
+	}
+	return false, nil
+}

internal/objects/gatewayclass/finalizer.go

@@ -0,0 +1,60 @@
+// Copyright Project Contour Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gatewayclass
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/projectcontour/contour-operator/pkg/slice"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	gatewayv1alpha1 "sigs.k8s.io/gateway-api/apis/v1alpha1"
+)
+
+const finalizer = gatewayv1alpha1.GatewayClassFinalizerGatewaysExist
+
+// IsFinalized returns true if gc is finalized.
+func IsFinalized(gc *gatewayv1alpha1.GatewayClass) bool {
+	for _, f := range gc.Finalizers {
+		if f == finalizer {
+			return true
+		}
+	}
+	return false
+}
+
+// EnsureFinalizer ensures the finalizer is added to the given gc.
+func EnsureFinalizer(ctx context.Context, cli client.Client, gc *gatewayv1alpha1.GatewayClass) error {
+	if !slice.ContainsString(gc.Finalizers, finalizer) {
+		updated := gc.DeepCopy()
+		updated.Finalizers = append(updated.Finalizers, finalizer)
+		if err := cli.Update(ctx, updated); err != nil {
+			return fmt.Errorf("failed to add finalizer %s: %w", finalizer, err)
+		}
+	}
+	return nil
+}
+
+// EnsureFinalizerRemoved ensures the finalizer is removed for the given gc.
+func EnsureFinalizerRemoved(ctx context.Context, cli client.Client, gc *gatewayv1alpha1.GatewayClass) error {
+	if slice.ContainsString(gc.Finalizers, finalizer) {
+		updated := gc.DeepCopy()
+		updated.Finalizers = slice.RemoveString(updated.Finalizers, finalizer)
+		if err := cli.Update(ctx, updated); err != nil {
+			return fmt.Errorf("failed to remove finalizer %s: %w", finalizer, err)
+		}
+	}
+	return nil
+}

internal/objects/gatewayclass/gatewayclass.go

@@ -70,3 +70,27 @@ func ParameterRefExists(ctx context.Context, cli client.Client, name, ns string)
 	}
 	return nil, false, nil
 }
+
+// OtherGatewayClassesRefContour returns true if GatewayClasses other than gc reference contour.
+func OtherGatewayClassesRefContour(ctx context.Context, cli client.Client, gc *gatewayv1alpha1.GatewayClass, contour *operatorv1alpha1.Contour) (bool, error) {
+	gcList := &gatewayv1alpha1.GatewayClassList{}
+	if err := cli.List(ctx, gcList); err != nil {
+		return false, fmt.Errorf("failed to list gateways")
+	}
+	if gcList != nil {
+		for _, g := range gcList.Items {
+			if g.Spec.ParametersRef != nil {
+				switch {
+				case g.Name == gc.Name && g.Namespace == gc.Namespace:
+					continue
+				case g.Spec.ParametersRef.Namespace == contour.Namespace &&
+					g.Spec.ParametersRef.Name == contour.Name &&
+					g.Spec.ParametersRef.Scope == "Namespace" &&
+					g.Spec.ParametersRef.Kind == contour.Kind:
+					return true, nil
+				}
+			}
+		}
+	}
+	return false, nil
+}

internal/operator/controller/contour/controller.go

@@ -173,6 +173,16 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 				}
 			}
 			r.log.Info("ensured contour", "namespace", contour.Namespace, "name", contour.Name)
+		case contour.GatewayClassSet():
+			if err := r.ensureContourForGatewayClass(ctx, contour); err != nil {
+				switch e := err.(type) {
+				case retryable.Error:
+					r.log.Error(e, "got retryable error; requeueing", "after", e.After())
+					return ctrl.Result{RequeueAfter: e.After()}, nil
+				default:
+					return ctrl.Result{}, err
+				}
+			}
 		default:
 			// Before doing anything with the contour, ensure it has a finalizer
 			// so it can cleaned-up later.
@@ -182,16 +192,22 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 			r.log.Info("finalized contour", "namespace", contour.Namespace, "name", contour.Name)
 		}
 	} else {
-		if err := r.ensureContourDeleted(ctx, contour); err != nil {
-			switch e := err.(type) {
-			case retryable.Error:
-				r.log.Error(e, "got retryable error; requeueing", "after", e.After())
-				return ctrl.Result{RequeueAfter: e.After()}, nil
-			default:
-				return ctrl.Result{}, err
+		switch {
+		case contour.GatewayClassSet():
+			// Do nothing since the contour is a dependent resource of the associated gatewayclass.
+			return ctrl.Result{}, nil
+		default:
+			if err := r.ensureContourDeleted(ctx, contour); err != nil {
+				switch e := err.(type) {
+				case retryable.Error:
+					r.log.Error(e, "got retryable error; requeueing", "after", e.After())
+					return ctrl.Result{RequeueAfter: e.After()}, nil
+				default:
+					return ctrl.Result{}, err
+				}
 			}
+			r.log.Info("deleted contour", "namespace", contour.Namespace, "name", contour.Name)
 		}
-		r.log.Info("deleted contour", "namespace", contour.Namespace, "name", contour.Name)
 	}
 	return ctrl.Result{}, nil
 }
@@ -200,77 +216,90 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 func (r *reconciler) ensureContour(ctx context.Context, contour *operatorv1alpha1.Contour) error {
 	var errs []error
 	cli := r.client
-	if contour.GatewayClassSet() {
-		gcRef := *contour.Spec.GatewayClassRef
-		gc, err := objgc.Get(ctx, r.client, gcRef)
-		if err != nil {
-			errs = append(errs, fmt.Errorf("failed to verify the existence of gatewayclass %s: %w", gcRef, err))
+	if err := objns.EnsureNamespace(ctx, cli, contour); err != nil {
+		errs = append(errs, fmt.Errorf("failed to ensure namespace %s for contour %s/%s: %w",
+			contour.Spec.Namespace.Name, contour.Namespace, contour.Name, err))
+	} else {
+		r.log.Info("ensured namespace for contour", "namespace", contour.Namespace, "name", contour.Name)
+	}
+	if err := objutil.EnsureRBAC(ctx, cli, contour); err != nil {
+		errs = append(errs, fmt.Errorf("failed to ensure rbac for contour %s/%s: %w", contour.Namespace, contour.Name, err))
+	} else {
+		r.log.Info("ensured rbac for contour", "namespace", contour.Namespace, "name", contour.Name)
+	}
+	if len(errs) == 0 {
+		if err := objcm.EnsureConfigMap(ctx, cli, contour); err != nil {
+			errs = append(errs, fmt.Errorf("failed to ensure configmap for contour %s/%s: %w", contour.Namespace, contour.Name, err))
 		} else {
-			owned := objgc.IsController(gc)
-			valid := false
-			if owned {
-				if err := validation.GatewayClass(gc); err != nil {
-					errs = append(errs, fmt.Errorf("invalid gatewayclass %s: %w", gc.Name, err))
-				} else {
-					valid = true
-				}
-			}
-			if err := status.SyncGatewayClass(ctx, r.client, gc, owned, valid); err != nil {
-				errs = append(errs, fmt.Errorf("failed to sync status for gatewayclass %s: %w", gcRef, err))
-			} else {
-				r.log.Info("synced gatewayclass status for contour", "namespace", contour.Namespace, "name", contour.Name)
-			}
+			r.log.Info("ensured configmap for contour", "namespace", contour.Namespace, "name", contour.Name)
 		}
-	} else {
-		if err := objns.EnsureNamespace(ctx, cli, contour); err != nil {
-			errs = append(errs, fmt.Errorf("failed to ensure namespace %s for contour %s/%s: %w",
-				contour.Spec.Namespace.Name, contour.Namespace, contour.Name, err))
+		contourImage := r.config.ContourImage
+		if err := objjob.EnsureJob(ctx, cli, contour, contourImage); err != nil {
+			errs = append(errs, fmt.Errorf("failed to ensure job for contour %s/%s: %w", contour.Namespace, contour.Name, err))
 		} else {
-			r.log.Info("ensured namespace for contour", "namespace", contour.Namespace, "name", contour.Name)
+			r.log.Info("ensured job for contour", "namespace", contour.Namespace, "name", contour.Name)
 		}
-		if err := objutil.EnsureRBAC(ctx, cli, contour); err != nil {
-			errs = append(errs, fmt.Errorf("failed to ensure rbac for contour %s/%s: %w", contour.Namespace, contour.Name, err))
+		if err := objdeploy.EnsureDeployment(ctx, cli, contour, contourImage); err != nil {
+			errs = append(errs, fmt.Errorf("failed to ensure deployment for contour %s/%s: %w", contour.Namespace, contour.Name, err))
 		} else {
-			r.log.Info("ensured rbac for contour", "namespace", contour.Namespace, "name", contour.Name)
+			r.log.Info("ensured deployment for contour", "namespace", contour.Namespace, "name", contour.Name)
 		}
-		if len(errs) == 0 {
-			if err := objcm.EnsureConfigMap(ctx, cli, contour); err != nil {
-				errs = append(errs, fmt.Errorf("failed to ensure configmap for contour %s/%s: %w", contour.Namespace, contour.Name, err))
-			} else {
-				r.log.Info("ensured configmap for contour", "namespace", contour.Namespace, "name", contour.Name)
-			}
-			contourImage := r.config.ContourImage
-			if err := objjob.EnsureJob(ctx, cli, contour, contourImage); err != nil {
-				errs = append(errs, fmt.Errorf("failed to ensure job for contour %s/%s: %w", contour.Namespace, contour.Name, err))
-			} else {
-				r.log.Info("ensured job for contour", "namespace", contour.Namespace, "name", contour.Name)
-			}
-			if err := objdeploy.EnsureDeployment(ctx, cli, contour, contourImage); err != nil {
-				errs = append(errs, fmt.Errorf("failed to ensure deployment for contour %s/%s: %w", contour.Namespace, contour.Name, err))
-			} else {
-				r.log.Info("ensured deployment for contour", "namespace", contour.Namespace, "name", contour.Name)
-			}
-			envoyImage := r.config.EnvoyImage
-			if err := objds.EnsureDaemonSet(ctx, cli, contour, contourImage, envoyImage); err != nil {
-				errs = append(errs, fmt.Errorf("failed to ensure daemonset for contour %s/%s: %w", contour.Namespace, contour.Name, err))
+		envoyImage := r.config.EnvoyImage
+		if err := objds.EnsureDaemonSet(ctx, cli, contour, contourImage, envoyImage); err != nil {
+			errs = append(errs, fmt.Errorf("failed to ensure daemonset for contour %s/%s: %w", contour.Namespace, contour.Name, err))
+		} else {
+			r.log.Info("ensured daemonset for contour", "namespace", contour.Namespace, "name", contour.Name)
+		}
+		if err := objsvc.EnsureContourService(ctx, cli, contour); err != nil {
+			errs = append(errs, fmt.Errorf("failed to ensure contour service for contour %s/%s: %w", contour.Namespace, contour.Name, err))
+		} else {
+			r.log.Info("ensured contour service for contour", "namespace", contour.Namespace, "name", contour.Name)
+		}
+		if contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.LoadBalancerServicePublishingType ||
+			contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.NodePortServicePublishingType {
+			if err := objsvc.EnsureEnvoyService(ctx, cli, contour); err != nil {
+				errs = append(errs, fmt.Errorf("failed to ensure envoy service for contour %s/%s: %w",
+					contour.Namespace, contour.Name, err))
 			} else {
-				r.log.Info("ensured daemonset for contour", "namespace", contour.Namespace, "name", contour.Name)
+				r.log.Info("ensured envoy service for contour", "namespace", contour.Namespace, "name", contour.Name)
 			}
-			if err := objsvc.EnsureContourService(ctx, cli, contour); err != nil {
-				errs = append(errs, fmt.Errorf("failed to ensure contour service for contour %s/%s: %w", contour.Namespace, contour.Name, err))
+		}
+	}
+	if err := status.SyncContour(ctx, cli, contour); err != nil {
+		errs = append(errs, fmt.Errorf("failed to sync status for contour %s/%s: %w", contour.Namespace, contour.Name, err))
+	} else {
+		r.log.Info("synced status for contour", "namespace", contour.Namespace, "name", contour.Name)
+	}
+	return retryable.NewMaybeRetryableAggregate(errs)
+}
+
+// ensureContourForGatewayClass ensures all necessary resources exist for the given contour
+// when the contour is being managed by a GatewayClass.
+func (r *reconciler) ensureContourForGatewayClass(ctx context.Context, contour *operatorv1alpha1.Contour) error {
+	if contour.Spec.GatewayClassRef == nil {
+		return fmt.Errorf("gatewayclass not set for contour %s/%s", contour.Namespace, contour.Name)
+	}
+	var errs []error
+	cli := r.client
+	gcRef := *contour.Spec.GatewayClassRef
+	gc, err := objgc.Get(ctx, cli, gcRef)
+	if err != nil {
+		errs = append(errs, fmt.Errorf("failed to verify the existence of gatewayclass %s: %w", gcRef, err))
+	} else {
+		owned := objgc.IsController(gc)
+		valid := false
+		if owned {
+			if err := validation.GatewayClass(gc); err != nil {
+				errs = append(errs, fmt.Errorf("invalid gatewayclass %s: %w", gc.Name, err))
 			} else {
-				r.log.Info("ensured contour service for contour", "namespace", contour.Namespace, "name", contour.Name)
-			}
-			if contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.LoadBalancerServicePublishingType ||
-				contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.NodePortServicePublishingType {
-				if err := objsvc.EnsureEnvoyService(ctx, cli, contour); err != nil {
-					errs = append(errs, fmt.Errorf("failed to ensure envoy service for contour %s/%s: %w",
-						contour.Namespace, contour.Name, err))
-				} else {
-					r.log.Info("ensured envoy service for contour", "namespace", contour.Namespace, "name", contour.Name)
-				}
+				valid = true
 			}
 		}
+		if err := status.SyncGatewayClass(ctx, r.client, gc, owned, valid); err != nil {
+			errs = append(errs, fmt.Errorf("failed to sync status for gatewayclass %s: %w", gcRef, err))
+		} else {
+			r.log.Info("synced gatewayclass status for contour", "namespace", contour.Namespace, "name", contour.Name)
+		}
 	}
 	if err := status.SyncContour(ctx, cli, contour); err != nil {
 		errs = append(errs, fmt.Errorf("failed to sync status for contour %s/%s: %w", contour.Namespace, contour.Name, err))
@@ -330,12 +359,12 @@ func (r *reconciler) ensureContourDeleted(ctx context.Context, contour *operator
 		} else {
 			r.log.Info("deleted namespace for contour", "namespace", contour.Namespace, "name", contour.Name)
 		}
-	}
-	if len(errs) == 0 {
-		if err := objcontour.EnsureFinalizerRemoved(ctx, cli, contour); err != nil {
-			errs = append(errs, fmt.Errorf("failed to remove finalizer from contour %s/%s: %w", contour.Namespace, contour.Name, err))
-		} else {
-			r.log.Info("removed finalizer from contour", "namespace", contour.Namespace, "name", contour.Name)
+		if len(errs) == 0 {
+			if err := objcontour.EnsureFinalizerRemoved(ctx, cli, contour); err != nil {
+				errs = append(errs, fmt.Errorf("failed to remove finalizer from contour %s/%s: %w", contour.Namespace, contour.Name, err))
+			} else {
+				r.log.Info("removed finalizer from contour", "namespace", contour.Namespace, "name", contour.Name)
+			}
 		}
 	}
 	return utilerrors.NewAggregate(errs)