Create envoy service with ClusterIPService type and gatewayClassRef (#…

…293)

operator does not create envoy service when envoy
is ClusterIPService type and gatewayClassRef.

This patch fixes it.

Signed-off-by: Kenjiro Nakayama <nakayamakenjiro@gmail.com>

internal/operator/controller/gateway/controller.go

@@ -242,7 +242,8 @@ func (r *reconciler) ensureGateway(ctx context.Context, gw *gatewayv1alpha1.Gate
 			r.log.Info("ensured contour service for contour", "namespace", contour.Namespace, "name", contour.Name)
 		}
 		if contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.LoadBalancerServicePublishingType ||
-			contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.NodePortServicePublishingType {
+			contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.NodePortServicePublishingType ||
+			contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.ClusterIPServicePublishingType {
 			if err := objsvc.EnsureEnvoyService(ctx, cli, contour); err != nil {
 				errs = append(errs, fmt.Errorf("failed to ensure envoy service for contour %s/%s: %w",
 					contour.Namespace, contour.Name, err))
@@ -263,7 +264,8 @@ func (r *reconciler) ensureGatewayDeleted(ctx context.Context, gw *gatewayv1alph
 		return fmt.Errorf("failed to get contour for gateway %s/%s", gw.Namespace, gw.Name)
 	}
 	if contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.LoadBalancerServicePublishingType ||
-		contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.NodePortServicePublishingType {
+		contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.NodePortServicePublishingType ||
+		contour.Spec.NetworkPublishing.Envoy.Type == operatorv1alpha1.ClusterIPServicePublishingType {
 		if err := objsvc.EnsureEnvoyServiceDeleted(ctx, cli, contour); err != nil {
 			errs = append(errs, fmt.Errorf("failed to delete envoy service for contour %s/%s: %w", contour.Namespace, contour.Name, err))
 		} else {

test/e2e/operator_test.go

@@ -29,6 +29,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	gatewayv1alpha1 "sigs.k8s.io/gateway-api/apis/v1alpha1"
 )
@@ -602,6 +603,152 @@ func TestGateway(t *testing.T) {
 	t.Logf("observed the deletion of namespace %s", cfg.SpecNs)
 }
 
+func TestGatewayClusterIP(t *testing.T) {
+	testName := "test-clusterip-gateway"
+	contourName := fmt.Sprintf("%s-contour", testName)
+	gcName := "test-gatewayclass"
+	cfg := objcontour.Config{
+		Name:         contourName,
+		Namespace:    operatorNs,
+		SpecNs:       specNs,
+		NetworkType:  operatorv1alpha1.ClusterIPServicePublishingType,
+		GatewayClass: &gcName,
+	}
+
+	cntr, err := newContour(ctx, kclient, cfg)
+	if err != nil {
+		t.Fatalf("failed to create contour %s/%s: %v", operatorNs, contourName, err)
+	}
+	t.Logf("created contour %s/%s", cntr.Namespace, cntr.Name)
+
+	if err := newOperatorGatewayClass(ctx, kclient, gcName, operatorNs, contourName); err != nil {
+		t.Fatalf("failed to create gatewayclass %s: %v", gcName, err)
+	}
+	t.Logf("created gatewayclass %s", gcName)
+
+	// The gatewayclass should now report admitted.
+	if err := waitForGatewayClassStatusConditions(ctx, kclient, 1*time.Minute, gcName, expectedGatewayClassConditions...); err != nil {
+		t.Fatalf("failed to observe expected status conditions for gatewayclass %s: %v", gcName, err)
+	}
+
+	// The contour should now report available.
+	if err := waitForContourStatusConditions(ctx, kclient, 1*time.Minute, contourName, operatorNs, expectedContourConditions...); err != nil {
+		t.Fatalf("failed to observe expected status conditions for contour %s/%s: %v", operatorNs, testName, err)
+	}
+	t.Logf("observed expected status conditions for contour %s/%s", testName, operatorNs)
+
+	// Create the gateway namespace if it doesn't exist.
+	if err := newNs(ctx, kclient, cfg.SpecNs); err != nil {
+		t.Fatalf("failed to create namespace %s: %v", cfg.SpecNs, err)
+	}
+	t.Logf("created namespace %s", cfg.SpecNs)
+
+	// Create the gateway. The gateway must be projectcontour/contour until the following issue is fixed:
+	// https://github.com/projectcontour/contour-operator/issues/241
+	gwName := "contour"
+	appName := fmt.Sprintf("%s-%s", testAppName, testName)
+	if err := newGateway(ctx, kclient, cfg.SpecNs, gwName, gcName, "app", appName); err != nil {
+		t.Fatalf("failed to create gateway %s/%s: %v", cfg.SpecNs, gwName, err)
+	}
+	t.Logf("created gateway %s/%s", cfg.SpecNs, gwName)
+
+	// The gateway should report admitted.
+	if err := waitForGatewayStatusConditions(ctx, kclient, 3*time.Minute, gwName, cfg.SpecNs, expectedGatewayConditions...); err != nil {
+		t.Fatalf("failed to observe expected status conditions for gateway %s/%s: %v", cfg.SpecNs, gwName, err)
+	}
+
+	// Create a sample workload for e2e testing.
+	if err := newDeployment(ctx, kclient, appName, cfg.SpecNs, testAppImage, testAppReplicas); err != nil {
+		t.Fatalf("failed to create deployment %s/%s: %v", cfg.SpecNs, appName, err)
+	}
+	t.Logf("created deployment %s/%s", cfg.SpecNs, appName)
+
+	if err := waitForDeploymentStatusConditions(ctx, kclient, 3*time.Minute, appName, cfg.SpecNs, expectedDeploymentConditions...); err != nil {
+		t.Fatalf("failed to observe expected status conditions for deployment %s/%s: %v", cfg.SpecNs, appName, err)
+	}
+	t.Logf("observed expected status conditions for deployment %s/%s", cfg.SpecNs, appName)
+
+	if err := newClusterIPService(ctx, kclient, appName, cfg.SpecNs, 80, 8080); err != nil {
+		t.Fatalf("failed to create service %s/%s: %v", cfg.SpecNs, appName, err)
+	}
+	t.Logf("created service %s/%s", cfg.SpecNs, appName)
+
+	if err := newHTTPRouteToSvc(ctx, kclient, appName, cfg.SpecNs, appName, "app", appName, "local.projectcontour.io", int32(80)); err != nil {
+		t.Fatalf("failed to create httproute %s/%s: %v", cfg.SpecNs, appName, err)
+	}
+	t.Logf("created httproute %s/%s", cfg.SpecNs, appName)
+
+	// Create the client Pod.
+	cliName := "test-client"
+	cliPod, err := newPod(ctx, kclient, specNs, cliName, "curlimages/curl:7.75.0", []string{"sleep", "600"})
+	if err != nil {
+		t.Fatalf("failed to create pod %s/%s: %v", specNs, cliName, err)
+	}
+	if err := waitForPodStatusConditions(ctx, kclient, 1*time.Minute, cliPod.Namespace, cliPod.Name, expectedPodConditions...); err != nil {
+		t.Fatalf("failed to observe expected conditions for pod %s/%s: %v", cliPod.Namespace, cliPod.Name, err)
+	}
+	t.Logf("observed expected status conditions for pod %s/%s", cliPod.Namespace, cliPod.Name)
+
+	// Get the Envoy ClusterIP to curl.
+	svcName := "envoy"
+	ip, err := envoyClusterIP(ctx, kclient, specNs, svcName)
+	if err != nil {
+		t.Fatalf("failed to get clusterIP for service %s/%s: %v", specNs, svcName, err)
+	}
+
+	// Curl the ingress from the client pod.
+	url := fmt.Sprintf("http://%s/", ip)
+	host := fmt.Sprintf("Host: %s", "local.projectcontour.io")
+	cmd := []string{"curl", "-H", host, "-s", "-w", "%{http_code}", url}
+	resp := "200"
+	// Polling until success since network seems not ready.
+	// It might be related to https://github.com/projectcontour/contour-operator/issues/296
+	// TODO: remove wait.PollImmediate.
+	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
+		if err := parse.StringInPodExec(specNs, cliName, resp, cmd); err != nil {
+			t.Logf("observed unexpected error: %v", err)
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatalf("failed to get http %s response for %s in pod %s/%s: %v", resp, url, specNs, cliName, err)
+	}
+
+	t.Logf("received http %s response for %s in pod %s/%s", resp, url, specNs, cliName)
+
+	// TODO [danehans]: Scrape operator logs for error messages before proceeding.
+	// xref: https://github.com/projectcontour/contour-operator/issues/211
+
+	// Ensure the gateway can be deleted and clean-up.
+	if err := deleteGateway(ctx, kclient, 3*time.Minute, gwName, cfg.SpecNs); err != nil {
+		t.Fatalf("failed to delete gateway %s/%s: %v", cfg.SpecNs, gwName, err)
+	}
+
+	// Ensure the gatewayclass can be deleted and clean-up.
+	if err := deleteGatewayClass(ctx, kclient, 3*time.Minute, gcName); err != nil {
+		t.Fatalf("failed to delete gatewayclass %s: %v", gcName, err)
+	}
+
+	// Ensure the contour can be deleted and clean-up.
+	if err := deleteContour(ctx, kclient, 3*time.Minute, contourName, operatorNs); err != nil {
+		t.Fatalf("failed to delete contour %s/%s: %v", operatorNs, contourName, err)
+	}
+
+	// Ensure the envoy service is cleaned up automatically.
+	if err := waitForServiceDeletion(ctx, kclient, 3*time.Minute, specNs, "envoy"); err != nil {
+		t.Fatalf("failed to delete contour %s/envoy: %v", specNs, err)
+	}
+	t.Logf("cleaned up envoy service %s/envoy", specNs)
+
+	// Delete the operand namespace since contour.spec.namespace.removeOnDeletion
+	// defaults to false.
+	if err := deleteNamespace(ctx, kclient, 5*time.Minute, cfg.SpecNs); err != nil {
+		t.Fatalf("failed to delete namespace %s: %v", cfg.SpecNs, err)
+	}
+	t.Logf("observed the deletion of namespace %s", cfg.SpecNs)
+}
+
 // TestGatewayOwnership ensures the operator only manages Gateway resources that it owns,
 // i.e. a gatewayclass that specifies the controller as the operator.
 func TestGatewayOwnership(t *testing.T) {