conformance: add support for GRPCRoute (#5776)

* set grpc as default protocol
* skip unsupported case.
* add TODO about test case conflicts

Signed-off-by: Jintao Zhang <zhangjintao9020@gmail.com>
Co-authored-by: Patryk Małek <patryk.malek@konghq.com>

CHANGELOG.md

@@ -102,6 +102,9 @@ Adding a new version? You'll need three changes:
   performance benefits, however, so labeling plugin configuration Secrets and
   enabling the filter is recommended as soon as is convenient.
   [#5856](https://github.com/Kong/kubernetes-ingress-controller/pull/5856)
+- Dynamically set the proxy protocol of GRPCRoute to `grpc` or `grpcs` based on the port listened by Gateway.
+  If you don't set the protocol for Service via `konghq.com/protocol` annotation, Kong will use `grpc` instead of `grpcs`.
+  [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776)
 - The `/debug/config/failed` and `/debug/config/successful` diagnostic
   endpoints now nest configuration dumps under a `config` key. These endpoints
   previously returned the configuration dump at the root. They now return
@@ -192,6 +195,8 @@ Adding a new version? You'll need three changes:
   [#5965](https://github.com/Kong/kubernetes-ingress-controller/pull/5965)
 - Fallback configuration no longer omits licenses and vaults.
   [#6048](https://github.com/Kong/kubernetes-ingress-controller/pull/6048)
+- Add support for Gateway API GRPCRoute and pass related Gateway API conformance test.
+  [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776)
 
 ### Fixed
 

examples/gateway-grpcroute-via-http.yaml

@@ -58,7 +58,7 @@ spec:
     protocol: HTTP
     port: 80
 ---
-apiVersion: gateway.networking.k8s.io/v1alpha2
+apiVersion: gateway.networking.k8s.io/v1
 kind: GRPCRoute
 metadata:
   name: grpcbin-via-http

examples/gateway-grpcroute-via-https.yaml

@@ -70,7 +70,7 @@ spec:
       certificateRefs:
       - name: grpcroute-example
 ---
-apiVersion: gateway.networking.k8s.io/v1alpha2
+apiVersion: gateway.networking.k8s.io/v1
 kind: GRPCRoute
 metadata:
   name: grpcbin-via-https

internal/controllers/gateway/grpcroute_controller.go

@@ -475,7 +475,7 @@ func (r *GRPCRouteReconciler) ensureGatewayReferenceStatusAdded(ctx context.Cont
 
 		// if the reference already exists and doesn't require any changes
 		// then just leave it alone.
-		parentRefKey := gateway.gateway.Namespace + "/" + gateway.gateway.Name
+		parentRefKey := fmt.Sprintf("%s/%s/%s", gateway.gateway.Namespace, gateway.gateway.Name, gateway.listenerName)
 		if existingGatewayParentStatus, exists := parentStatuses[parentRefKey]; exists {
 			//  check if the parentRef and controllerName are equal, and whether the new condition is present in existing conditions
 			if reflect.DeepEqual(existingGatewayParentStatus.ParentRef, gatewayParentStatus.ParentRef) &&

internal/controllers/gateway/route_parent_status.go

@@ -46,6 +46,11 @@ func routeParentStatusKey[routeT gatewayapi.RouteT](
 			namespace,
 			parentRef.GetName(),
 			parentRef.GetSectionName().OrEmpty())
+	case *gatewayapi.GRPCRoute:
+		return fmt.Sprintf("%s/%s/%s",
+			namespace,
+			parentRef.GetName(),
+			parentRef.GetSectionName().OrEmpty())
 	default:
 		return fmt.Sprintf("%s/%s", namespace, parentRef.GetName())
 	}

internal/dataplane/translator/subtranslator/grpcroute_test.go

@@ -16,13 +16,13 @@ import (
 
 var grpcRouteGVK = schema.GroupVersionKind{
 	Group:   "gateway.networking.k8s.io",
-	Version: "v1alpha2",
+	Version: "v1",
 	Kind:    "GRPCRoute",
 }
 
 var grpcRouteTypeMeta = metav1.TypeMeta{
 	Kind:       "GRPCRoute",
-	APIVersion: "gateway.networking.k8s.io/v1alpha2",
+	APIVersion: "gateway.networking.k8s.io/v1",
 }
 
 func makeTestGRPCRoute(
@@ -33,7 +33,7 @@ func makeTestGRPCRoute(
 	return &gatewayapi.GRPCRoute{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "GRPCRoute",
-			APIVersion: "gateway.networking.k8s.io/v1alpha2",
+			APIVersion: "gateway.networking.k8s.io/v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        name,
@@ -98,7 +98,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) {
 							"k8s-namespace:default",
 							"k8s-kind:GRPCRoute",
 							"k8s-group:gateway.networking.k8s.io",
-							"k8s-version:v1alpha2",
+							"k8s-version:v1",
 						),
 					},
 				},
@@ -139,7 +139,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) {
 							"k8s-namespace:default",
 							"k8s-kind:GRPCRoute",
 							"k8s-group:gateway.networking.k8s.io",
-							"k8s-version:v1alpha2",
+							"k8s-version:v1",
 						),
 					},
 				},
@@ -194,7 +194,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) {
 							"k8s-namespace:default",
 							"k8s-kind:GRPCRoute",
 							"k8s-group:gateway.networking.k8s.io",
-							"k8s-version:v1alpha2",
+							"k8s-version:v1",
 						),
 					},
 				},
@@ -214,7 +214,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) {
 							"k8s-namespace:default",
 							"k8s-kind:GRPCRoute",
 							"k8s-group:gateway.networking.k8s.io",
-							"k8s-version:v1alpha2",
+							"k8s-version:v1",
 						),
 					},
 				},
@@ -243,7 +243,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) {
 							"k8s-namespace:default",
 							"k8s-kind:GRPCRoute",
 							"k8s-group:gateway.networking.k8s.io",
-							"k8s-version:v1alpha2",
+							"k8s-version:v1",
 						),
 					},
 				},
@@ -270,7 +270,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) {
 							"k8s-namespace:default",
 							"k8s-kind:GRPCRoute",
 							"k8s-group:gateway.networking.k8s.io",
-							"k8s-version:v1alpha2",
+							"k8s-version:v1",
 						),
 						Paths: kong.StringSlice("/"),
 					},

internal/dataplane/translator/translate_grpcroute.go

@@ -53,10 +53,9 @@ func (t *Translator) ingressRulesFromGRPCRoute(result *ingressRules, grpcroute *
 	// each rule may represent a different set of backend services that will be accepting
 	// traffic, so we make separate routes and Kong services for every present rule.
 	for ruleNumber, rule := range spec.Rules {
-		// Create a service and attach the routes to it. Protocol for Service can be set via K8s object annotation
-		// "konghq.com/protocol", by default use "grpcs" to not break existing behavior when annotation is not specified.
+		// Create a service and attach the routes to it.
 		service, err := generateKongServiceFromBackendRefWithRuleNumber(
-			t.logger, t.storer, result, grpcroute, ruleNumber, "grpcs", grpcBackendRefsToBackendRefs(rule.BackendRefs)...,
+			t.logger, t.storer, result, grpcroute, ruleNumber, t.getProtocolForKongService(grpcroute), grpcBackendRefsToBackendRefs(rule.BackendRefs)...,
 		)
 		if err != nil {
 			return err
@@ -116,15 +115,14 @@ func (t *Translator) ingressRulesFromGRPCRouteWithPriority(
 
 	serviceName := subtranslator.KongServiceNameFromSplitGRPCRouteMatch(match)
 
-	// Create a service and attach the routes to it. Protocol for Service can be set via K8s object annotation
-	// "konghq.com/protocol", by default use "grpcs" to not break existing behavior when annotation is not specified.
+	// Create a service and attach the routes to it.
 	kongService, _ := generateKongServiceFromBackendRefWithName(
 		t.logger,
 		t.storer,
 		rules,
 		serviceName,
 		grpcRoute,
-		"grpcs",
+		t.getProtocolForKongService(grpcRoute),
 		grpcBackendRefsToBackendRefs(grpcRouteRule.BackendRefs)...,
 	)
 	kongService.Routes = append(
@@ -144,3 +142,13 @@ func grpcBackendRefsToBackendRefs(grpcBackendRef []gatewayapi.GRPCBackendRef) []
 	}
 	return backendRefs
 }
+
+// getProtocolForKongService returns the protocol for the Kong service configuration.
+// In order to get the protocol, provided route's parentRefs are searched for a Gateway that has the matching listening ports.
+func (t *Translator) getProtocolForKongService(grpcRoute *gatewayapi.GRPCRoute) string {
+	// When Gateway listens on HTTP use "grpc" protocol for the service. Otherwise for HTTPS use "grpcs".
+	if len(t.getGatewayListeningPorts(grpcRoute.Namespace, gatewayapi.HTTPProtocolType, grpcRoute.Spec.ParentRefs)) > 0 {
+		return "grpc"
+	}
+	return "grpcs"
+}

test/conformance/gateway_conformance_test.go

@@ -24,12 +24,27 @@ import (
 var skippedTestsForTraditionalRoutes = []string{
 	// core conformance
 	tests.HTTPRouteHeaderMatching.ShortName,
+	// There is an issue with KIC when processing this scenario.
+	// TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6136
+	tests.GRPCRouteListenerHostnameMatching.ShortName,
+	// tests.GRPCRouteHeaderMatching.ShortName and tests.GRPCExactMethodMatching.ShortName may
+	// have some conflicts, skipping either one will still pass normally.
+	// TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6144
+	tests.GRPCExactMethodMatching.ShortName,
+}
+
+var skippedTestsForExpressionRoutes = []string{
+	// When processing this scenario, the Kong's expressions router requires `priority`
+	// to be specified for routes.
+	// We cannot provide that for routes that are part of the conformance suite.
+	tests.GRPCRouteListenerHostnameMatching.ShortName,
 }
 
 var traditionalRoutesSupportedFeatures = []features.SupportedFeature{
 	// core features
 	features.SupportGateway,
 	features.SupportHTTPRoute,
+	features.SupportGRPCRoute,
 	// extended features
 	features.SupportHTTPRouteResponseHeaderModification,
 	features.SupportHTTPRoutePathRewrite,
@@ -43,6 +58,7 @@ var expressionRoutesSupportedFeatures = []features.SupportedFeature{
 	// core features
 	features.SupportGateway,
 	features.SupportHTTPRoute,
+	features.SupportGRPCRoute,
 	// extended features
 	features.SupportHTTPRouteQueryParamMatching,
 	features.SupportHTTPRouteMethodMatching,
@@ -70,6 +86,7 @@ func TestGatewayConformance(t *testing.T) {
 		supportedFeatures = traditionalRoutesSupportedFeatures
 		mode = string(dpconf.RouterFlavorTraditionalCompatible)
 	case dpconf.RouterFlavorExpressions:
+		skippedTests = skippedTestsForExpressionRoutes
 		supportedFeatures = expressionRoutesSupportedFeatures
 		mode = string(dpconf.RouterFlavorExpressions)
 	default:
@@ -86,6 +103,7 @@ func TestGatewayConformance(t *testing.T) {
 	opts.SkipTests = skippedTests
 	opts.ConformanceProfiles = sets.New(
 		suite.GatewayHTTPConformanceProfileName,
+		suite.GatewayGRPCConformanceProfileName,
 	)
 	opts.Implementation = conformancev1.Implementation{
 		Organization: metadata.Organization,

test/conformance/suite_test.go

@@ -82,6 +82,10 @@ func TestMain(m *testing.M) {
 		kongBuilder = kongBuilder.WithProxyEnvVar("router_flavor", string(dpconf.RouterFlavorExpressions))
 	}
 
+	// The test cases for GRPCRoute in the current GatewayAPI all use the h2c protocol.
+	// In order to pass conformance tests, the proxy must listen http2 and http on the same port.
+	kongBuilder.WithProxyEnvVar("PROXY_LISTEN", `0.0.0.0:8000 http2\, 0.0.0.0:8443 http2 ssl`)
+
 	// Pin the Helm chart version.
 	kongBuilder.WithHelmChartVersion(testenv.KongHelmChartVersion())
 

test/consts.go

@@ -24,7 +24,9 @@ const (
 	// GRPCBinImage is the container image name we use for deploying the "grpcbin" GRPC testing tool.
 	// See: https://github.com/Kong/grpcbin
 	GRPCBinImage = "kong/grpcbin:latest"
-	GRPCBinPort  = 9001
+
+	GRPCBinPort  int32 = 9000
+	GRPCSBinPort int32 = 9001
 
 	// EnvironmentCleanupTimeout is the amount of time that will be given by the test suite to the
 	// testing environment to perform its cleanup when the test suite is shutting down.

test/integration/isolated/grpc_test.go

@@ -22,15 +22,13 @@ import (
 	"google.golang.org/grpc/metadata"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	k8stypes "k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 	"sigs.k8s.io/e2e-framework/pkg/features"
 	gatewayclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned"
 
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util/builder"
 	"github.com/kong/kubernetes-ingress-controller/v3/test"
-	"github.com/kong/kubernetes-ingress-controller/v3/test/helpers/certificate"
 	"github.com/kong/kubernetes-ingress-controller/v3/test/integration/consts"
 	"github.com/kong/kubernetes-ingress-controller/v3/test/internal/helpers"
 	"github.com/kong/kubernetes-ingress-controller/v3/test/internal/testlabels"
@@ -43,9 +41,12 @@ func TestGRPCRouteEssentials(t *testing.T) {
 		New("essentials").
 		WithLabel(testlabels.NetworkingFamily, testlabels.NetworkingFamilyGatewayAPI).
 		WithLabel(testlabels.Kind, testlabels.KindGRPCRoute).
-		WithSetup("deploy kong addon into cluster", featureSetup()).
-		Assess("deploying Gateway and example GRPC service (without konghq.com/protocol annotation) exposed via GRPCRoute over HTTPS", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
-			// On purpose omit protocol annotation to test defaulting to "grpcs" that is preserved to not break users' configs.
+		WithSetup("deploy kong addon into cluster", featureSetup(
+			withKongProxyEnvVars(map[string]string{
+				"PROXY_LISTEN": `0.0.0.0:8000 http2\, 0.0.0.0:8443 http2 ssl`,
+			}),
+		)).
+		Assess("deploying Gateway and example GRPC service (without konghq.com/protocol annotation) exposed via GRPCRoute over HTTP", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
 			cleaner := GetFromCtxForT[*clusters.Cleaner](ctx, t)
 			cluster := GetClusterFromCtx(ctx)
 			namespace := GetNamespaceForT(ctx, t)
@@ -61,45 +62,12 @@ func TestGRPCRouteEssentials(t *testing.T) {
 			assert.NoError(t, err)
 			cleaner.Add(gwc)
 
-			t.Log("configuring secret")
-			const tlsRouteHostname = "tls-route.example"
-			tlsRouteExampleTLSCert, tlsRouteExampleTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithCommonName(tlsRouteHostname))
-			const tlsSecretName = "secret-test"
-			secret := &corev1.Secret{
-				ObjectMeta: metav1.ObjectMeta{
-					UID:       k8stypes.UID("7428fb98-180b-4702-a91f-61351a33c6e8"),
-					Name:      tlsSecretName,
-					Namespace: namespace,
-				},
-				Data: map[string][]byte{
-					"tls.crt": tlsRouteExampleTLSCert,
-					"tls.key": tlsRouteExampleTLSKey,
-				},
-			}
-
-			t.Log("deploying secret")
-			secret, err = cluster.Client().CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
-			assert.NoError(t, err)
-			cleaner.Add(secret)
-
 			t.Log("deploying a new gateway")
 			gateway, err := helpers.DeployGateway(ctx, gatewayClient, namespace, gatewayClassName, func(gw *gatewayapi.Gateway) {
-				// Besides default HTTP listener, add a HTTPS listener.
-				gw.Spec.Listeners = append(
-					gw.Spec.Listeners,
-					builder.NewListener("https").
-						HTTPS().
-						WithPort(ktfkong.DefaultProxyTLSServicePort).
-						WithHostname(testHostname).
-						WithTLSConfig(&gatewayapi.GatewayTLSConfig{
-							CertificateRefs: []gatewayapi.SecretObjectReference{
-								{
-									Name: gatewayapi.ObjectName(secret.Name),
-								},
-							},
-						}).
-						Build(),
-				)
+				gw.Spec.Listeners = builder.NewListener("grpc").
+					HTTP().
+					WithPort(ktfkong.DefaultProxyHTTPPort).
+					IntoSlice()
 			})
 			assert.NoError(t, err)
 			cleaner.Add(gateway)
@@ -168,7 +136,7 @@ func TestGRPCRouteEssentials(t *testing.T) {
 			return ctx
 		}).
 		Assess("checking if GRPCRoute is linked correctly and client can connect properly to the exposed service", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
-			grpcAddr := GetHTTPSURLFromCtx(ctx).Host // For GRPC, we use the same address as for HTTPS, but without the scheme (https://).
+			grpcAddr := GetHTTPURLFromCtx(ctx).Host // For GRPC, we use the same address as for HTTP, but without the scheme (http://).
 			namespace := GetNamespaceForT(ctx, t)
 			gatewayClient := GetFromCtxForT[*gatewayclient.Clientset](ctx, t)
 			grpcRoute := GetFromCtxForT[*gatewayapi.GRPCRoute](ctx, t)
@@ -184,14 +152,14 @@ func TestGRPCRouteEssentials(t *testing.T) {
 
 			t.Log("waiting for routes from GRPCRoute to become operational")
 			assert.Eventually(t, func() bool {
-				err := grpcEchoResponds(ctx, grpcAddr, testHostname, "kong", true)
+				err := grpcEchoResponds(ctx, grpcAddr, testHostname, "kong", false)
 				if err != nil {
 					t.Log(err)
 				}
 				return err == nil
 			}, consts.IngressWait, consts.WaitTick)
 
-			client, closeGrpcConn, err := grpcBinClient(grpcAddr, testHostname, true)
+			client, closeGrpcConn, err := grpcBinClient(grpcAddr, testHostname, false)
 			assert.NoError(t, err)
 			t.Cleanup(func() {
 				err := closeGrpcConn()