diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b022708ee..d01a87bef6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,9 @@ Adding a new version? You'll need three changes: performance benefits, however, so labeling plugin configuration Secrets and enabling the filter is recommended as soon as is convenient. [#5856](https://github.com/Kong/kubernetes-ingress-controller/pull/5856) +- Dynamically set the proxy protocol of GRPCRoute to `grpc` or `grpcs` based on the port listened by Gateway. + If you don't set the protocol for Service via `konghq.com/protocol` annotation, Kong will use `grpc` instead of `grpcs`. + [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776) - The `/debug/config/failed` and `/debug/config/successful` diagnostic endpoints now nest configuration dumps under a `config` key. These endpoints previously returned the configuration dump at the root. They now return @@ -192,6 +195,8 @@ Adding a new version? You'll need three changes: [#5965](https://github.com/Kong/kubernetes-ingress-controller/pull/5965) - Fallback configuration no longer omits licenses and vaults. [#6048](https://github.com/Kong/kubernetes-ingress-controller/pull/6048) +- Add support for Gateway API GRPCRoute and pass related Gateway API conformance test. + [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776) ### Fixed diff --git a/examples/gateway-grpcroute-via-http.yaml b/examples/gateway-grpcroute-via-http.yaml index bc7ebe3839..50b61c7999 100644 --- a/examples/gateway-grpcroute-via-http.yaml +++ b/examples/gateway-grpcroute-via-http.yaml @@ -58,7 +58,7 @@ spec: protocol: HTTP port: 80 --- -apiVersion: gateway.networking.k8s.io/v1alpha2 +apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: name: grpcbin-via-http diff --git a/examples/gateway-grpcroute-via-https.yaml b/examples/gateway-grpcroute-via-https.yaml index 3317bfc837..12e0871ec3 100644 --- a/examples/gateway-grpcroute-via-https.yaml +++ b/examples/gateway-grpcroute-via-https.yaml @@ -70,7 +70,7 @@ spec: certificateRefs: - name: grpcroute-example --- -apiVersion: gateway.networking.k8s.io/v1alpha2 +apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: name: grpcbin-via-https diff --git a/internal/controllers/gateway/grpcroute_controller.go b/internal/controllers/gateway/grpcroute_controller.go index 0b5cf7de7c..c9906c53a4 100644 --- a/internal/controllers/gateway/grpcroute_controller.go +++ b/internal/controllers/gateway/grpcroute_controller.go @@ -475,7 +475,7 @@ func (r *GRPCRouteReconciler) ensureGatewayReferenceStatusAdded(ctx context.Cont // if the reference already exists and doesn't require any changes // then just leave it alone. - parentRefKey := gateway.gateway.Namespace + "/" + gateway.gateway.Name + parentRefKey := fmt.Sprintf("%s/%s/%s", gateway.gateway.Namespace, gateway.gateway.Name, gateway.listenerName) if existingGatewayParentStatus, exists := parentStatuses[parentRefKey]; exists { // check if the parentRef and controllerName are equal, and whether the new condition is present in existing conditions if reflect.DeepEqual(existingGatewayParentStatus.ParentRef, gatewayParentStatus.ParentRef) && diff --git a/internal/controllers/gateway/route_parent_status.go b/internal/controllers/gateway/route_parent_status.go index 90719a8fee..f369f8f7d3 100644 --- a/internal/controllers/gateway/route_parent_status.go +++ b/internal/controllers/gateway/route_parent_status.go @@ -46,6 +46,11 @@ func routeParentStatusKey[routeT gatewayapi.RouteT]( namespace, parentRef.GetName(), parentRef.GetSectionName().OrEmpty()) + case *gatewayapi.GRPCRoute: + return fmt.Sprintf("%s/%s/%s", + namespace, + parentRef.GetName(), + parentRef.GetSectionName().OrEmpty()) default: return fmt.Sprintf("%s/%s", namespace, parentRef.GetName()) } diff --git a/internal/dataplane/translator/subtranslator/grpcroute_test.go b/internal/dataplane/translator/subtranslator/grpcroute_test.go index ada1c15518..246aef4441 100644 --- a/internal/dataplane/translator/subtranslator/grpcroute_test.go +++ b/internal/dataplane/translator/subtranslator/grpcroute_test.go @@ -16,13 +16,13 @@ import ( var grpcRouteGVK = schema.GroupVersionKind{ Group: "gateway.networking.k8s.io", - Version: "v1alpha2", + Version: "v1", Kind: "GRPCRoute", } var grpcRouteTypeMeta = metav1.TypeMeta{ Kind: "GRPCRoute", - APIVersion: "gateway.networking.k8s.io/v1alpha2", + APIVersion: "gateway.networking.k8s.io/v1", } func makeTestGRPCRoute( @@ -33,7 +33,7 @@ func makeTestGRPCRoute( return &gatewayapi.GRPCRoute{ TypeMeta: metav1.TypeMeta{ Kind: "GRPCRoute", - APIVersion: "gateway.networking.k8s.io/v1alpha2", + APIVersion: "gateway.networking.k8s.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -98,7 +98,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -139,7 +139,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -194,7 +194,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -214,7 +214,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -243,7 +243,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -270,7 +270,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), Paths: kong.StringSlice("/"), }, diff --git a/internal/dataplane/translator/translate_grpcroute.go b/internal/dataplane/translator/translate_grpcroute.go index 2487333df1..f88407d3be 100644 --- a/internal/dataplane/translator/translate_grpcroute.go +++ b/internal/dataplane/translator/translate_grpcroute.go @@ -53,10 +53,9 @@ func (t *Translator) ingressRulesFromGRPCRoute(result *ingressRules, grpcroute * // each rule may represent a different set of backend services that will be accepting // traffic, so we make separate routes and Kong services for every present rule. for ruleNumber, rule := range spec.Rules { - // Create a service and attach the routes to it. Protocol for Service can be set via K8s object annotation - // "konghq.com/protocol", by default use "grpcs" to not break existing behavior when annotation is not specified. + // Create a service and attach the routes to it. service, err := generateKongServiceFromBackendRefWithRuleNumber( - t.logger, t.storer, result, grpcroute, ruleNumber, "grpcs", grpcBackendRefsToBackendRefs(rule.BackendRefs)..., + t.logger, t.storer, result, grpcroute, ruleNumber, t.getProtocolForKongService(grpcroute), grpcBackendRefsToBackendRefs(rule.BackendRefs)..., ) if err != nil { return err @@ -116,15 +115,14 @@ func (t *Translator) ingressRulesFromGRPCRouteWithPriority( serviceName := subtranslator.KongServiceNameFromSplitGRPCRouteMatch(match) - // Create a service and attach the routes to it. Protocol for Service can be set via K8s object annotation - // "konghq.com/protocol", by default use "grpcs" to not break existing behavior when annotation is not specified. + // Create a service and attach the routes to it. kongService, _ := generateKongServiceFromBackendRefWithName( t.logger, t.storer, rules, serviceName, grpcRoute, - "grpcs", + t.getProtocolForKongService(grpcRoute), grpcBackendRefsToBackendRefs(grpcRouteRule.BackendRefs)..., ) kongService.Routes = append( @@ -144,3 +142,13 @@ func grpcBackendRefsToBackendRefs(grpcBackendRef []gatewayapi.GRPCBackendRef) [] } return backendRefs } + +// getProtocolForKongService returns the protocol for the Kong service configuration. +// In order to get the protocol, provided route's parentRefs are searched for a Gateway that has the matching listening ports. +func (t *Translator) getProtocolForKongService(grpcRoute *gatewayapi.GRPCRoute) string { + // When Gateway listens on HTTP use "grpc" protocol for the service. Otherwise for HTTPS use "grpcs". + if len(t.getGatewayListeningPorts(grpcRoute.Namespace, gatewayapi.HTTPProtocolType, grpcRoute.Spec.ParentRefs)) > 0 { + return "grpc" + } + return "grpcs" +} diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index 014bf6b3a6..c7f89fc338 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -24,12 +24,27 @@ import ( var skippedTestsForTraditionalRoutes = []string{ // core conformance tests.HTTPRouteHeaderMatching.ShortName, + // There is an issue with KIC when processing this scenario. + // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6136 + tests.GRPCRouteListenerHostnameMatching.ShortName, + // tests.GRPCRouteHeaderMatching.ShortName and tests.GRPCExactMethodMatching.ShortName may + // have some conflicts, skipping either one will still pass normally. + // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6144 + tests.GRPCExactMethodMatching.ShortName, +} + +var skippedTestsForExpressionRoutes = []string{ + // When processing this scenario, the Kong's expressions router requires `priority` + // to be specified for routes. + // We cannot provide that for routes that are part of the conformance suite. + tests.GRPCRouteListenerHostnameMatching.ShortName, } var traditionalRoutesSupportedFeatures = []features.SupportedFeature{ // core features features.SupportGateway, features.SupportHTTPRoute, + features.SupportGRPCRoute, // extended features features.SupportHTTPRouteResponseHeaderModification, features.SupportHTTPRoutePathRewrite, @@ -43,6 +58,7 @@ var expressionRoutesSupportedFeatures = []features.SupportedFeature{ // core features features.SupportGateway, features.SupportHTTPRoute, + features.SupportGRPCRoute, // extended features features.SupportHTTPRouteQueryParamMatching, features.SupportHTTPRouteMethodMatching, @@ -70,6 +86,7 @@ func TestGatewayConformance(t *testing.T) { supportedFeatures = traditionalRoutesSupportedFeatures mode = string(dpconf.RouterFlavorTraditionalCompatible) case dpconf.RouterFlavorExpressions: + skippedTests = skippedTestsForExpressionRoutes supportedFeatures = expressionRoutesSupportedFeatures mode = string(dpconf.RouterFlavorExpressions) default: @@ -86,6 +103,7 @@ func TestGatewayConformance(t *testing.T) { opts.SkipTests = skippedTests opts.ConformanceProfiles = sets.New( suite.GatewayHTTPConformanceProfileName, + suite.GatewayGRPCConformanceProfileName, ) opts.Implementation = conformancev1.Implementation{ Organization: metadata.Organization, diff --git a/test/conformance/suite_test.go b/test/conformance/suite_test.go index 51c37929ed..1e0abe24b8 100644 --- a/test/conformance/suite_test.go +++ b/test/conformance/suite_test.go @@ -82,6 +82,10 @@ func TestMain(m *testing.M) { kongBuilder = kongBuilder.WithProxyEnvVar("router_flavor", string(dpconf.RouterFlavorExpressions)) } + // The test cases for GRPCRoute in the current GatewayAPI all use the h2c protocol. + // In order to pass conformance tests, the proxy must listen http2 and http on the same port. + kongBuilder.WithProxyEnvVar("PROXY_LISTEN", `0.0.0.0:8000 http2\, 0.0.0.0:8443 http2 ssl`) + // Pin the Helm chart version. kongBuilder.WithHelmChartVersion(testenv.KongHelmChartVersion()) diff --git a/test/consts.go b/test/consts.go index 7bdb4e05fb..65d47d268e 100644 --- a/test/consts.go +++ b/test/consts.go @@ -24,7 +24,9 @@ const ( // GRPCBinImage is the container image name we use for deploying the "grpcbin" GRPC testing tool. // See: https://github.com/Kong/grpcbin GRPCBinImage = "kong/grpcbin:latest" - GRPCBinPort = 9001 + + GRPCBinPort int32 = 9000 + GRPCSBinPort int32 = 9001 // EnvironmentCleanupTimeout is the amount of time that will be given by the test suite to the // testing environment to perform its cleanup when the test suite is shutting down. diff --git a/test/integration/isolated/grpc_test.go b/test/integration/isolated/grpc_test.go index 0681cbd974..a1388ef328 100644 --- a/test/integration/isolated/grpc_test.go +++ b/test/integration/isolated/grpc_test.go @@ -22,7 +22,6 @@ import ( "google.golang.org/grpc/metadata" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8stypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" gatewayclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" @@ -30,7 +29,6 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/util/builder" "github.com/kong/kubernetes-ingress-controller/v3/test" - "github.com/kong/kubernetes-ingress-controller/v3/test/helpers/certificate" "github.com/kong/kubernetes-ingress-controller/v3/test/integration/consts" "github.com/kong/kubernetes-ingress-controller/v3/test/internal/helpers" "github.com/kong/kubernetes-ingress-controller/v3/test/internal/testlabels" @@ -43,9 +41,12 @@ func TestGRPCRouteEssentials(t *testing.T) { New("essentials"). WithLabel(testlabels.NetworkingFamily, testlabels.NetworkingFamilyGatewayAPI). WithLabel(testlabels.Kind, testlabels.KindGRPCRoute). - WithSetup("deploy kong addon into cluster", featureSetup()). - Assess("deploying Gateway and example GRPC service (without konghq.com/protocol annotation) exposed via GRPCRoute over HTTPS", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { - // On purpose omit protocol annotation to test defaulting to "grpcs" that is preserved to not break users' configs. + WithSetup("deploy kong addon into cluster", featureSetup( + withKongProxyEnvVars(map[string]string{ + "PROXY_LISTEN": `0.0.0.0:8000 http2\, 0.0.0.0:8443 http2 ssl`, + }), + )). + Assess("deploying Gateway and example GRPC service (without konghq.com/protocol annotation) exposed via GRPCRoute over HTTP", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { cleaner := GetFromCtxForT[*clusters.Cleaner](ctx, t) cluster := GetClusterFromCtx(ctx) namespace := GetNamespaceForT(ctx, t) @@ -61,45 +62,12 @@ func TestGRPCRouteEssentials(t *testing.T) { assert.NoError(t, err) cleaner.Add(gwc) - t.Log("configuring secret") - const tlsRouteHostname = "tls-route.example" - tlsRouteExampleTLSCert, tlsRouteExampleTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithCommonName(tlsRouteHostname)) - const tlsSecretName = "secret-test" - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - UID: k8stypes.UID("7428fb98-180b-4702-a91f-61351a33c6e8"), - Name: tlsSecretName, - Namespace: namespace, - }, - Data: map[string][]byte{ - "tls.crt": tlsRouteExampleTLSCert, - "tls.key": tlsRouteExampleTLSKey, - }, - } - - t.Log("deploying secret") - secret, err = cluster.Client().CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) - cleaner.Add(secret) - t.Log("deploying a new gateway") gateway, err := helpers.DeployGateway(ctx, gatewayClient, namespace, gatewayClassName, func(gw *gatewayapi.Gateway) { - // Besides default HTTP listener, add a HTTPS listener. - gw.Spec.Listeners = append( - gw.Spec.Listeners, - builder.NewListener("https"). - HTTPS(). - WithPort(ktfkong.DefaultProxyTLSServicePort). - WithHostname(testHostname). - WithTLSConfig(&gatewayapi.GatewayTLSConfig{ - CertificateRefs: []gatewayapi.SecretObjectReference{ - { - Name: gatewayapi.ObjectName(secret.Name), - }, - }, - }). - Build(), - ) + gw.Spec.Listeners = builder.NewListener("grpc"). + HTTP(). + WithPort(ktfkong.DefaultProxyHTTPPort). + IntoSlice() }) assert.NoError(t, err) cleaner.Add(gateway) @@ -168,7 +136,7 @@ func TestGRPCRouteEssentials(t *testing.T) { return ctx }). Assess("checking if GRPCRoute is linked correctly and client can connect properly to the exposed service", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { - grpcAddr := GetHTTPSURLFromCtx(ctx).Host // For GRPC, we use the same address as for HTTPS, but without the scheme (https://). + grpcAddr := GetHTTPURLFromCtx(ctx).Host // For GRPC, we use the same address as for HTTP, but without the scheme (http://). namespace := GetNamespaceForT(ctx, t) gatewayClient := GetFromCtxForT[*gatewayclient.Clientset](ctx, t) grpcRoute := GetFromCtxForT[*gatewayapi.GRPCRoute](ctx, t) @@ -184,14 +152,14 @@ func TestGRPCRouteEssentials(t *testing.T) { t.Log("waiting for routes from GRPCRoute to become operational") assert.Eventually(t, func() bool { - err := grpcEchoResponds(ctx, grpcAddr, testHostname, "kong", true) + err := grpcEchoResponds(ctx, grpcAddr, testHostname, "kong", false) if err != nil { t.Log(err) } return err == nil }, consts.IngressWait, consts.WaitTick) - client, closeGrpcConn, err := grpcBinClient(grpcAddr, testHostname, true) + client, closeGrpcConn, err := grpcBinClient(grpcAddr, testHostname, false) assert.NoError(t, err) t.Cleanup(func() { err := closeGrpcConn() diff --git a/test/integration/isolated/ingress_test.go b/test/integration/isolated/ingress_test.go index 6b14d4b851..ef531444d8 100644 --- a/test/integration/isolated/ingress_test.go +++ b/test/integration/isolated/ingress_test.go @@ -72,23 +72,19 @@ func TestIngressGRPC(t *testing.T) { gRPC kongProtocolAnnotation = "grpc" gRPCS kongProtocolAnnotation = "grpcs" ) - const ( - gRPCBinPort int32 = 9000 - gRPCSBinPort int32 = 9001 - ) t.Log("deploying a minimal gRPC container deployment to test Ingress routes") container := generators.NewContainer("grpcbin", test.GRPCBinImage, 0) // Overwrite ports to specify gRPC over HTTP (9000) and gRPC over HTTPS (9001). - container.Ports = []corev1.ContainerPort{{ContainerPort: gRPCBinPort, Name: string(gRPC)}, {ContainerPort: gRPCSBinPort, Name: string(gRPCS)}} + container.Ports = []corev1.ContainerPort{{ContainerPort: test.GRPCBinPort, Name: string(gRPC)}, {ContainerPort: test.GRPCSBinPort, Name: string(gRPCS)}} deployment := generators.NewDeploymentForContainer(container) deployment, err = cluster.Client().AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{}) assert.NoError(t, err) cleaner.Add(deployment) exposeWithService := func(p kongProtocolAnnotation) *corev1.Service { - grpcBinPort := gRPCBinPort + grpcBinPort := test.GRPCBinPort if p == gRPCS { - grpcBinPort = gRPCSBinPort + grpcBinPort = test.GRPCSBinPort } kongProtocol := string(p) t.Logf("exposing deployment gRPC (%s) port %s via service", kongProtocol, deployment.Name) @@ -122,7 +118,7 @@ func TestIngressGRPC(t *testing.T) { Service: &netv1.IngressServiceBackend{ Name: serviceGRPCS.Name, Port: netv1.ServiceBackendPort{ - Number: gRPCSBinPort, + Number: test.GRPCSBinPort, }, }, }, @@ -142,7 +138,7 @@ func TestIngressGRPC(t *testing.T) { Service: &netv1.IngressServiceBackend{ Name: serviceGRPC.Name, Port: netv1.ServiceBackendPort{ - Number: gRPCBinPort, + Number: test.GRPCBinPort, }, }, }, diff --git a/test/internal/helpers/gatewayapi.go b/test/internal/helpers/gatewayapi.go index e0d2780758..8ea6169cf8 100644 --- a/test/internal/helpers/gatewayapi.go +++ b/test/internal/helpers/gatewayapi.go @@ -116,7 +116,7 @@ func gatewayLinkStatusMatches( switch protocolType { case gatewayapi.HTTPProtocolType: route, err := c.GatewayV1().HTTPRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) - groute, gerr := c.GatewayV1alpha2().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) + groute, gerr := c.GatewayV1().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil && gerr != nil { t.Logf("error getting http route: %v", err) t.Logf("error getting grpc route: %v", gerr) @@ -211,7 +211,7 @@ func verifyProgrammedConditionStatus(t *testing.T, switch protocolType { case gatewayapi.HTTPProtocolType: route, err := c.GatewayV1().HTTPRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) - groute, gerr := c.GatewayV1alpha2().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) + groute, gerr := c.GatewayV1().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil && gerr != nil { t.Logf("error getting http route: %v", err) t.Logf("error getting grpc route: %v", err)