From 502847ca04c98a055d2a84f9da1de13381c95ef7 Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Sat, 18 May 2024 06:36:55 +0000 Subject: [PATCH] set grpc as default protocol Signed-off-by: Jintao Zhang --- examples/gateway-grpcroute-via-http.yaml | 2 +- examples/gateway-grpcroute-via-https.yaml | 2 +- .../subtranslator/grpcroute_test.go | 18 +++--- .../translator/translate_grpcroute.go | 19 ++++-- test/conformance/gateway_conformance_test.go | 1 + test/consts.go | 3 +- test/integration/isolated/grpc_test.go | 58 +++++-------------- test/internal/helpers/gatewayapi.go | 4 +- 8 files changed, 42 insertions(+), 65 deletions(-) diff --git a/examples/gateway-grpcroute-via-http.yaml b/examples/gateway-grpcroute-via-http.yaml index bc7ebe3839..50b61c7999 100644 --- a/examples/gateway-grpcroute-via-http.yaml +++ b/examples/gateway-grpcroute-via-http.yaml @@ -58,7 +58,7 @@ spec: protocol: HTTP port: 80 --- -apiVersion: gateway.networking.k8s.io/v1alpha2 +apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: name: grpcbin-via-http diff --git a/examples/gateway-grpcroute-via-https.yaml b/examples/gateway-grpcroute-via-https.yaml index 3317bfc837..12e0871ec3 100644 --- a/examples/gateway-grpcroute-via-https.yaml +++ b/examples/gateway-grpcroute-via-https.yaml @@ -70,7 +70,7 @@ spec: certificateRefs: - name: grpcroute-example --- -apiVersion: gateway.networking.k8s.io/v1alpha2 +apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: name: grpcbin-via-https diff --git a/internal/dataplane/translator/subtranslator/grpcroute_test.go b/internal/dataplane/translator/subtranslator/grpcroute_test.go index ada1c15518..246aef4441 100644 --- a/internal/dataplane/translator/subtranslator/grpcroute_test.go +++ b/internal/dataplane/translator/subtranslator/grpcroute_test.go @@ -16,13 +16,13 @@ import ( var grpcRouteGVK = schema.GroupVersionKind{ Group: "gateway.networking.k8s.io", - Version: "v1alpha2", + Version: "v1", Kind: "GRPCRoute", } var grpcRouteTypeMeta = metav1.TypeMeta{ Kind: "GRPCRoute", - APIVersion: "gateway.networking.k8s.io/v1alpha2", + APIVersion: "gateway.networking.k8s.io/v1", } func makeTestGRPCRoute( @@ -33,7 +33,7 @@ func makeTestGRPCRoute( return &gatewayapi.GRPCRoute{ TypeMeta: metav1.TypeMeta{ Kind: "GRPCRoute", - APIVersion: "gateway.networking.k8s.io/v1alpha2", + APIVersion: "gateway.networking.k8s.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -98,7 +98,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -139,7 +139,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -194,7 +194,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -214,7 +214,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -243,7 +243,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -270,7 +270,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), Paths: kong.StringSlice("/"), }, diff --git a/internal/dataplane/translator/translate_grpcroute.go b/internal/dataplane/translator/translate_grpcroute.go index 2487333df1..be99d0d143 100644 --- a/internal/dataplane/translator/translate_grpcroute.go +++ b/internal/dataplane/translator/translate_grpcroute.go @@ -53,10 +53,9 @@ func (t *Translator) ingressRulesFromGRPCRoute(result *ingressRules, grpcroute * // each rule may represent a different set of backend services that will be accepting // traffic, so we make separate routes and Kong services for every present rule. for ruleNumber, rule := range spec.Rules { - // Create a service and attach the routes to it. Protocol for Service can be set via K8s object annotation - // "konghq.com/protocol", by default use "grpcs" to not break existing behavior when annotation is not specified. + // Create a service and attach the routes to it. service, err := generateKongServiceFromBackendRefWithRuleNumber( - t.logger, t.storer, result, grpcroute, ruleNumber, "grpcs", grpcBackendRefsToBackendRefs(rule.BackendRefs)..., + t.logger, t.storer, result, grpcroute, ruleNumber, t.getProtocolForKongService(grpcroute), grpcBackendRefsToBackendRefs(rule.BackendRefs)..., ) if err != nil { return err @@ -116,15 +115,14 @@ func (t *Translator) ingressRulesFromGRPCRouteWithPriority( serviceName := subtranslator.KongServiceNameFromSplitGRPCRouteMatch(match) - // Create a service and attach the routes to it. Protocol for Service can be set via K8s object annotation - // "konghq.com/protocol", by default use "grpcs" to not break existing behavior when annotation is not specified. + // Create a service and attach the routes to it. kongService, _ := generateKongServiceFromBackendRefWithName( t.logger, t.storer, rules, serviceName, grpcRoute, - "grpcs", + t.getProtocolForKongService(grpcRoute), grpcBackendRefsToBackendRefs(grpcRouteRule.BackendRefs)..., ) kongService.Routes = append( @@ -144,3 +142,12 @@ func grpcBackendRefsToBackendRefs(grpcBackendRef []gatewayapi.GRPCBackendRef) [] } return backendRefs } + +// getProtocolForKongService returns the protocol for the Kong service based on the Gateway listening ports +func (t *Translator) getProtocolForKongService(grpcRoute *gatewayapi.GRPCRoute) string { + // When Gateway listens on HTTP use "grpc" protocol for the service. Otherwise for HTTPS use "grpcs". + if len(t.getGatewayListeningPorts(grpcRoute.Namespace, gatewayapi.HTTPProtocolType, grpcRoute.Spec.ParentRefs)) > 0 { + return "grpc" + } + return "grpcs" +} diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index 014bf6b3a6..9567589e79 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -43,6 +43,7 @@ var expressionRoutesSupportedFeatures = []features.SupportedFeature{ // core features features.SupportGateway, features.SupportHTTPRoute, + features.SupportGRPCRoute, // extended features features.SupportHTTPRouteQueryParamMatching, features.SupportHTTPRouteMethodMatching, diff --git a/test/consts.go b/test/consts.go index 7bdb4e05fb..d3a9db6f5d 100644 --- a/test/consts.go +++ b/test/consts.go @@ -24,7 +24,8 @@ const ( // GRPCBinImage is the container image name we use for deploying the "grpcbin" GRPC testing tool. // See: https://github.com/Kong/grpcbin GRPCBinImage = "kong/grpcbin:latest" - GRPCBinPort = 9001 + GRPCBinPort = 9000 + GRPCSBinPort = 9001 // EnvironmentCleanupTimeout is the amount of time that will be given by the test suite to the // testing environment to perform its cleanup when the test suite is shutting down. diff --git a/test/integration/isolated/grpc_test.go b/test/integration/isolated/grpc_test.go index 0681cbd974..a1388ef328 100644 --- a/test/integration/isolated/grpc_test.go +++ b/test/integration/isolated/grpc_test.go @@ -22,7 +22,6 @@ import ( "google.golang.org/grpc/metadata" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8stypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" gatewayclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" @@ -30,7 +29,6 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/util/builder" "github.com/kong/kubernetes-ingress-controller/v3/test" - "github.com/kong/kubernetes-ingress-controller/v3/test/helpers/certificate" "github.com/kong/kubernetes-ingress-controller/v3/test/integration/consts" "github.com/kong/kubernetes-ingress-controller/v3/test/internal/helpers" "github.com/kong/kubernetes-ingress-controller/v3/test/internal/testlabels" @@ -43,9 +41,12 @@ func TestGRPCRouteEssentials(t *testing.T) { New("essentials"). WithLabel(testlabels.NetworkingFamily, testlabels.NetworkingFamilyGatewayAPI). WithLabel(testlabels.Kind, testlabels.KindGRPCRoute). - WithSetup("deploy kong addon into cluster", featureSetup()). - Assess("deploying Gateway and example GRPC service (without konghq.com/protocol annotation) exposed via GRPCRoute over HTTPS", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { - // On purpose omit protocol annotation to test defaulting to "grpcs" that is preserved to not break users' configs. + WithSetup("deploy kong addon into cluster", featureSetup( + withKongProxyEnvVars(map[string]string{ + "PROXY_LISTEN": `0.0.0.0:8000 http2\, 0.0.0.0:8443 http2 ssl`, + }), + )). + Assess("deploying Gateway and example GRPC service (without konghq.com/protocol annotation) exposed via GRPCRoute over HTTP", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { cleaner := GetFromCtxForT[*clusters.Cleaner](ctx, t) cluster := GetClusterFromCtx(ctx) namespace := GetNamespaceForT(ctx, t) @@ -61,45 +62,12 @@ func TestGRPCRouteEssentials(t *testing.T) { assert.NoError(t, err) cleaner.Add(gwc) - t.Log("configuring secret") - const tlsRouteHostname = "tls-route.example" - tlsRouteExampleTLSCert, tlsRouteExampleTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithCommonName(tlsRouteHostname)) - const tlsSecretName = "secret-test" - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - UID: k8stypes.UID("7428fb98-180b-4702-a91f-61351a33c6e8"), - Name: tlsSecretName, - Namespace: namespace, - }, - Data: map[string][]byte{ - "tls.crt": tlsRouteExampleTLSCert, - "tls.key": tlsRouteExampleTLSKey, - }, - } - - t.Log("deploying secret") - secret, err = cluster.Client().CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) - cleaner.Add(secret) - t.Log("deploying a new gateway") gateway, err := helpers.DeployGateway(ctx, gatewayClient, namespace, gatewayClassName, func(gw *gatewayapi.Gateway) { - // Besides default HTTP listener, add a HTTPS listener. - gw.Spec.Listeners = append( - gw.Spec.Listeners, - builder.NewListener("https"). - HTTPS(). - WithPort(ktfkong.DefaultProxyTLSServicePort). - WithHostname(testHostname). - WithTLSConfig(&gatewayapi.GatewayTLSConfig{ - CertificateRefs: []gatewayapi.SecretObjectReference{ - { - Name: gatewayapi.ObjectName(secret.Name), - }, - }, - }). - Build(), - ) + gw.Spec.Listeners = builder.NewListener("grpc"). + HTTP(). + WithPort(ktfkong.DefaultProxyHTTPPort). + IntoSlice() }) assert.NoError(t, err) cleaner.Add(gateway) @@ -168,7 +136,7 @@ func TestGRPCRouteEssentials(t *testing.T) { return ctx }). Assess("checking if GRPCRoute is linked correctly and client can connect properly to the exposed service", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { - grpcAddr := GetHTTPSURLFromCtx(ctx).Host // For GRPC, we use the same address as for HTTPS, but without the scheme (https://). + grpcAddr := GetHTTPURLFromCtx(ctx).Host // For GRPC, we use the same address as for HTTP, but without the scheme (http://). namespace := GetNamespaceForT(ctx, t) gatewayClient := GetFromCtxForT[*gatewayclient.Clientset](ctx, t) grpcRoute := GetFromCtxForT[*gatewayapi.GRPCRoute](ctx, t) @@ -184,14 +152,14 @@ func TestGRPCRouteEssentials(t *testing.T) { t.Log("waiting for routes from GRPCRoute to become operational") assert.Eventually(t, func() bool { - err := grpcEchoResponds(ctx, grpcAddr, testHostname, "kong", true) + err := grpcEchoResponds(ctx, grpcAddr, testHostname, "kong", false) if err != nil { t.Log(err) } return err == nil }, consts.IngressWait, consts.WaitTick) - client, closeGrpcConn, err := grpcBinClient(grpcAddr, testHostname, true) + client, closeGrpcConn, err := grpcBinClient(grpcAddr, testHostname, false) assert.NoError(t, err) t.Cleanup(func() { err := closeGrpcConn() diff --git a/test/internal/helpers/gatewayapi.go b/test/internal/helpers/gatewayapi.go index cfa42fea88..78bc86883f 100644 --- a/test/internal/helpers/gatewayapi.go +++ b/test/internal/helpers/gatewayapi.go @@ -116,7 +116,7 @@ func gatewayLinkStatusMatches( switch protocolType { case gatewayapi.HTTPProtocolType: route, err := c.GatewayV1().HTTPRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) - groute, gerr := c.GatewayV1alpha2().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) + groute, gerr := c.GatewayV1().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil && gerr != nil { t.Logf("error getting http route: %v", err) t.Logf("error getting grpc route: %v", err) @@ -211,7 +211,7 @@ func verifyProgrammedConditionStatus(t *testing.T, switch protocolType { case gatewayapi.HTTPProtocolType: route, err := c.GatewayV1().HTTPRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) - groute, gerr := c.GatewayV1alpha2().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) + groute, gerr := c.GatewayV1().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil && gerr != nil { t.Logf("error getting http route: %v", err) t.Logf("error getting grpc route: %v", err)