diff --git a/cmd/glbc/main.go b/cmd/glbc/main.go index 82ab194ec5..f2dfae301f 100644 --- a/cmd/glbc/main.go +++ b/cmd/glbc/main.go @@ -263,6 +263,7 @@ func main() { EnableIngressRegionalExternal: flags.F.EnableIngressRegionalExternal, EnableWeightedL4ILB: flags.F.EnableWeightedL4ILB, EnableWeightedL4NetLB: flags.F.EnableWeightedL4NetLB, + EnableZonalAffinity: flags.F.EnableZonalAffinity, DisableL4LBFirewall: flags.F.DisableL4LBFirewall, } ctx := ingctx.NewControllerContext(kubeClient, backendConfigClient, frontendConfigClient, firewallCRClient, svcNegClient, ingParamsClient, svcAttachmentClient, networkClient, nodeTopologyClient, eventRecorderKubeClient, cloud, namer, kubeSystemUID, ctxConfig, rootLogger) diff --git a/pkg/annotations/service.go b/pkg/annotations/service.go index d80a758c59..180ac62623 100644 --- a/pkg/annotations/service.go +++ b/pkg/annotations/service.go @@ -20,6 +20,7 @@ import ( "encoding/json" "errors" "fmt" + "strconv" "strings" v1 "k8s.io/api/core/v1" @@ -135,6 +136,9 @@ const ( WeightedL4AnnotationKey = "networking.gke.io/weighted-load-balancing" // Service annotation value for using pods-per-node Weighted load balancing in both ILB and NetlB WeightedL4AnnotationPodsPerNode = "pods-per-node" + + // Service annotation key for using the Zonal Affinity feature with ILB + ZonalAffinitySpilloverRatioKey = "networking.gke.io/zonal-affinity-spillover-ratio" ) // NegAnnotation is the format of the annotation associated with the @@ -304,6 +308,19 @@ func HasWeightedLBPodsPerNodeAnnotation(service *v1.Service) bool { return false } +// HasValidZonalAffinitySpilloverAnnotation checks if the given service has valid zonal affinity spillover ratio annotation +func HasValidZonalAffinitySpilloverAnnotation(service *v1.Service) (bool, float64) { + if service == nil { + return false, 0 + } + if val, ok := service.Annotations[ZonalAffinitySpilloverRatioKey]; ok { + if ratio, err := strconv.ParseFloat(val, 64); err == nil && ratio >= 0 && ratio <= 1 { + return true, ratio + } + } + return false, 0 +} + // OnlyStatusAnnotationsChanged returns true if the only annotation change between the 2 services is the NEG or ILB // resources annotations. // Note : This assumes that the annotations in old and new service are different. If they are identical, this will diff --git a/pkg/backends/backends.go b/pkg/backends/backends.go index c8b1470a84..779e5b33cb 100644 --- a/pkg/backends/backends.go +++ b/pkg/backends/backends.go @@ -35,6 +35,7 @@ const ( DefaultConnectionDrainingTimeoutSeconds = 30 defaultTrackingMode = "PER_CONNECTION" PerSessionTrackingMode = "PER_SESSION" // the only one supported with strong session affinity + DefaultZonalAffinitySpillover = "ZONAL_AFFINITY_SPILL_CROSS_ZONE" ) // LocalityLBPolicyType is the type of locality lb policy the backend service should use. @@ -84,15 +85,18 @@ func NewPoolWithConnectionTrackingPolicy(cloud *gce.Cloud, namer namer.BackendNa // L4BackendServiceParams encapsulates parameters for ensuring an L4 BackendService. type L4BackendServiceParams struct { - Name string - HealthCheckLink string - Protocol string - SessionAffinity string - Scheme string - NamespacedName types.NamespacedName - NetworkInfo *network.NetworkInfo - ConnectionTrackingPolicy *composite.BackendServiceConnectionTrackingPolicy - LocalityLbPolicy LocalityLBPolicyType + Name string + Version meta.Version + HealthCheckLink string + Protocol string + SessionAffinity string + Scheme string + NamespacedName types.NamespacedName + NetworkInfo *network.NetworkInfo + ConnectionTrackingPolicy *composite.BackendServiceConnectionTrackingPolicy + LocalityLbPolicy LocalityLBPolicyType + EnableZonalAffinity bool + ZonalAffinitySpilloverRatio float64 } // ensureDescription updates the BackendService Description with the expected value @@ -355,6 +359,7 @@ func (b *Backends) EnsureL4BackendService(params L4BackendServiceParams, beLogge expectedBS := &composite.BackendService{ Name: params.Name, + Version: params.Version, Protocol: params.Protocol, Description: desc, HealthChecks: []string{params.HealthCheckLink}, @@ -363,6 +368,16 @@ func (b *Backends) EnsureL4BackendService(params L4BackendServiceParams, beLogge LocalityLbPolicy: string(params.LocalityLbPolicy), } + if params.EnableZonalAffinity && params.ZonalAffinitySpilloverRatio >= 0 && params.ZonalAffinitySpilloverRatio <= 1 { + beLogger.V(2).Info("EnsureL4BackendService: using Zonal Affinity with spillover ratio", "ratio", params.ZonalAffinitySpilloverRatio) + expectedBS.NetworkPassThroughLbTrafficPolicy = &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: DefaultZonalAffinitySpillover, + SpilloverRatio: params.ZonalAffinitySpilloverRatio, + }, + } + } + // We need this configuration only for Strong Session Affinity feature if b.useConnectionTrackingPolicy { beLogger.V(2).Info(fmt.Sprintf("EnsureL4BackendService: using connection tracking policy: %+v", params.ConnectionTrackingPolicy)) @@ -449,6 +464,13 @@ func backendSvcEqual(newBS, oldBS *composite.BackendService, compareConnectionTr (newBS.LocalityLbPolicy == string(LocalityLBPolicyDefault) && oldBS.LocalityLbPolicy == string(LocalityLBPolicyMaglev)) || (newBS.LocalityLbPolicy == string(LocalityLBPolicyMaglev) && oldBS.LocalityLbPolicy == string(LocalityLBPolicyDefault))) + // If zonal affinity is set, needs to be equal + svcsEqual = svcsEqual && + (newBS.NetworkPassThroughLbTrafficPolicy == nil) == (oldBS.NetworkPassThroughLbTrafficPolicy == nil) && + (newBS.NetworkPassThroughLbTrafficPolicy == nil || + (newBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity == nil) == (oldBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity == nil) && + (newBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity == nil || (newBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.Spillover == oldBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.Spillover && + newBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.SpilloverRatio == oldBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.SpilloverRatio))) return svcsEqual } diff --git a/pkg/backends/backends_test.go b/pkg/backends/backends_test.go index aa6dfea5fb..89a0548d6c 100644 --- a/pkg/backends/backends_test.go +++ b/pkg/backends/backends_test.go @@ -626,6 +626,72 @@ func TestBackendSvcEqual(t *testing.T) { }, wantEqual: false, }, + { + desc: "Test existing backend service diff with zonal affinity feature enabled", + oldBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.7, + }, + }, + }, + newBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.7, + }, + }, + }, + wantEqual: true, + }, + { + desc: "Test existing backend service diff with zonal affinity feature enabled but different ratio", + oldBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.7, + }, + }, + }, + newBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.3, + }, + }, + }, + wantEqual: false, + }, + { + desc: "Test existing backend service diff enabling zonal affinity feature", + oldBackendService: &composite.BackendService{}, + newBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.3, + }, + }, + }, + wantEqual: false, + }, + { + desc: "Test existing backend service diff enabling zonal affinity feature", + oldBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.3, + }, + }, + }, + newBackendService: &composite.BackendService{}, + wantEqual: false, + }, } { tc := tc t.Run(tc.desc, func(t *testing.T) { diff --git a/pkg/context/context.go b/pkg/context/context.go index 0604da9281..e984f80983 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -148,6 +148,7 @@ type ControllerContextConfig struct { EnableIngressRegionalExternal bool EnableWeightedL4ILB bool EnableWeightedL4NetLB bool + EnableZonalAffinity bool DisableL4LBFirewall bool } diff --git a/pkg/flags/flags.go b/pkg/flags/flags.go index 38fbc02d94..86ee721f43 100644 --- a/pkg/flags/flags.go +++ b/pkg/flags/flags.go @@ -136,6 +136,7 @@ var ( EnableMultiSubnetClusterPhase1 bool EnableWeightedL4ILB bool EnableWeightedL4NetLB bool + EnableZonalAffinity bool EnableDiscretePortForwarding bool EnableMultiProjectMode bool }{ @@ -320,6 +321,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5 flag.BoolVar(&F.EnableMultiSubnetClusterPhase1, "enable-multi-subnet-cluster-phase1", false, "Enable Phase 1 Multi Subnet support for all controllers that are running.") flag.BoolVar(&F.EnableWeightedL4ILB, "enable-weighted-l4-ilb", false, "Enable Weighted Load balancing for L4 ILB.") flag.BoolVar(&F.EnableWeightedL4NetLB, "enable-weighted-l4-netlb", false, "EnableWeighted Load balancing for L4 NetLB .") + flag.BoolVar(&F.EnableZonalAffinity, "enable-zonal-affinity", false, "Enable Zonal Affinity for L4 ILB.") flag.Float32Var(&F.KubeClientQPS, "kube-client-qps", 0.0, "The QPS that the controllers' kube client should adhere to through client side throttling. If zero, client will be created with default settings.") flag.IntVar(&F.KubeClientBurst, "kube-client-burst", 0, "The burst QPS that the controllers' kube client should adhere to through client side throttling. If zero, client will be created with default settings.") flag.BoolVar(&F.EnableDiscretePortForwarding, "enable-discrete-port-forwarding", false, "Enable forwarding of individual ports instead of port ranges.") diff --git a/pkg/l4lb/l4controller.go b/pkg/l4lb/l4controller.go index 13557250a0..0aaff96be7 100644 --- a/pkg/l4lb/l4controller.go +++ b/pkg/l4lb/l4controller.go @@ -287,6 +287,7 @@ func (l4c *L4Controller) processServiceCreateOrUpdate(service *v1.Service, svcLo DualStackEnabled: l4c.enableDualStack, NetworkResolver: l4c.networkResolver, EnableWeightedLB: l4c.ctx.EnableWeightedL4ILB, + EnableZonalAffinity: l4c.ctx.EnableZonalAffinity, DisableNodesFirewallProvisioning: l4c.ctx.DisableL4LBFirewall, } l4 := loadbalancers.NewL4Handler(l4ilbParams, svcLogger) @@ -368,6 +369,7 @@ func (l4c *L4Controller) processServiceDeletion(key string, svc *v1.Service, svc DualStackEnabled: l4c.enableDualStack, NetworkResolver: l4c.networkResolver, EnableWeightedLB: l4c.ctx.EnableWeightedL4ILB, + EnableZonalAffinity: l4c.ctx.EnableZonalAffinity, DisableNodesFirewallProvisioning: l4c.ctx.DisableL4LBFirewall, } l4 := loadbalancers.NewL4Handler(l4ilbParams, svcLogger) diff --git a/pkg/loadbalancers/l4.go b/pkg/loadbalancers/l4.go index 22b7436c2e..f78964f75f 100644 --- a/pkg/loadbalancers/l4.go +++ b/pkg/loadbalancers/l4.go @@ -70,6 +70,7 @@ type L4 struct { network network.NetworkInfo networkResolver network.Resolver enableWeightedLB bool + enableZonalAffinity bool disableNodesFirewallProvisioning bool svcLogger klog.Logger } @@ -107,6 +108,7 @@ type L4ILBParams struct { DualStackEnabled bool NetworkResolver network.Resolver EnableWeightedLB bool + EnableZonalAffinity bool DisableNodesFirewallProvisioning bool } @@ -126,6 +128,7 @@ func NewL4Handler(params *L4ILBParams, logger klog.Logger) *L4 { enableDualStack: params.DualStackEnabled, networkResolver: params.NetworkResolver, enableWeightedLB: params.EnableWeightedLB, + enableZonalAffinity: params.EnableZonalAffinity, disableNodesFirewallProvisioning: params.DisableNodesFirewallProvisioning, svcLogger: logger, } @@ -485,17 +488,27 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service localityLbPolicy := l4.determineBackendServiceLocalityPolicy() + svcEnableZonalAffinity, zonalAffinitySpilloverRatio := annotations.HasValidZonalAffinitySpilloverAnnotation(svc) + finalEnableZonalAffinity := l4.enableZonalAffinity && svcEnableZonalAffinity + + version := meta.VersionGA + if finalEnableZonalAffinity { + version = meta.VersionAlpha + } // ensure backend service backendParams := backends.L4BackendServiceParams{ - Name: bsName, - HealthCheckLink: hcLink, - Protocol: string(protocol), - SessionAffinity: string(l4.Service.Spec.SessionAffinity), - Scheme: string(cloud.SchemeInternal), - NamespacedName: l4.NamespacedName, - NetworkInfo: &l4.network, - ConnectionTrackingPolicy: noConnectionTrackingPolicy, - LocalityLbPolicy: localityLbPolicy, + Name: bsName, + Version: version, + HealthCheckLink: hcLink, + Protocol: string(protocol), + SessionAffinity: string(l4.Service.Spec.SessionAffinity), + Scheme: string(cloud.SchemeInternal), + NamespacedName: l4.NamespacedName, + NetworkInfo: &l4.network, + ConnectionTrackingPolicy: noConnectionTrackingPolicy, + LocalityLbPolicy: localityLbPolicy, + EnableZonalAffinity: finalEnableZonalAffinity, + ZonalAffinitySpilloverRatio: zonalAffinitySpilloverRatio, } bs, _, err := l4.backendPool.EnsureL4BackendService(backendParams, l4.svcLogger) if err != nil { diff --git a/pkg/loadbalancers/l4_test.go b/pkg/loadbalancers/l4_test.go index 18c4f41b56..fb03c5ac40 100644 --- a/pkg/loadbalancers/l4_test.go +++ b/pkg/loadbalancers/l4_test.go @@ -2311,6 +2311,109 @@ func TestWeightedILB(t *testing.T) { } +func TestZonalAffinity(t *testing.T) { + t.Parallel() + + tests := []struct { + desc string + addAnnotationForZonalAffinity bool + zonalAffinityFlagEnabled bool + zonalAffinitySpilloverRatioAnnotation string + wantZonalAffinityEnabled bool + wantZonalAffinitySpilloverRatio float64 + }{ + { + desc: "Flag enabled, Service with spillover ratio", + addAnnotationForZonalAffinity: true, + zonalAffinityFlagEnabled: true, + zonalAffinitySpilloverRatioAnnotation: "0.2", + wantZonalAffinityEnabled: true, + wantZonalAffinitySpilloverRatio: 0.2, + }, + { + desc: "Flag enabled, Service without spillover ratio", + addAnnotationForZonalAffinity: false, + zonalAffinityFlagEnabled: true, + zonalAffinitySpilloverRatioAnnotation: "", + wantZonalAffinityEnabled: false, + wantZonalAffinitySpilloverRatio: -1, + }, + { + desc: "Flag DISABLED, Service with spillover ratio", + addAnnotationForZonalAffinity: true, + zonalAffinityFlagEnabled: false, + zonalAffinitySpilloverRatioAnnotation: "0.5", + wantZonalAffinityEnabled: false, + wantZonalAffinitySpilloverRatio: 0.5, + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + // t.Parallel() + + svc := test.NewL4ILBService(false, 8080) + + if tc.addAnnotationForZonalAffinity { + svc.Annotations[annotations.ZonalAffinitySpilloverRatioKey] = tc.zonalAffinitySpilloverRatioAnnotation + } + nodeNames := []string{"test-node-1"} + vals := gce.DefaultTestClusterValues() + fakeGCE := getFakeGCECloud(vals) + + namer := namer_util.NewL4Namer(kubeSystemUID, nil) + + networkInfo := network.DefaultNetwork(fakeGCE) + + l4ilbParams := &L4ILBParams{ + Service: svc, + Cloud: fakeGCE, + Namer: namer, + Recorder: record.NewFakeRecorder(100), + NetworkResolver: network.NewFakeResolver(networkInfo), + EnableZonalAffinity: tc.zonalAffinityFlagEnabled, + } + l4 := NewL4Handler(l4ilbParams, klog.TODO()) + l4.healthChecks = healthchecksl4.Fake(fakeGCE, l4ilbParams.Recorder) + + if _, err := test.CreateAndInsertNodes(l4.cloud, nodeNames, vals.ZoneName); err != nil { + t.Errorf("Unexpected error when adding nodes %v", err) + } + + result := l4.EnsureInternalLoadBalancer(nodeNames, svc) + if result.Error != nil { + t.Fatalf("Failed to ensure internal loadBalancer, err %v", result.Error) + } + backendServiceName := l4.namer.L4Backend(l4.Service.Namespace, l4.Service.Name) + key := meta.RegionalKey(backendServiceName, l4.cloud.Region()) + bs, err := composite.GetBackendService(l4.cloud, key, meta.VersionAlpha, klog.TODO()) + if err != nil { + t.Fatalf("failed to read BackendService, %v", err) + } + + if bs.NetworkPassThroughLbTrafficPolicy != nil && (!tc.wantZonalAffinityEnabled || !tc.addAnnotationForZonalAffinity) { + t.Fatalf("Unexpected BackendService ZonalAffinity, got value, wanted nil") + } + + if tc.wantZonalAffinityEnabled && tc.addAnnotationForZonalAffinity { + if bs.NetworkPassThroughLbTrafficPolicy == nil || bs.NetworkPassThroughLbTrafficPolicy.ZonalAffinity == nil { + t.Fatalf("Unexpected BackendService ZonalAffinity, got nil") + } + + if bs.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.Spillover != backends.DefaultZonalAffinitySpillover { + t.Errorf("Unexpected BackendService ZonalAffinity Spillover: got %q, want %q", bs.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.Spillover, backends.DefaultZonalAffinitySpillover) + } + + if bs.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.SpilloverRatio != tc.wantZonalAffinitySpilloverRatio { + t.Errorf("Unexpected BackendService ZonalAffinity SpilloverRatio: got %v, want %v", bs.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.SpilloverRatio, tc.wantZonalAffinitySpilloverRatio) + } + } + }) + } + +} + func TestDisableILBIngressFirewall(t *testing.T) { t.Parallel() fakeGCE := getFakeGCECloud(gce.DefaultTestClusterValues())