Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Zonal Affinity for ILB #2690

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Zonal Affinity for ILB #2690

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
421dfa1
zonal affinity
08volt Sep 26, 2024
a2932fb
fix
08volt Oct 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/glbc/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,7 @@ func main() {
EnableIngressRegionalExternal: flags.F.EnableIngressRegionalExternal,
EnableWeightedL4ILB: flags.F.EnableWeightedL4ILB,
EnableWeightedL4NetLB: flags.F.EnableWeightedL4NetLB,
EnableZonalAffinity: flags.F.EnableZonalAffinity,
DisableL4LBFirewall: flags.F.DisableL4LBFirewall,
}
ctx := ingctx.NewControllerContext(kubeClient, backendConfigClient, frontendConfigClient, firewallCRClient, svcNegClient, ingParamsClient, svcAttachmentClient, networkClient, nodeTopologyClient, eventRecorderKubeClient, cloud, namer, kubeSystemUID, ctxConfig, rootLogger)
Expand Down
17 changes: 17 additions & 0 deletions pkg/annotations/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"

v1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -135,6 +136,9 @@ const (
WeightedL4AnnotationKey = "networking.gke.io/weighted-load-balancing"
// Service annotation value for using pods-per-node Weighted load balancing in both ILB and NetlB
WeightedL4AnnotationPodsPerNode = "pods-per-node"

// Service annotation key for using the Zonal Affinity feature with ILB
ZonalAffinitySpilloverRatioKey = "networking.gke.io/zonal-affinity-spillover-ratio"
)

// NegAnnotation is the format of the annotation associated with the
Expand Down Expand Up @@ -304,6 +308,19 @@ func HasWeightedLBPodsPerNodeAnnotation(service *v1.Service) bool {
return false
}

// HasValidZonalAffinitySpilloverAnnotation checks if the given service has valid zonal affinity spillover ratio annotation
func HasValidZonalAffinitySpilloverAnnotation(service *v1.Service) (bool, float64) {
if service == nil {
return false, 0
}
if val, ok := service.Annotations[ZonalAffinitySpilloverRatioKey]; ok {
if ratio, err := strconv.ParseFloat(val, 64); err == nil && ratio >= 0 && ratio <= 1 {
return true, ratio
}
}
return false, 0
}

// OnlyStatusAnnotationsChanged returns true if the only annotation change between the 2 services is the NEG or ILB
// resources annotations.
// Note : This assumes that the annotations in old and new service are different. If they are identical, this will
Expand Down
40 changes: 31 additions & 9 deletions pkg/backends/backends.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ const (
DefaultConnectionDrainingTimeoutSeconds = 30
defaultTrackingMode = "PER_CONNECTION"
PerSessionTrackingMode = "PER_SESSION" // the only one supported with strong session affinity
DefaultZonalAffinitySpillover = "ZONAL_AFFINITY_SPILL_CROSS_ZONE"
)

// LocalityLBPolicyType is the type of locality lb policy the backend service should use.
Expand Down Expand Up @@ -84,15 +85,18 @@ func NewPoolWithConnectionTrackingPolicy(cloud *gce.Cloud, namer namer.BackendNa

// L4BackendServiceParams encapsulates parameters for ensuring an L4 BackendService.
type L4BackendServiceParams struct {
Name string
HealthCheckLink string
Protocol string
SessionAffinity string
Scheme string
NamespacedName types.NamespacedName
NetworkInfo *network.NetworkInfo
ConnectionTrackingPolicy *composite.BackendServiceConnectionTrackingPolicy
LocalityLbPolicy LocalityLBPolicyType
Name string
Version meta.Version
HealthCheckLink string
Protocol string
SessionAffinity string
Scheme string
NamespacedName types.NamespacedName
NetworkInfo *network.NetworkInfo
ConnectionTrackingPolicy *composite.BackendServiceConnectionTrackingPolicy
LocalityLbPolicy LocalityLBPolicyType
EnableZonalAffinity bool
ZonalAffinitySpilloverRatio float64
}

// ensureDescription updates the BackendService Description with the expected value
Expand Down Expand Up @@ -355,6 +359,7 @@ func (b *Backends) EnsureL4BackendService(params L4BackendServiceParams, beLogge

expectedBS := &composite.BackendService{
Name: params.Name,
Version: params.Version,
Protocol: params.Protocol,
Description: desc,
HealthChecks: []string{params.HealthCheckLink},
Expand All @@ -363,6 +368,16 @@ func (b *Backends) EnsureL4BackendService(params L4BackendServiceParams, beLogge
LocalityLbPolicy: string(params.LocalityLbPolicy),
}

if params.EnableZonalAffinity && params.ZonalAffinitySpilloverRatio >= 0 && params.ZonalAffinitySpilloverRatio <= 1 {
beLogger.V(2).Info("EnsureL4BackendService: using Zonal Affinity with spillover ratio", "ratio", params.ZonalAffinitySpilloverRatio)
expectedBS.NetworkPassThroughLbTrafficPolicy = &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
Spillover: DefaultZonalAffinitySpillover,
SpilloverRatio: params.ZonalAffinitySpilloverRatio,
},
}
}

// We need this configuration only for Strong Session Affinity feature
if b.useConnectionTrackingPolicy {
beLogger.V(2).Info(fmt.Sprintf("EnsureL4BackendService: using connection tracking policy: %+v", params.ConnectionTrackingPolicy))
Expand Down Expand Up @@ -449,6 +464,13 @@ func backendSvcEqual(newBS, oldBS *composite.BackendService, compareConnectionTr
(newBS.LocalityLbPolicy == string(LocalityLBPolicyDefault) && oldBS.LocalityLbPolicy == string(LocalityLBPolicyMaglev)) ||
(newBS.LocalityLbPolicy == string(LocalityLBPolicyMaglev) && oldBS.LocalityLbPolicy == string(LocalityLBPolicyDefault)))

// If zonal affinity is set, needs to be equal
svcsEqual = svcsEqual &&
(newBS.NetworkPassThroughLbTrafficPolicy == nil) == (oldBS.NetworkPassThroughLbTrafficPolicy == nil) &&
(newBS.NetworkPassThroughLbTrafficPolicy == nil ||
(newBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity == nil) == (oldBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity == nil) &&
(newBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity == nil || (newBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.Spillover == oldBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.Spillover &&
newBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.SpilloverRatio == oldBS.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.SpilloverRatio)))
return svcsEqual
}

Expand Down
66 changes: 66 additions & 0 deletions pkg/backends/backends_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,72 @@ func TestBackendSvcEqual(t *testing.T) {
},
wantEqual: false,
},
{
desc: "Test existing backend service diff with zonal affinity feature enabled",
oldBackendService: &composite.BackendService{
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
SpilloverRatio: 0.7,
},
},
},
newBackendService: &composite.BackendService{
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
SpilloverRatio: 0.7,
},
},
},
wantEqual: true,
},
{
desc: "Test existing backend service diff with zonal affinity feature enabled but different ratio",
oldBackendService: &composite.BackendService{
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
SpilloverRatio: 0.7,
},
},
},
newBackendService: &composite.BackendService{
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
SpilloverRatio: 0.3,
},
},
},
wantEqual: false,
},
{
desc: "Test existing backend service diff enabling zonal affinity feature",
oldBackendService: &composite.BackendService{},
newBackendService: &composite.BackendService{
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
SpilloverRatio: 0.3,
},
},
},
wantEqual: false,
},
{
desc: "Test existing backend service diff enabling zonal affinity feature",
oldBackendService: &composite.BackendService{
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
SpilloverRatio: 0.3,
},
},
},
newBackendService: &composite.BackendService{},
wantEqual: false,
},
} {
tc := tc
t.Run(tc.desc, func(t *testing.T) {
Expand Down
1 change: 1 addition & 0 deletions pkg/context/context.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ type ControllerContextConfig struct {
EnableIngressRegionalExternal bool
EnableWeightedL4ILB bool
EnableWeightedL4NetLB bool
EnableZonalAffinity bool
DisableL4LBFirewall bool
}

Expand Down
2 changes: 2 additions & 0 deletions pkg/flags/flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ var (
EnableMultiSubnetClusterPhase1 bool
EnableWeightedL4ILB bool
EnableWeightedL4NetLB bool
EnableZonalAffinity bool
EnableDiscretePortForwarding bool
EnableMultiProjectMode bool
}{
Expand Down Expand Up @@ -320,6 +321,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5
flag.BoolVar(&F.EnableMultiSubnetClusterPhase1, "enable-multi-subnet-cluster-phase1", false, "Enable Phase 1 Multi Subnet support for all controllers that are running.")
flag.BoolVar(&F.EnableWeightedL4ILB, "enable-weighted-l4-ilb", false, "Enable Weighted Load balancing for L4 ILB.")
flag.BoolVar(&F.EnableWeightedL4NetLB, "enable-weighted-l4-netlb", false, "EnableWeighted Load balancing for L4 NetLB .")
flag.BoolVar(&F.EnableZonalAffinity, "enable-zonal-affinity", false, "Enable Zonal Affinity for L4 ILB.")
flag.Float32Var(&F.KubeClientQPS, "kube-client-qps", 0.0, "The QPS that the controllers' kube client should adhere to through client side throttling. If zero, client will be created with default settings.")
flag.IntVar(&F.KubeClientBurst, "kube-client-burst", 0, "The burst QPS that the controllers' kube client should adhere to through client side throttling. If zero, client will be created with default settings.")
flag.BoolVar(&F.EnableDiscretePortForwarding, "enable-discrete-port-forwarding", false, "Enable forwarding of individual ports instead of port ranges.")
Expand Down
2 changes: 2 additions & 0 deletions pkg/l4lb/l4controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,7 @@ func (l4c *L4Controller) processServiceCreateOrUpdate(service *v1.Service, svcLo
DualStackEnabled: l4c.enableDualStack,
NetworkResolver: l4c.networkResolver,
EnableWeightedLB: l4c.ctx.EnableWeightedL4ILB,
EnableZonalAffinity: l4c.ctx.EnableZonalAffinity,
DisableNodesFirewallProvisioning: l4c.ctx.DisableL4LBFirewall,
}
l4 := loadbalancers.NewL4Handler(l4ilbParams, svcLogger)
Expand Down Expand Up @@ -368,6 +369,7 @@ func (l4c *L4Controller) processServiceDeletion(key string, svc *v1.Service, svc
DualStackEnabled: l4c.enableDualStack,
NetworkResolver: l4c.networkResolver,
EnableWeightedLB: l4c.ctx.EnableWeightedL4ILB,
EnableZonalAffinity: l4c.ctx.EnableZonalAffinity,
DisableNodesFirewallProvisioning: l4c.ctx.DisableL4LBFirewall,
}
l4 := loadbalancers.NewL4Handler(l4ilbParams, svcLogger)
Expand Down
31 changes: 22 additions & 9 deletions pkg/loadbalancers/l4.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ type L4 struct {
network network.NetworkInfo
networkResolver network.Resolver
enableWeightedLB bool
enableZonalAffinity bool
disableNodesFirewallProvisioning bool
svcLogger klog.Logger
}
Expand Down Expand Up @@ -107,6 +108,7 @@ type L4ILBParams struct {
DualStackEnabled bool
NetworkResolver network.Resolver
EnableWeightedLB bool
EnableZonalAffinity bool
DisableNodesFirewallProvisioning bool
}

Expand All @@ -126,6 +128,7 @@ func NewL4Handler(params *L4ILBParams, logger klog.Logger) *L4 {
enableDualStack: params.DualStackEnabled,
networkResolver: params.NetworkResolver,
enableWeightedLB: params.EnableWeightedLB,
enableZonalAffinity: params.EnableZonalAffinity,
disableNodesFirewallProvisioning: params.DisableNodesFirewallProvisioning,
svcLogger: logger,
}
Expand Down Expand Up @@ -485,17 +488,27 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service

localityLbPolicy := l4.determineBackendServiceLocalityPolicy()

svcEnableZonalAffinity, zonalAffinitySpilloverRatio := annotations.HasValidZonalAffinitySpilloverAnnotation(svc)
finalEnableZonalAffinity := l4.enableZonalAffinity && svcEnableZonalAffinity

version := meta.VersionGA
if finalEnableZonalAffinity {
version = meta.VersionAlpha
}
// ensure backend service
backendParams := backends.L4BackendServiceParams{
Name: bsName,
HealthCheckLink: hcLink,
Protocol: string(protocol),
SessionAffinity: string(l4.Service.Spec.SessionAffinity),
Scheme: string(cloud.SchemeInternal),
NamespacedName: l4.NamespacedName,
NetworkInfo: &l4.network,
ConnectionTrackingPolicy: noConnectionTrackingPolicy,
LocalityLbPolicy: localityLbPolicy,
Name: bsName,
Version: version,
HealthCheckLink: hcLink,
Protocol: string(protocol),
SessionAffinity: string(l4.Service.Spec.SessionAffinity),
Scheme: string(cloud.SchemeInternal),
NamespacedName: l4.NamespacedName,
NetworkInfo: &l4.network,
ConnectionTrackingPolicy: noConnectionTrackingPolicy,
LocalityLbPolicy: localityLbPolicy,
EnableZonalAffinity: finalEnableZonalAffinity,
ZonalAffinitySpilloverRatio: zonalAffinitySpilloverRatio,
}
bs, _, err := l4.backendPool.EnsureL4BackendService(backendParams, l4.svcLogger)
if err != nil {
Expand Down
Loading