From f1f3d2bcca2cfac14cb4468af8ba790c05f270c8 Mon Sep 17 00:00:00 2001 From: Jon Fearer Date: Mon, 22 Nov 2021 11:56:48 -0700 Subject: [PATCH 1/9] feat(alb-target): begin adding support for alb target type --- .../elbv2/v1beta1/targetgroupbinding_types.go | 4 +- .../elbv2.k8s.aws_targetgroupbindings.yaml | 1 + .../service/eventhandlers/service_events.go | 5 +- pkg/deploy/elbv2/target_group_manager_test.go | 37 ++++ pkg/deploy/elbv2/target_group_synthesizer.go | 5 +- pkg/model/elbv2/target_group.go | 17 +- pkg/service/model_build_target_group.go | 41 ++++ pkg/service/model_builder.go | 6 + pkg/targetgroupbinding/resource_manager.go | 9 +- scripts/lib/eksctl.sh | 2 +- test/e2e/service/nlb_alb_target.go | 81 ++++++++ test/e2e/service/nlb_alb_target_test.go | 187 ++++++++++++++++++ webhooks/elbv2/targetgroupbinding_mutator.go | 2 + 13 files changed, 383 insertions(+), 14 deletions(-) create mode 100644 test/e2e/service/nlb_alb_target.go create mode 100644 test/e2e/service/nlb_alb_target_test.go diff --git a/apis/elbv2/v1beta1/targetgroupbinding_types.go b/apis/elbv2/v1beta1/targetgroupbinding_types.go index d1cf2f8d1..3a0a8287e 100644 --- a/apis/elbv2/v1beta1/targetgroupbinding_types.go +++ b/apis/elbv2/v1beta1/targetgroupbinding_types.go @@ -21,16 +21,18 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -// +kubebuilder:validation:Enum=instance;ip +// +kubebuilder:validation:Enum=instance;ip;alb // TargetType is the targetType of your ELBV2 TargetGroup. // // * with `instance` TargetType, nodes with nodePort for your service will be registered as targets // * with `ip` TargetType, Pods with containerPort for your service will be registered as targets +// * with `alb` TargetType, an application load balancer will be registered as the target type TargetType string const ( TargetTypeInstance TargetType = "instance" TargetTypeIP TargetType = "ip" + TargetTypeALB TargetType = "alb" ) // +kubebuilder:validation:Enum=ipv4;ipv6 diff --git a/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml b/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml index acb61231a..7d5190bf8 100644 --- a/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml +++ b/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml @@ -307,6 +307,7 @@ spec: enum: - instance - ip + - alb type: string required: - serviceRef diff --git a/controllers/service/eventhandlers/service_events.go b/controllers/service/eventhandlers/service_events.go index be53ab9da..91029d807 100644 --- a/controllers/service/eventhandlers/service_events.go +++ b/controllers/service/eventhandlers/service_events.go @@ -58,7 +58,7 @@ func (h *enqueueRequestsForServiceEvent) Generic(e event.GenericEvent, queue wor } func (h *enqueueRequestsForServiceEvent) isServiceSupported(service *corev1.Service) bool { - lbType := "" + var lbType string _ = h.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixLoadBalancerType, &lbType, service.Annotations) if lbType == svcpkg.LoadBalancerTypeNLBIP { return true @@ -66,7 +66,8 @@ func (h *enqueueRequestsForServiceEvent) isServiceSupported(service *corev1.Serv var lbTargetType string _ = h.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixTargetType, &lbTargetType, service.Annotations) if lbType == svcpkg.LoadBalancerTypeExternal && (lbTargetType == svcpkg.LoadBalancerTargetTypeIP || - lbTargetType == svcpkg.LoadBalancerTargetTypeInstance) { + lbTargetType == svcpkg.LoadBalancerTargetTypeInstance || + lbTargetType == svcpkg.LoadBalancerTargetTypeALB) { return true } return false diff --git a/pkg/deploy/elbv2/target_group_manager_test.go b/pkg/deploy/elbv2/target_group_manager_test.go index 09eb779af..a20ce0e4d 100644 --- a/pkg/deploy/elbv2/target_group_manager_test.go +++ b/pkg/deploy/elbv2/target_group_manager_test.go @@ -509,6 +509,43 @@ func Test_buildSDKCreateTargetGroupInput(t *testing.T) { IpAddressType: awssdk.String("ipv6"), }, }, + { + name: "standard case alb target", + args: args{ + tgSpec: elbv2model.TargetGroupSpec{ + Name: "my-tg", + TargetType: elbv2model.TargetTypeALB, + Port: 8080, + Protocol: elbv2model.ProtocolHTTP, + IPAddressType: &ipAddressTypeIPv4, + HealthCheckConfig: &elbv2model.TargetGroupHealthCheckConfig{ + Port: &port9090, + Protocol: &protocolHTTP, + Path: awssdk.String("/healthcheck"), + Matcher: &elbv2model.HealthCheckMatcher{HTTPCode: awssdk.String("200")}, + IntervalSeconds: awssdk.Int64(10), + TimeoutSeconds: awssdk.Int64(5), + HealthyThresholdCount: awssdk.Int64(3), + UnhealthyThresholdCount: awssdk.Int64(2), + }, + }, + }, + want: &elbv2sdk.CreateTargetGroupInput{ + HealthCheckEnabled: awssdk.Bool(true), + HealthCheckIntervalSeconds: awssdk.Int64(10), + HealthCheckPath: awssdk.String("/healthcheck"), + HealthCheckPort: awssdk.String("9090"), + HealthCheckProtocol: awssdk.String("HTTP"), + HealthCheckTimeoutSeconds: awssdk.Int64(5), + HealthyThresholdCount: awssdk.Int64(3), + Matcher: &elbv2sdk.Matcher{HttpCode: awssdk.String("200")}, + UnhealthyThresholdCount: awssdk.Int64(2), + Name: awssdk.String("my-tg"), + Port: awssdk.Int64(8080), + Protocol: awssdk.String("HTTP"), + TargetType: awssdk.String("alb"), + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/deploy/elbv2/target_group_synthesizer.go b/pkg/deploy/elbv2/target_group_synthesizer.go index e508b7953..12ff30d4c 100644 --- a/pkg/deploy/elbv2/target_group_synthesizer.go +++ b/pkg/deploy/elbv2/target_group_synthesizer.go @@ -40,7 +40,10 @@ type targetGroupSynthesizer struct { func (s *targetGroupSynthesizer) Synthesize(ctx context.Context) error { var resTGs []*elbv2model.TargetGroup - s.stack.ListResources(&resTGs) + err := s.stack.ListResources(&resTGs) + if err != nil { + return err + } sdkTGs, err := s.findSDKTargetGroups(ctx) if err != nil { return err diff --git a/pkg/model/elbv2/target_group.go b/pkg/model/elbv2/target_group.go index ddeb02d87..08c9fbd33 100644 --- a/pkg/model/elbv2/target_group.go +++ b/pkg/model/elbv2/target_group.go @@ -9,7 +9,7 @@ import ( var _ core.Resource = &TargetGroup{} -// TargetGroup represents a ELBV2 TargetGroup +// TargetGroup represents a ELBV2 TargetGroup. type TargetGroup struct { core.ResourceMeta `json:"-"` @@ -32,12 +32,12 @@ func NewTargetGroup(stack core.Stack, id string, spec TargetGroupSpec) *TargetGr return tg } -// SetStatus sets the TargetGroup's status +// SetStatus sets the TargetGroup's status. func (tg *TargetGroup) SetStatus(status TargetGroupStatus) { tg.Status = &status } -// LoadBalancerARN returns The Amazon Resource Name (ARN) of the target group. +// TargetGroupARN returns The Amazon Resource Name (ARN) of the target group. func (tg *TargetGroup) TargetGroupARN() core.StringToken { return core.NewResourceFieldStringToken(tg, "status/targetGroupARN", func(ctx context.Context, res core.Resource, fieldPath string) (s string, err error) { @@ -55,6 +55,7 @@ type TargetType string const ( TargetTypeInstance TargetType = "instance" TargetTypeIP TargetType = "ip" + TargetTypeALB TargetType = "alb" ) type TargetGroupIPAddressType string @@ -64,7 +65,7 @@ const ( TargetGroupIPAddressTypeIPv6 TargetGroupIPAddressType = "ipv6" ) -// Information to use when checking for a successful response from a target. +// HealthCheckMatcher contains information to use when checking for a successful response from a target. type HealthCheckMatcher struct { // The HTTP codes. HTTPCode *string `json:"httpCode,omitempty"` @@ -73,7 +74,7 @@ type HealthCheckMatcher struct { GRPCCode *string `json:"grpcCode,omitempty"` } -// Configuration for TargetGroup's HealthCheck. +// TargetGroupHealthCheckConfig contains configuration for TargetGroup's HealthCheck. type TargetGroupHealthCheckConfig struct { // The port the load balancer uses when performing health checks on targets. // +optional @@ -108,7 +109,7 @@ type TargetGroupHealthCheckConfig struct { UnhealthyThresholdCount *int64 `json:"unhealthyThresholdCount,omitempty"` } -// Specifies a target group attribute. +// TargetGroupAttribute specifies a target group attribute. type TargetGroupAttribute struct { // The name of the attribute. Key string `json:"key"` @@ -117,7 +118,7 @@ type TargetGroupAttribute struct { Value string `json:"value"` } -// TargetGroupSpec defines the observed state of TargetGroup +// TargetGroupSpec defines the observed state of TargetGroup. type TargetGroupSpec struct { // The name of the target group. Name string `json:"name"` @@ -152,7 +153,7 @@ type TargetGroupSpec struct { Tags map[string]string `json:"tags,omitempty"` } -// TargetGroupStatus defines the observed state of TargetGroup +// TargetGroupStatus defines the observed state of TargetGroup. type TargetGroupStatus struct { // The Amazon Resource Name (ARN) of the target group. TargetGroupARN string `json:"targetGroupARN"` diff --git a/pkg/service/model_build_target_group.go b/pkg/service/model_build_target_group.go index 4046912a9..b0f8181d3 100644 --- a/pkg/service/model_build_target_group.go +++ b/pkg/service/model_build_target_group.go @@ -95,6 +95,9 @@ func (t *defaultModelBuildTask) buildTargetGroupHealthCheckConfig(ctx context.Co t.service.Spec.Type == corev1.ServiceTypeLoadBalancer { return t.buildTargetGroupHealthCheckConfigForInstanceModeLocal(ctx) } + if targetType == elbv2model.TargetTypeALB { + return t.buildTargetGroupHealthCheckConfigForALBTargetType(ctx) + } return t.buildTargetGroupHealthCheckConfigDefault(ctx) } @@ -168,6 +171,41 @@ func (t *defaultModelBuildTask) buildTargetGroupHealthCheckConfigForInstanceMode }, nil } +func (t *defaultModelBuildTask) buildTargetGroupHealthCheckConfigForALBTargetType(ctx context.Context) (*elbv2model.TargetGroupHealthCheckConfig, error) { + healthCheckProtocol, err := t.buildTargetGroupHealthCheckProtocol(ctx, t.defaultHealthCheckProtocolForALBTargetType) + if err != nil { + return nil, err + } + var healthCheckPathPtr *string + if healthCheckProtocol != elbv2model.ProtocolTCP { + healthCheckPathPtr = t.buildTargetGroupHealthCheckPath(ctx, t.defaultHealthCheckPath) + } + healthCheckPort, err := t.buildTargetGroupHealthCheckPort(ctx, t.defaultHealthCheckPort) + if err != nil { + return nil, err + } + intervalSeconds, err := t.buildTargetGroupHealthCheckIntervalSeconds(ctx, t.defaultHealthCheckInterval) + if err != nil { + return nil, err + } + healthyThresholdCount, err := t.buildTargetGroupHealthCheckHealthyThresholdCount(ctx, t.defaultHealthCheckHealthyThreshold) + if err != nil { + return nil, err + } + unhealthyThresholdCount, err := t.buildTargetGroupHealthCheckUnhealthyThresholdCount(ctx, t.defaultHealthCheckUnhealthyThreshold) + if err != nil { + return nil, err + } + return &elbv2model.TargetGroupHealthCheckConfig{ + Port: &healthCheckPort, + Protocol: &healthCheckProtocol, + Path: healthCheckPathPtr, + IntervalSeconds: &intervalSeconds, + HealthyThresholdCount: &healthyThresholdCount, + UnhealthyThresholdCount: &unhealthyThresholdCount, + }, nil +} + var invalidTargetGroupNamePattern = regexp.MustCompile("[[:^alnum:]]") func (t *defaultModelBuildTask) buildTargetGroupName(_ context.Context, svcPort intstr.IntOrString, tgPort int64, @@ -349,6 +387,9 @@ func (t *defaultModelBuildTask) buildTargetType(_ context.Context) (elbv2model.T } return elbv2model.TargetTypeInstance, nil } + if lbType == LoadBalancerTypeExternal && lbTargetType == LoadBalancerTargetTypeALB { + return elbv2model.TargetTypeALB, nil + } return "", errors.Errorf("unsupported target type \"%v\" for load balancer type \"%v\"", lbTargetType, lbType) } diff --git a/pkg/service/model_builder.go b/pkg/service/model_builder.go index 6b42c308f..ea9d32f96 100644 --- a/pkg/service/model_builder.go +++ b/pkg/service/model_builder.go @@ -23,6 +23,7 @@ const ( LoadBalancerTypeExternal = "external" LoadBalancerTargetTypeIP = "ip" LoadBalancerTargetTypeInstance = "instance" + LoadBalancerTargetTypeALB = "alb" lbAttrsDeletionProtection = "deletion_protection.enabled" ) @@ -102,6 +103,8 @@ func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service defaultHealthCheckTimeoutForInstanceModeLocal: 6, defaultHealthCheckHealthyThresholdForInstanceModeLocal: 2, defaultHealthCheckUnhealthyThresholdForInstanceModeLocal: 2, + + defaultHealthCheckProtocolForALBTargetType: elbv2model.ProtocolHTTP, } if err := task.run(ctx); err != nil { @@ -154,6 +157,9 @@ type defaultModelBuildTask struct { defaultHealthCheckTimeoutForInstanceModeLocal int64 defaultHealthCheckHealthyThresholdForInstanceModeLocal int64 defaultHealthCheckUnhealthyThresholdForInstanceModeLocal int64 + + // Default health check settings for an NLB that targets an ALB + defaultHealthCheckProtocolForALBTargetType elbv2model.Protocol } func (t *defaultModelBuildTask) run(ctx context.Context) error { diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index dcc6b2c08..8e017af79 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -89,7 +89,10 @@ func (m *defaultResourceManager) Reconcile(ctx context.Context, tgb *elbv2api.Ta if *tgb.Spec.TargetType == elbv2api.TargetTypeIP { return m.reconcileWithIPTargetType(ctx, tgb) } - return m.reconcileWithInstanceTargetType(ctx, tgb) + if *tgb.Spec.TargetType == elbv2api.TargetTypeInstance { + return m.reconcileWithInstanceTargetType(ctx, tgb) + } + return m.reconcileWithALBTargetType(ctx, tgb) } func (m *defaultResourceManager) Cleanup(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { @@ -202,6 +205,10 @@ func (m *defaultResourceManager) reconcileWithInstanceTargetType(ctx context.Con return nil } +func (m *defaultResourceManager) reconcileWithALBTargetType(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { + return nil +} + func (m *defaultResourceManager) cleanupTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { targets, err := m.targetsManager.ListTargets(ctx, tgb.Spec.TargetGroupARN) if err != nil { diff --git a/scripts/lib/eksctl.sh b/scripts/lib/eksctl.sh index 6647ad32a..6f5e80092 100644 --- a/scripts/lib/eksctl.sh +++ b/scripts/lib/eksctl.sh @@ -129,7 +129,7 @@ eksctl::delete_cluster() { return 1 fi - echo "ueleted cluster ${cluster_name}" + echo "deleted cluster ${cluster_name}" return 0 } diff --git a/test/e2e/service/nlb_alb_target.go b/test/e2e/service/nlb_alb_target.go new file mode 100644 index 000000000..05f0b3556 --- /dev/null +++ b/test/e2e/service/nlb_alb_target.go @@ -0,0 +1,81 @@ +package service + +import ( + "context" + "crypto/tls" + "fmt" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "net/http" + "sigs.k8s.io/aws-load-balancer-controller/test/framework" + "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" + "time" +) + +type NLBALBTestStack struct { + resourceStack *resourceStack +} + +func (s *NLBALBTestStack) Deploy(ctx context.Context, f *framework.Framework, svc *corev1.Service, dp *appsv1.Deployment) error { + s.resourceStack = NewResourceStack(dp, svc, "service-ip-e2e", false) + return s.resourceStack.Deploy(ctx, f) +} + +func (s *NLBALBTestStack) UpdateServiceAnnotations(ctx context.Context, f *framework.Framework, svcAnnotations map[string]string) error { + return s.resourceStack.UpdateServiceAnnotations(ctx, f, svcAnnotations) +} + +func (s *NLBALBTestStack) ScaleDeployment(ctx context.Context, f *framework.Framework, numReplicas int32) error { + return s.resourceStack.ScaleDeployment(ctx, f, numReplicas) +} + +func (s *NLBALBTestStack) Cleanup(ctx context.Context, f *framework.Framework) error { + return s.resourceStack.Cleanup(ctx, f) +} + +func (s *NLBALBTestStack) GetLoadBalancerIngressHostName() string { + return s.resourceStack.GetLoadBalancerIngressHostname() +} + +func (s *NLBALBTestStack) SendTrafficToLB(ctx context.Context, f *framework.Framework) error { + httpClient := http.Client{Timeout: utils.PollIntervalMedium} + protocol := "http" + if s.listenerTLS() { + protocol = "https" + httpClient.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + } + // Choose the first port for now, TODO: verify all listeners + port := s.resourceStack.svc.Spec.Ports[0].Port + noerr := false + for i := 0; i < 10; i++ { + resp, err := httpClient.Get(fmt.Sprintf("%s://%s:%v/from-tls-client", protocol, s.GetLoadBalancerIngressHostName(), port)) + if err != nil { + time.Sleep(2 * utils.PollIntervalLong) + continue + } + noerr = true + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("Unexpected HTTP status code %v", resp.StatusCode) + } + if resp.StatusCode == http.StatusOK { + break + } + } + if noerr { + return nil + } + return fmt.Errorf("Unsuccessful after 10 retries") +} + +func (s *NLBALBTestStack) listenerTLS() bool { + _, ok := s.resourceStack.svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-ssl-cert"] + return ok +} + +func (s *NLBALBTestStack) targetGroupTLS() bool { + return false +} diff --git a/test/e2e/service/nlb_alb_target_test.go b/test/e2e/service/nlb_alb_target_test.go new file mode 100644 index 000000000..2fd7d77ef --- /dev/null +++ b/test/e2e/service/nlb_alb_target_test.go @@ -0,0 +1,187 @@ +package service + +import ( + "context" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" +) + +var _ = Describe("k8s nlb service with alb target reconciled by the aws load balancer controller", func() { + var ( + ctx context.Context + deployment *appsv1.Deployment + numReplicas int32 + name string + dnsName string + lbARN string + labels map[string]string + stack NLBALBTestStack + ) + + BeforeEach(func() { + ctx = context.Background() + numReplicas = 3 + stack = NLBALBTestStack{} + name = "nlb-alb-e2e" + labels = map[string]string{ + "app.kubernetes.io/name": "nlb-alb", + "app.kubernetes.io/instance": name, + } + deployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &numReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "app", + ImagePullPolicy: corev1.PullAlways, + Image: defaultTestImage, + Ports: []corev1.ContainerPort{ + { + ContainerPort: appContainerPort, + }, + }, + }, + }, + }, + }, + }, + } + }) + + AfterEach(func() { + err := stack.Cleanup(ctx, tf) + Expect(err).NotTo(HaveOccurred()) + }) + + Context("NLB with ALB target configuration", func() { + var ( + svc *corev1.Service + ) + BeforeEach(func() { + svc = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "external", + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing", + "service.beta.kubernetes.io/aws-load-balancer-target-type": "alb", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: labels, + Ports: []corev1.ServicePort{ + { + Port: 80, + TargetPort: intstr.FromInt(80), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + }) + It("Should create and verify internet-facing NLB with ALB target", func() { + By("deploying stack", func() { + err := stack.Deploy(ctx, tf, svc, deployment) + Expect(err).NotTo(HaveOccurred()) + }) + By("checking service status for lb dns name", func() { + dnsName = stack.GetLoadBalancerIngressHostName() + Expect(dnsName).ToNot(BeEmpty()) + }) + + By("querying AWS loadbalancer from the dns name", func() { + var err error + lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName) + Expect(err).NotTo(HaveOccurred()) + Expect(lbARN).ToNot(BeEmpty()) + }) + By("Verify Service with AWS", func() { + err := verifyAWSLoadBalancerResources(ctx, tf, lbARN, LoadBalancerExpectation{ + Type: "network", + Scheme: "internet-facing", + TargetType: "alb", + Listeners: map[string]string{ + "80": "TCP", + }, + TargetGroups: map[string]string{ + "80": "TCP", + }, + NumTargets: int(numReplicas), + TargetGroupHC: &TargetGroupHC{ + Protocol: "TCP", + Port: "traffic-port", + Interval: 10, + Timeout: 10, + HealthyThreshold: 3, + UnhealthyThreshold: 3, + }, + }) + Expect(err).ToNot(HaveOccurred()) + }) + By("waiting for target group targets to be healthy", func() { + err := waitUntilTargetsAreHealthy(ctx, tf, lbARN, int(numReplicas)) + Expect(err).NotTo(HaveOccurred()) + }) + By("Send traffic to LB", func() { + err := stack.SendTrafficToLB(ctx, tf) + Expect(err).ToNot(HaveOccurred()) + }) + By("Specifying Healthcheck annotations", func() { + err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol": "HTTP", + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-port": "80", + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-path": "/healthz", + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval": "30", + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-timeout": "6", + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold": "2", + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold": "2", + }) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + return getTargetGroupHealthCheckProtocol(ctx, tf, lbARN) == "HTTP" + }, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue()) + + err = verifyAWSLoadBalancerResources(ctx, tf, lbARN, LoadBalancerExpectation{ + Type: "network", + Scheme: "internet-facing", + TargetType: "alb", + Listeners: map[string]string{ + "80": "TCP", + }, + TargetGroups: map[string]string{ + "80": "TCP", + }, + NumTargets: int(numReplicas), + TargetGroupHC: &TargetGroupHC{ + Protocol: "HTTP", + Port: "80", + Path: "/healthz", + Interval: 30, + Timeout: 6, + HealthyThreshold: 2, + UnhealthyThreshold: 2, + }, + }) + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) +}) diff --git a/webhooks/elbv2/targetgroupbinding_mutator.go b/webhooks/elbv2/targetgroupbinding_mutator.go index 2d46faf8a..2904ef464 100644 --- a/webhooks/elbv2/targetgroupbinding_mutator.go +++ b/webhooks/elbv2/targetgroupbinding_mutator.go @@ -65,6 +65,8 @@ func (m *targetGroupBindingMutator) defaultingTargetType(ctx context.Context, tg targetType = elbv2api.TargetTypeInstance case elbv2sdk.TargetTypeEnumIp: targetType = elbv2api.TargetTypeIP + case elbv2sdk.TargetTypeEnumAlb: + targetType = elbv2api.TargetTypeALB default: return errors.Errorf("unsupported TargetType: %v", sdkTargetType) } From 5ad66702a764376999170f39aec68bdde955403c Mon Sep 17 00:00:00 2001 From: Jon Fearer Date: Wed, 24 Nov 2021 14:41:24 -0700 Subject: [PATCH 2/9] feat(alb-target): update models and builders for registering alb target --- .../elbv2/v1beta1/targetgroupbinding_types.go | 11 ++++ controllers/service/service_controller.go | 2 +- pkg/deploy/elbv2/target_group_manager.go | 19 ++++++ pkg/model/elbv2/target_group.go | 6 ++ pkg/model/elbv2/target_group_binding.go | 7 ++ pkg/service/model_build_target_group.go | 64 ++++++++++++++++--- pkg/service/model_builder.go | 7 +- 7 files changed, 106 insertions(+), 10 deletions(-) diff --git a/apis/elbv2/v1beta1/targetgroupbinding_types.go b/apis/elbv2/v1beta1/targetgroupbinding_types.go index 3a0a8287e..841167164 100644 --- a/apis/elbv2/v1beta1/targetgroupbinding_types.go +++ b/apis/elbv2/v1beta1/targetgroupbinding_types.go @@ -45,6 +45,7 @@ const ( ) // ServiceReference defines reference to a Kubernetes Service and its ServicePort. +// This is only applicable for `instance` and `ip` target types. type ServiceReference struct { // Name is the name of the Service. Name string `json:"name"` @@ -53,6 +54,16 @@ type ServiceReference struct { Port intstr.IntOrString `json:"port"` } +// IngressReference defines reference to a Kubernetes Ingress and a port that it forwards traffic to. +// This is only applicable for the `alb` target type. +type IngressReference struct { + // Name is the name of the Ingress. + Name string `json:"name"` + + // Port is one of the ports listed in the Ingress spec. + Port intstr.IntOrString `json:"port"` +} + // IPBlock defines source/destination IPBlock in networking rules. type IPBlock struct { // CIDR is the network CIDR. diff --git a/controllers/service/service_controller.go b/controllers/service/service_controller.go index b288bde8a..be1691e4b 100644 --- a/controllers/service/service_controller.go +++ b/controllers/service/service_controller.go @@ -42,7 +42,7 @@ func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorde trackingProvider := tracking.NewDefaultProvider(serviceTagPrefix, config.ClusterName) elbv2TaggingManager := elbv2.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), config.FeatureGate, logger) modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcResolver, trackingProvider, - elbv2TaggingManager, config.ClusterName, config.DefaultTags, config.ExternalManagedTags, config.DefaultSSLPolicy) + elbv2TaggingManager, k8sClient, config.ClusterName, config.DefaultTags, config.ExternalManagedTags, config.DefaultSSLPolicy) stackMarshaller := deploy.NewDefaultStackMarshaller() stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, config, serviceTagPrefix, logger) return &serviceReconciler{ diff --git a/pkg/deploy/elbv2/target_group_manager.go b/pkg/deploy/elbv2/target_group_manager.go index d6e658919..444c9af8d 100644 --- a/pkg/deploy/elbv2/target_group_manager.go +++ b/pkg/deploy/elbv2/target_group_manager.go @@ -83,6 +83,16 @@ func (m *defaultTargetGroupManager) Create(ctx context.Context, resTG *elbv2mode "stackID", resTG.Stack().StackID(), "resourceID", resTG.ID(), "arn", awssdk.StringValue(sdkTG.TargetGroup.TargetGroupArn)) + + // TODO this should probably happen in the reconciler like the other targets(?) + if resTG.Spec.TargetType == elbv2model.TargetTypeALB && resTG.Spec.TargetLoadBalancerARN != "" { + registerReq := buildSDKRegisterLoadBalancerTargetInput(resTG.Spec, sdkTG.TargetGroup.TargetGroupArn) + _, err = m.elbv2Client.RegisterTargetsWithContext(ctx, registerReq) + if err != nil { + return elbv2model.TargetGroupStatus{}, err + } + } + if err := m.attributesReconciler.Reconcile(ctx, resTG, sdkTG); err != nil { return elbv2model.TargetGroupStatus{}, err } @@ -239,6 +249,15 @@ func buildSDKModifyTargetGroupInput(tgSpec elbv2model.TargetGroupSpec) *elbv2sdk return sdkObj } +func buildSDKRegisterLoadBalancerTargetInput(tgSpec elbv2model.TargetGroupSpec, tgARN *string) *elbv2sdk.RegisterTargetsInput { + return &elbv2sdk.RegisterTargetsInput{ + TargetGroupArn: tgARN, + Targets: []*elbv2sdk.TargetDescription{ + {Id: &tgSpec.TargetLoadBalancerARN, Port: &tgSpec.TargetLoadBalancerPort}, + }, + } +} + func buildSDKMatcher(modelMatcher elbv2model.HealthCheckMatcher) *elbv2sdk.Matcher { return &elbv2sdk.Matcher{ GrpcCode: modelMatcher.GRPCCode, diff --git a/pkg/model/elbv2/target_group.go b/pkg/model/elbv2/target_group.go index 08c9fbd33..125fb855b 100644 --- a/pkg/model/elbv2/target_group.go +++ b/pkg/model/elbv2/target_group.go @@ -148,6 +148,12 @@ type TargetGroupSpec struct { // +optional TargetGroupAttributes []TargetGroupAttribute `json:"targetGroupAttributes,omitempty"` + // The ARN of the application load balancer target. Only applicable if target type is "alb" + TargetLoadBalancerARN string `json:"targetLoadBalancerArn,omitempty"` + + // The port of the application load balancer target to forward traffic to. Only applicable if target type is "alb" + TargetLoadBalancerPort int64 `json:"targetLoadBalancerPort,omitempty"` + // The tags. // +optional Tags map[string]string `json:"tags,omitempty"` diff --git a/pkg/model/elbv2/target_group_binding.go b/pkg/model/elbv2/target_group_binding.go index 939947614..f9b0320e3 100644 --- a/pkg/model/elbv2/target_group_binding.go +++ b/pkg/model/elbv2/target_group_binding.go @@ -90,8 +90,15 @@ type TargetGroupBindingSpec struct { TargetType *elbv2api.TargetType `json:"targetType,omitempty"` // serviceRef is a reference to a Kubernetes Service and ServicePort. + // This is only applicable for `instance` and `ip` target types. + // +optional ServiceRef elbv2api.ServiceReference `json:"serviceRef"` + // ingressRef is a reference to a Kubernetes Ingress and a port listed in the Ingress spec. + // This is only applicable for the `alb` target type. + // +optional + IngressRef elbv2api.IngressReference `json:"ingressRef"` + // networking provides the networking setup for ELBV2 LoadBalancer to access targets in TargetGroup. // +optional Networking *TargetGroupBindingNetworking `json:"networking,omitempty"` diff --git a/pkg/service/model_build_target_group.go b/pkg/service/model_build_target_group.go index b0f8181d3..262e770c7 100644 --- a/pkg/service/model_build_target_group.go +++ b/pkg/service/model_build_target_group.go @@ -5,7 +5,10 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "k8s.io/api/networking/v1beta1" + "k8s.io/apimachinery/pkg/labels" "regexp" + "sigs.k8s.io/controller-runtime/pkg/client" "sort" "strconv" "strings" @@ -78,15 +81,26 @@ func (t *defaultModelBuildTask) buildTargetGroupSpec(ctx context.Context, tgProt if ipAddressType == elbv2model.TargetGroupIPAddressTypeIPv6 { return elbv2model.TargetGroupSpec{}, errors.New("ipv6 target group not supported for NLB") } + targetLoadBalancerARN, err := t.buildTargetLoadBalancerArn(ctx, targetType) + if err != nil { + return elbv2model.TargetGroupSpec{}, err + } + targetLoadBalancerPort, err := t.buildTargetLoadBalancerPort(ctx, targetType) + if err != nil { + return elbv2model.TargetGroupSpec{}, err + } return elbv2model.TargetGroupSpec{ - Name: tgName, - TargetType: targetType, - Port: targetPort, - Protocol: tgProtocol, - IPAddressType: &ipAddressType, - HealthCheckConfig: healthCheckConfig, - TargetGroupAttributes: tgAttrs, - Tags: tags, + Name: tgName, + TargetType: targetType, + Port: targetPort, + Protocol: tgProtocol, + IPAddressType: &ipAddressType, + HealthCheckConfig: healthCheckConfig, + TargetGroupAttributes: tgAttrs, + // TODO should probably use TargetGroupBindingResourceSpec instead(?) + TargetLoadBalancerARN: targetLoadBalancerARN, + TargetLoadBalancerPort: targetLoadBalancerPort, + Tags: tags, }, nil } @@ -531,6 +545,40 @@ func (t *defaultModelBuildTask) buildTargetGroupIPAddressType(_ context.Context, return elbv2model.TargetGroupIPAddressTypeIPv4, nil } +func (t *defaultModelBuildTask) buildTargetLoadBalancerArn(ctx context.Context, targetType elbv2model.TargetType) (string, error) { + var targetLoadBalancerARN string + if targetType == elbv2model.TargetTypeALB { + selectorSet := t.service.Spec.Selector + if len(selectorSet) > 0 { + selector := labels.SelectorFromSet(selectorSet) + ingressList := &v1beta1.IngressList{} + err := t.k8sClient.List(ctx, ingressList, &client.ListOptions{LabelSelector: selector}) + if err != nil { + return "", err + } + if len(ingressList.Items) == 0 { + return "", errors.New(fmt.Sprintf("no ingresses found with matching selector: %v", selectorSet)) + } + if len(ingressList.Items) > 1 { + return "", errors.New(fmt.Sprintf("multiple ingresses found with provided selector: %v", selectorSet)) + } + // TODO use the ingress to look up the matching ARN + targetLoadBalancerARN = "" + } else { + return "", errors.New("selector must be specified when target type is alb") + } + } + return targetLoadBalancerARN, nil +} + +func (t *defaultModelBuildTask) buildTargetLoadBalancerPort(_ context.Context, targetType elbv2model.TargetType) (int64, error) { + var targetLoadBalancerPort int64 + if targetType == elbv2model.TargetTypeALB { + targetLoadBalancerPort = int64(t.service.Spec.Ports[0].Port) + } + return targetLoadBalancerPort, nil +} + func (t *defaultModelBuildTask) buildTargetGroupBindingNodeSelector(_ context.Context, targetType elbv2model.TargetType) (*metav1.LabelSelector, error) { if targetType != elbv2model.TargetTypeInstance { return nil, nil diff --git a/pkg/service/model_builder.go b/pkg/service/model_builder.go index ea9d32f96..232c01e8b 100644 --- a/pkg/service/model_builder.go +++ b/pkg/service/model_builder.go @@ -4,6 +4,7 @@ import ( "context" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" "strconv" "sync" @@ -35,7 +36,7 @@ type ModelBuilder interface { // NewDefaultModelBuilder construct a new defaultModelBuilder func NewDefaultModelBuilder(annotationParser annotations.Parser, subnetsResolver networking.SubnetsResolver, - vpcResolver networking.VPCResolver, trackingProvider tracking.Provider, elbv2TaggingManager elbv2deploy.TaggingManager, + vpcResolver networking.VPCResolver, trackingProvider tracking.Provider, elbv2TaggingManager elbv2deploy.TaggingManager, k8sClient client.Client, clusterName string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string) *defaultModelBuilder { return &defaultModelBuilder{ annotationParser: annotationParser, @@ -43,6 +44,7 @@ func NewDefaultModelBuilder(annotationParser annotations.Parser, subnetsResolver vpcResolver: vpcResolver, trackingProvider: trackingProvider, elbv2TaggingManager: elbv2TaggingManager, + k8sClient: k8sClient, clusterName: clusterName, defaultTags: defaultTags, externalManagedTags: sets.NewString(externalManagedTags...), @@ -58,6 +60,7 @@ type defaultModelBuilder struct { vpcResolver networking.VPCResolver trackingProvider tracking.Provider elbv2TaggingManager elbv2deploy.TaggingManager + k8sClient client.Client clusterName string defaultTags map[string]string @@ -74,6 +77,7 @@ func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service vpcResolver: b.vpcResolver, trackingProvider: b.trackingProvider, elbv2TaggingManager: b.elbv2TaggingManager, + k8sClient: b.k8sClient, service: service, stack: stack, @@ -120,6 +124,7 @@ type defaultModelBuildTask struct { vpcResolver networking.VPCResolver trackingProvider tracking.Provider elbv2TaggingManager elbv2deploy.TaggingManager + k8sClient client.Client service *corev1.Service From 540f7adc8c8ffb35fc4cc7924fe90c4c6aa7c4c1 Mon Sep 17 00:00:00 2001 From: Jon Fearer Date: Tue, 30 Nov 2021 13:14:11 -0700 Subject: [PATCH 3/9] feat(alb-target): update targetgroupbinding to support ingress ref --- .../elbv2/v1beta1/targetgroupbinding_types.go | 7 ++ .../elbv2.k8s.aws_targetgroupbindings.yaml | 19 ++- .../elbv2/target_group_binding_manager.go | 1 + pkg/deploy/elbv2/target_group_manager.go | 18 --- pkg/model/elbv2/target_group.go | 6 - pkg/service/model_build_target_group.go | 114 ++++++++---------- pkg/targetgroupbinding/resource_manager.go | 47 ++++++++ 7 files changed, 123 insertions(+), 89 deletions(-) diff --git a/apis/elbv2/v1beta1/targetgroupbinding_types.go b/apis/elbv2/v1beta1/targetgroupbinding_types.go index 841167164..2aa7caa2e 100644 --- a/apis/elbv2/v1beta1/targetgroupbinding_types.go +++ b/apis/elbv2/v1beta1/targetgroupbinding_types.go @@ -145,8 +145,15 @@ type TargetGroupBindingSpec struct { TargetType *TargetType `json:"targetType,omitempty"` // serviceRef is a reference to a Kubernetes Service and ServicePort. + // This is only applicable for `instance` and `ip` target types. + // +optional ServiceRef ServiceReference `json:"serviceRef"` + // ingressRef is a reference to a Kubernetes Ingress and a port listed in the Ingress spec. + // This is only applicable for the `alb` target type. + // +optional + IngressRef IngressReference `json:"ingressRef"` + // networking defines the networking rules to allow ELBV2 LoadBalancer to access targets in TargetGroup. // +optional Networking *TargetGroupBindingNetworking `json:"networking,omitempty"` diff --git a/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml b/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml index 7d5190bf8..49844162b 100644 --- a/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml +++ b/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml @@ -188,6 +188,22 @@ spec: spec: description: TargetGroupBindingSpec defines the desired state of TargetGroupBinding properties: + ingressRef: + description: ingressRef is a reference to a Kubernetes Ingress and a port listed in the Ingress spec. This is only applicable for the `alb` target type. + properties: + name: + description: Name is the name of the Ingress. + type: string + port: + anyOf: + - type: integer + - type: string + description: Port is one of the ports listed in the Ingress spec. + x-kubernetes-int-or-string: true + required: + - name + - port + type: object ipAddressType: description: ipAddressType specifies whether the target group is of type IPv4 or IPv6. If unspecified, it will be automatically inferred. enum: @@ -283,7 +299,7 @@ spec: type: object type: object serviceRef: - description: serviceRef is a reference to a Kubernetes Service and ServicePort. + description: serviceRef is a reference to a Kubernetes Service and ServicePort. This is only applicable for `instance` and `ip` target types. properties: name: description: Name is the name of the Service. @@ -310,7 +326,6 @@ spec: - alb type: string required: - - serviceRef - targetGroupARN type: object status: diff --git a/pkg/deploy/elbv2/target_group_binding_manager.go b/pkg/deploy/elbv2/target_group_binding_manager.go index 7081fbb2e..908c88bdc 100644 --- a/pkg/deploy/elbv2/target_group_binding_manager.go +++ b/pkg/deploy/elbv2/target_group_binding_manager.go @@ -176,6 +176,7 @@ func buildK8sTargetGroupBindingSpec(ctx context.Context, resTGB *elbv2model.Targ TargetGroupARN: tgARN, TargetType: resTGB.Spec.Template.Spec.TargetType, ServiceRef: resTGB.Spec.Template.Spec.ServiceRef, + IngressRef: resTGB.Spec.Template.Spec.IngressRef, } if resTGB.Spec.Template.Spec.Networking != nil { diff --git a/pkg/deploy/elbv2/target_group_manager.go b/pkg/deploy/elbv2/target_group_manager.go index 444c9af8d..5d2fb8f57 100644 --- a/pkg/deploy/elbv2/target_group_manager.go +++ b/pkg/deploy/elbv2/target_group_manager.go @@ -84,15 +84,6 @@ func (m *defaultTargetGroupManager) Create(ctx context.Context, resTG *elbv2mode "resourceID", resTG.ID(), "arn", awssdk.StringValue(sdkTG.TargetGroup.TargetGroupArn)) - // TODO this should probably happen in the reconciler like the other targets(?) - if resTG.Spec.TargetType == elbv2model.TargetTypeALB && resTG.Spec.TargetLoadBalancerARN != "" { - registerReq := buildSDKRegisterLoadBalancerTargetInput(resTG.Spec, sdkTG.TargetGroup.TargetGroupArn) - _, err = m.elbv2Client.RegisterTargetsWithContext(ctx, registerReq) - if err != nil { - return elbv2model.TargetGroupStatus{}, err - } - } - if err := m.attributesReconciler.Reconcile(ctx, resTG, sdkTG); err != nil { return elbv2model.TargetGroupStatus{}, err } @@ -249,15 +240,6 @@ func buildSDKModifyTargetGroupInput(tgSpec elbv2model.TargetGroupSpec) *elbv2sdk return sdkObj } -func buildSDKRegisterLoadBalancerTargetInput(tgSpec elbv2model.TargetGroupSpec, tgARN *string) *elbv2sdk.RegisterTargetsInput { - return &elbv2sdk.RegisterTargetsInput{ - TargetGroupArn: tgARN, - Targets: []*elbv2sdk.TargetDescription{ - {Id: &tgSpec.TargetLoadBalancerARN, Port: &tgSpec.TargetLoadBalancerPort}, - }, - } -} - func buildSDKMatcher(modelMatcher elbv2model.HealthCheckMatcher) *elbv2sdk.Matcher { return &elbv2sdk.Matcher{ GrpcCode: modelMatcher.GRPCCode, diff --git a/pkg/model/elbv2/target_group.go b/pkg/model/elbv2/target_group.go index 125fb855b..08c9fbd33 100644 --- a/pkg/model/elbv2/target_group.go +++ b/pkg/model/elbv2/target_group.go @@ -148,12 +148,6 @@ type TargetGroupSpec struct { // +optional TargetGroupAttributes []TargetGroupAttribute `json:"targetGroupAttributes,omitempty"` - // The ARN of the application load balancer target. Only applicable if target type is "alb" - TargetLoadBalancerARN string `json:"targetLoadBalancerArn,omitempty"` - - // The port of the application load balancer target to forward traffic to. Only applicable if target type is "alb" - TargetLoadBalancerPort int64 `json:"targetLoadBalancerPort,omitempty"` - // The tags. // +optional Tags map[string]string `json:"tags,omitempty"` diff --git a/pkg/service/model_build_target_group.go b/pkg/service/model_build_target_group.go index 262e770c7..2b85c5e07 100644 --- a/pkg/service/model_build_target_group.go +++ b/pkg/service/model_build_target_group.go @@ -5,10 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "k8s.io/api/networking/v1beta1" - "k8s.io/apimachinery/pkg/labels" "regexp" - "sigs.k8s.io/controller-runtime/pkg/client" "sort" "strconv" "strings" @@ -81,26 +78,15 @@ func (t *defaultModelBuildTask) buildTargetGroupSpec(ctx context.Context, tgProt if ipAddressType == elbv2model.TargetGroupIPAddressTypeIPv6 { return elbv2model.TargetGroupSpec{}, errors.New("ipv6 target group not supported for NLB") } - targetLoadBalancerARN, err := t.buildTargetLoadBalancerArn(ctx, targetType) - if err != nil { - return elbv2model.TargetGroupSpec{}, err - } - targetLoadBalancerPort, err := t.buildTargetLoadBalancerPort(ctx, targetType) - if err != nil { - return elbv2model.TargetGroupSpec{}, err - } return elbv2model.TargetGroupSpec{ - Name: tgName, - TargetType: targetType, - Port: targetPort, - Protocol: tgProtocol, - IPAddressType: &ipAddressType, - HealthCheckConfig: healthCheckConfig, - TargetGroupAttributes: tgAttrs, - // TODO should probably use TargetGroupBindingResourceSpec instead(?) - TargetLoadBalancerARN: targetLoadBalancerARN, - TargetLoadBalancerPort: targetLoadBalancerPort, - Tags: tags, + Name: tgName, + TargetType: targetType, + Port: targetPort, + Protocol: tgProtocol, + IPAddressType: &ipAddressType, + HealthCheckConfig: healthCheckConfig, + TargetGroupAttributes: tgAttrs, + Tags: tags, }, nil } @@ -435,6 +421,14 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingSpec(ctx context.Context, if targetType == elbv2api.TargetTypeInstance { targetPort = intstr.FromInt(int(port.NodePort)) } + serviceRef, err := t.buildTargetGroupBindingServiceRef(ctx, targetType, port) + if err != nil { + return elbv2model.TargetGroupBindingResourceSpec{}, err + } + ingressRef, err := t.buildTargetGroupBindingIngressRef(ctx, targetType, port) + if err != nil { + return elbv2model.TargetGroupBindingResourceSpec{}, err + } defaultSourceRanges := []string{"0.0.0.0/0"} if (port.Protocol == corev1.ProtocolUDP || preserveClientIP) && scheme == elbv2model.LoadBalancerSchemeInternal { defaultSourceRanges, err = t.vpcResolver.ResolveCIDRs(ctx) @@ -452,13 +446,11 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingSpec(ctx context.Context, Spec: elbv2model.TargetGroupBindingSpec{ TargetGroupARN: targetGroup.TargetGroupARN(), TargetType: &targetType, - ServiceRef: elbv2api.ServiceReference{ - Name: t.service.Name, - Port: intstr.FromInt(int(port.Port)), - }, - Networking: tgbNetworking, - NodeSelector: nodeSelector, - IPAddressType: (*elbv2api.TargetGroupIPAddressType)(targetGroup.Spec.IPAddressType), + ServiceRef: serviceRef, + IngressRef: ingressRef, + Networking: tgbNetworking, + NodeSelector: nodeSelector, + IPAddressType: (*elbv2api.TargetGroupIPAddressType)(targetGroup.Spec.IPAddressType), }, }, }, nil @@ -545,40 +537,6 @@ func (t *defaultModelBuildTask) buildTargetGroupIPAddressType(_ context.Context, return elbv2model.TargetGroupIPAddressTypeIPv4, nil } -func (t *defaultModelBuildTask) buildTargetLoadBalancerArn(ctx context.Context, targetType elbv2model.TargetType) (string, error) { - var targetLoadBalancerARN string - if targetType == elbv2model.TargetTypeALB { - selectorSet := t.service.Spec.Selector - if len(selectorSet) > 0 { - selector := labels.SelectorFromSet(selectorSet) - ingressList := &v1beta1.IngressList{} - err := t.k8sClient.List(ctx, ingressList, &client.ListOptions{LabelSelector: selector}) - if err != nil { - return "", err - } - if len(ingressList.Items) == 0 { - return "", errors.New(fmt.Sprintf("no ingresses found with matching selector: %v", selectorSet)) - } - if len(ingressList.Items) > 1 { - return "", errors.New(fmt.Sprintf("multiple ingresses found with provided selector: %v", selectorSet)) - } - // TODO use the ingress to look up the matching ARN - targetLoadBalancerARN = "" - } else { - return "", errors.New("selector must be specified when target type is alb") - } - } - return targetLoadBalancerARN, nil -} - -func (t *defaultModelBuildTask) buildTargetLoadBalancerPort(_ context.Context, targetType elbv2model.TargetType) (int64, error) { - var targetLoadBalancerPort int64 - if targetType == elbv2model.TargetTypeALB { - targetLoadBalancerPort = int64(t.service.Spec.Ports[0].Port) - } - return targetLoadBalancerPort, nil -} - func (t *defaultModelBuildTask) buildTargetGroupBindingNodeSelector(_ context.Context, targetType elbv2model.TargetType) (*metav1.LabelSelector, error) { if targetType != elbv2model.TargetTypeInstance { return nil, nil @@ -595,6 +553,36 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingNodeSelector(_ context.Co }, nil } +func (t *defaultModelBuildTask) buildTargetGroupBindingServiceRef(_ context.Context, targetType elbv2api.TargetType, + port corev1.ServicePort) (elbv2api.ServiceReference, error) { + var serviceRef elbv2api.ServiceReference + if targetType != elbv2api.TargetTypeALB { + serviceRef = elbv2api.ServiceReference{ + Name: t.service.Name, + Port: intstr.FromInt(int(port.Port)), + } + } + return serviceRef, nil +} + +func (t *defaultModelBuildTask) buildTargetGroupBindingIngressRef(_ context.Context, targetType elbv2api.TargetType, + port corev1.ServicePort) (elbv2api.IngressReference, error) { + var ingressRef elbv2api.IngressReference + if targetType == elbv2api.TargetTypeALB { + var ingressRefName string + if value, ok := t.service.Spec.Selector["ingress"]; ok { + ingressRefName = value + } else { + return elbv2api.IngressReference{}, errors.New("ingress selector must be specified with target type is alb") + } + ingressRef = elbv2api.IngressReference{ + Name: ingressRefName, + Port: intstr.FromInt(int(port.Port)), + } + } + return ingressRef, nil +} + func (t *defaultModelBuildTask) buildHealthCheckNetworkingIngressRules(trafficSource, hcSource []elbv2model.NetworkingPeer, tgPort, hcPort intstr.IntOrString, tgProtocol corev1.Protocol, preserveClientIP, customSoureRanges bool) []elbv2model.NetworkingIngressRule { if tgProtocol != corev1.ProtocolUDP && diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index 8e017af79..f4158e6c7 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "k8s.io/api/networking/v1beta1" "net" "time" @@ -59,6 +60,7 @@ func NewDefaultResourceManager(k8sClient client.Client, elbv2Client services.ELB logger: logger, vpcID: vpcID, vpcInfoProvider: vpcInfoProvider, + elbv2Client: elbv2Client, targetHealthRequeueDuration: defaultTargetHealthRequeueDuration, enableEndpointSlices: useEndpointSlices, @@ -77,6 +79,7 @@ type defaultResourceManager struct { logger logr.Logger vpcInfoProvider networking.VPCInfoProvider vpcID string + elbv2Client services.ELBV2 targetHealthRequeueDuration time.Duration enableEndpointSlices bool @@ -206,9 +209,53 @@ func (m *defaultResourceManager) reconcileWithInstanceTargetType(ctx context.Con } func (m *defaultResourceManager) reconcileWithALBTargetType(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { + tgARN := tgb.Spec.TargetGroupARN + loadBalancerARN, err := m.buildTargetLoadBalancerArn(ctx, tgb) + if err != nil { + return err + } + err = m.targetsManager.RegisterTargets(ctx, tgARN, []elbv2sdk.TargetDescription{ + { + Id: awssdk.String(loadBalancerARN), + Port: awssdk.Int64(int64(tgb.Spec.IngressRef.Port.IntVal)), + }, + }) + if err != nil { + return err + } return nil } +func (m *defaultResourceManager) buildTargetLoadBalancerArn(ctx context.Context, tgb *elbv2api.TargetGroupBinding) (string, error) { + ingressName := tgb.Spec.IngressRef.Name + ingressKey := types.NamespacedName{Namespace: tgb.Namespace, Name: ingressName} + ingress := &v1beta1.Ingress{} + err := m.k8sClient.Get(ctx, ingressKey, ingress) + if err != nil { + return "", err + } + ingressHostName := ingress.Status.LoadBalancer.Ingress[0].Hostname + if ingressHostName == "" { + return "", errors.New(fmt.Sprintf("ingress %s has not been assigned a hostname", ingressName)) + } + loadBalancers, err := m.elbv2Client.DescribeLoadBalancersAsList(ctx, &elbv2sdk.DescribeLoadBalancersInput{}) + if err != nil { + return "", err + } + var ingressLoadBalancer *elbv2sdk.LoadBalancer + for _, loadBalancer := range loadBalancers { + loadBalancerHostname := loadBalancer.DNSName + if *loadBalancerHostname == ingressHostName { + ingressLoadBalancer = loadBalancer + break + } + } + if ingressLoadBalancer == nil { + return "", errors.New(fmt.Sprintf("unable to find matching load balancer for ingress %s", ingressName)) + } + return *ingressLoadBalancer.LoadBalancerArn, nil +} + func (m *defaultResourceManager) cleanupTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { targets, err := m.targetsManager.ListTargets(ctx, tgb.Spec.TargetGroupARN) if err != nil { From 0de4b7a896a978a72f09c81c7ea7ed9d9d09a2ea Mon Sep 17 00:00:00 2001 From: Jon Fearer Date: Tue, 30 Nov 2021 16:56:07 -0700 Subject: [PATCH 4/9] feat(alb-target): fix issue with target group deletion --- .../elbv2/target_group_binding_manager.go | 2 +- pkg/deploy/elbv2/target_group_manager.go | 1 - pkg/deploy/elbv2/target_group_synthesizer.go | 5 +--- pkg/targetgroupbinding/resource_manager.go | 24 +++++++++++++++++-- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/pkg/deploy/elbv2/target_group_binding_manager.go b/pkg/deploy/elbv2/target_group_binding_manager.go index 908c88bdc..edffab34b 100644 --- a/pkg/deploy/elbv2/target_group_binding_manager.go +++ b/pkg/deploy/elbv2/target_group_binding_manager.go @@ -126,7 +126,7 @@ func (m *defaultTargetGroupBindingManager) Delete(ctx context.Context, tgb *elbv return err } if err := m.waitUntilTargetGroupBindingDeleted(ctx, tgb); err != nil { - return errors.Wrap(err, "failed to wait targetGroupBinding deletion") + return errors.Wrap(err, "failed to wait for targetGroupBinding deletion") } m.logger.Info("deleted targetGroupBinding", "targetGroupBinding", k8s.NamespacedName(tgb)) diff --git a/pkg/deploy/elbv2/target_group_manager.go b/pkg/deploy/elbv2/target_group_manager.go index 5d2fb8f57..d6e658919 100644 --- a/pkg/deploy/elbv2/target_group_manager.go +++ b/pkg/deploy/elbv2/target_group_manager.go @@ -83,7 +83,6 @@ func (m *defaultTargetGroupManager) Create(ctx context.Context, resTG *elbv2mode "stackID", resTG.Stack().StackID(), "resourceID", resTG.ID(), "arn", awssdk.StringValue(sdkTG.TargetGroup.TargetGroupArn)) - if err := m.attributesReconciler.Reconcile(ctx, resTG, sdkTG); err != nil { return elbv2model.TargetGroupStatus{}, err } diff --git a/pkg/deploy/elbv2/target_group_synthesizer.go b/pkg/deploy/elbv2/target_group_synthesizer.go index 12ff30d4c..e508b7953 100644 --- a/pkg/deploy/elbv2/target_group_synthesizer.go +++ b/pkg/deploy/elbv2/target_group_synthesizer.go @@ -40,10 +40,7 @@ type targetGroupSynthesizer struct { func (s *targetGroupSynthesizer) Synthesize(ctx context.Context) error { var resTGs []*elbv2model.TargetGroup - err := s.stack.ListResources(&resTGs) - if err != nil { - return err - } + s.stack.ListResources(&resTGs) sdkTGs, err := s.findSDKTargetGroups(ctx) if err != nil { return err diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index 863ad3793..d533767e7 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -4,9 +4,8 @@ import ( "context" "encoding/json" "fmt" - "k8s.io/api/networking/v1beta1" - "net" "inet.af/netaddr" + "k8s.io/api/networking/v1beta1" "time" "k8s.io/client-go/tools/record" @@ -218,6 +217,14 @@ func (m *defaultResourceManager) reconcileWithInstanceTargetType(ctx context.Con func (m *defaultResourceManager) reconcileWithALBTargetType(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { tgARN := tgb.Spec.TargetGroupARN + targets, err := m.targetsManager.ListTargets(ctx, tgARN) + if err != nil { + return err + } + // Target groups with alb target type can only have one target load balancer + if len(targets) == 1 { + return nil + } loadBalancerARN, err := m.buildTargetLoadBalancerArn(ctx, tgb) if err != nil { return err @@ -272,6 +279,9 @@ func (m *defaultResourceManager) cleanupTargets(ctx context.Context, tgb *elbv2a } return err } + if *tgb.Spec.TargetType == elbv2api.TargetTypeALB { + targets = stripAvailabilityZonesFromTargets(targets) + } if err := m.deregisterTargets(ctx, tgb.Spec.TargetGroupARN, targets); err != nil { if isELBV2TargetGroupNotFoundError(err) { return nil @@ -561,3 +571,13 @@ func isELBV2TargetGroupNotFoundError(err error) bool { } return false } + +func stripAvailabilityZonesFromTargets(targets []TargetInfo) []TargetInfo { + var strippedTargets []TargetInfo + for _, target := range targets { + strippedTarget := target + strippedTarget.Target.AvailabilityZone = nil + strippedTargets = append(strippedTargets, strippedTarget) + } + return strippedTargets +} From 78b3a8a05bb4055b7fa98f4085dadd290d0c2a8d Mon Sep 17 00:00:00 2001 From: Jon Fearer Date: Tue, 30 Nov 2021 19:24:00 -0700 Subject: [PATCH 5/9] refactor(resource-manager): fix err redeclare --- pkg/targetgroupbinding/resource_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index d533767e7..5be83a13d 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -282,7 +282,7 @@ func (m *defaultResourceManager) cleanupTargets(ctx context.Context, tgb *elbv2a if *tgb.Spec.TargetType == elbv2api.TargetTypeALB { targets = stripAvailabilityZonesFromTargets(targets) } - if err := m.deregisterTargets(ctx, tgb.Spec.TargetGroupARN, targets); err != nil { + if err = m.deregisterTargets(ctx, tgb.Spec.TargetGroupARN, targets); err != nil { if isELBV2TargetGroupNotFoundError(err) { return nil } From 1b656194ada3cc476039051408cedd9e85d6dd62 Mon Sep 17 00:00:00 2001 From: Jon Fearer Date: Tue, 30 Nov 2021 19:47:12 -0700 Subject: [PATCH 6/9] feat(alb-target): len check; comment --- pkg/targetgroupbinding/resource_manager.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index 5be83a13d..876326efe 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -249,8 +249,12 @@ func (m *defaultResourceManager) buildTargetLoadBalancerArn(ctx context.Context, if err != nil { return "", err } - ingressHostName := ingress.Status.LoadBalancer.Ingress[0].Hostname - if ingressHostName == "" { + var ingressHostname string + loadBalancerStatus := ingress.Status.LoadBalancer.Ingress + if len(loadBalancerStatus) > 0 { + ingressHostname = loadBalancerStatus[0].Hostname + } + if ingressHostname == "" { return "", errors.New(fmt.Sprintf("ingress %s has not been assigned a hostname", ingressName)) } loadBalancers, err := m.elbv2Client.DescribeLoadBalancersAsList(ctx, &elbv2sdk.DescribeLoadBalancersInput{}) @@ -260,7 +264,7 @@ func (m *defaultResourceManager) buildTargetLoadBalancerArn(ctx context.Context, var ingressLoadBalancer *elbv2sdk.LoadBalancer for _, loadBalancer := range loadBalancers { loadBalancerHostname := loadBalancer.DNSName - if *loadBalancerHostname == ingressHostName { + if *loadBalancerHostname == ingressHostname { ingressLoadBalancer = loadBalancer break } @@ -279,6 +283,7 @@ func (m *defaultResourceManager) cleanupTargets(ctx context.Context, tgb *elbv2a } return err } + // Per AWS docs, AvailabilityZone parameter is not supported if the target type is alb if *tgb.Spec.TargetType == elbv2api.TargetTypeALB { targets = stripAvailabilityZonesFromTargets(targets) } From 59b1a5eead1a2e8ce4aeed78fe4aa01b10d61917 Mon Sep 17 00:00:00 2001 From: Jon Fearer Date: Wed, 1 Dec 2021 14:31:12 -0700 Subject: [PATCH 7/9] feat(alb-target): cleanup --- pkg/service/model_build_target_group.go | 2 +- pkg/targetgroupbinding/resource_manager.go | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/service/model_build_target_group.go b/pkg/service/model_build_target_group.go index 2b85c5e07..b225c2933 100644 --- a/pkg/service/model_build_target_group.go +++ b/pkg/service/model_build_target_group.go @@ -573,7 +573,7 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingIngressRef(_ context.Cont if value, ok := t.service.Spec.Selector["ingress"]; ok { ingressRefName = value } else { - return elbv2api.IngressReference{}, errors.New("ingress selector must be specified with target type is alb") + return elbv2api.IngressReference{}, errors.New("ingress selector must be specified when target type is alb") } ingressRef = elbv2api.IngressReference{ Name: ingressRefName, diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index 876326efe..44cebcdc4 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -229,16 +229,12 @@ func (m *defaultResourceManager) reconcileWithALBTargetType(ctx context.Context, if err != nil { return err } - err = m.targetsManager.RegisterTargets(ctx, tgARN, []elbv2sdk.TargetDescription{ + return m.targetsManager.RegisterTargets(ctx, tgARN, []elbv2sdk.TargetDescription{ { Id: awssdk.String(loadBalancerARN), Port: awssdk.Int64(int64(tgb.Spec.IngressRef.Port.IntVal)), }, }) - if err != nil { - return err - } - return nil } func (m *defaultResourceManager) buildTargetLoadBalancerArn(ctx context.Context, tgb *elbv2api.TargetGroupBinding) (string, error) { From 34d1c638824b9dc13332bb3656c5ca1c79e0169d Mon Sep 17 00:00:00 2001 From: Jon Fearer Date: Wed, 1 Dec 2021 16:20:38 -0700 Subject: [PATCH 8/9] feat(alb-target): fix issue with listing alb targets --- pkg/service/model_builder.go | 2 +- pkg/targetgroupbinding/resource_manager.go | 18 ++++-------------- pkg/targetgroupbinding/targets_manager.go | 12 +++++++++--- pkg/targetgroupbinding/utils.go | 10 ++++++++++ 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/pkg/service/model_builder.go b/pkg/service/model_builder.go index bed3636b5..7c5ca3d60 100644 --- a/pkg/service/model_builder.go +++ b/pkg/service/model_builder.go @@ -37,7 +37,7 @@ type ModelBuilder interface { // NewDefaultModelBuilder construct a new defaultModelBuilder func NewDefaultModelBuilder(annotationParser annotations.Parser, subnetsResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, vpcID string, trackingProvider tracking.Provider, elbv2TaggingManager elbv2deploy.TaggingManager, - k8sClient client.Client, clusterName string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string) *defaultModelBuilder { + k8sClient client.Client, clusterName string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string) *defaultModelBuilder { return &defaultModelBuilder{ annotationParser: annotationParser, subnetsResolver: subnetsResolver, diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index 44cebcdc4..da6fa194c 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -133,7 +133,7 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context, } tgARN := tgb.Spec.TargetGroupARN - targets, err := m.targetsManager.ListTargets(ctx, tgARN) + targets, err := m.targetsManager.ListTargets(ctx, tgARN, *tgb.Spec.TargetType) if err != nil { return err } @@ -191,7 +191,7 @@ func (m *defaultResourceManager) reconcileWithInstanceTargetType(ctx context.Con return err } tgARN := tgb.Spec.TargetGroupARN - targets, err := m.targetsManager.ListTargets(ctx, tgARN) + targets, err := m.targetsManager.ListTargets(ctx, tgARN, *tgb.Spec.TargetType) if err != nil { return err } @@ -217,7 +217,7 @@ func (m *defaultResourceManager) reconcileWithInstanceTargetType(ctx context.Con func (m *defaultResourceManager) reconcileWithALBTargetType(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { tgARN := tgb.Spec.TargetGroupARN - targets, err := m.targetsManager.ListTargets(ctx, tgARN) + targets, err := m.targetsManager.ListTargets(ctx, tgARN, *tgb.Spec.TargetType) if err != nil { return err } @@ -272,7 +272,7 @@ func (m *defaultResourceManager) buildTargetLoadBalancerArn(ctx context.Context, } func (m *defaultResourceManager) cleanupTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { - targets, err := m.targetsManager.ListTargets(ctx, tgb.Spec.TargetGroupARN) + targets, err := m.targetsManager.ListTargets(ctx, tgb.Spec.TargetGroupARN, *tgb.Spec.TargetType) if err != nil { if isELBV2TargetGroupNotFoundError(err) { return nil @@ -572,13 +572,3 @@ func isELBV2TargetGroupNotFoundError(err error) bool { } return false } - -func stripAvailabilityZonesFromTargets(targets []TargetInfo) []TargetInfo { - var strippedTargets []TargetInfo - for _, target := range targets { - strippedTarget := target - strippedTarget.Target.AvailabilityZone = nil - strippedTargets = append(strippedTargets, strippedTarget) - } - return strippedTargets -} diff --git a/pkg/targetgroupbinding/targets_manager.go b/pkg/targetgroupbinding/targets_manager.go index f003e8196..2b2efb7fd 100644 --- a/pkg/targetgroupbinding/targets_manager.go +++ b/pkg/targetgroupbinding/targets_manager.go @@ -6,6 +6,7 @@ import ( elbv2sdk "github.com/aws/aws-sdk-go/service/elbv2" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/cache" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" "sync" "time" @@ -26,7 +27,7 @@ type TargetsManager interface { DeregisterTargets(ctx context.Context, tgARN string, targets []elbv2sdk.TargetDescription) error // List Targets from TargetGroup. - ListTargets(ctx context.Context, tgARN string) ([]TargetInfo, error) + ListTargets(ctx context.Context, tgARN string, targetType elbv2api.TargetType) ([]TargetInfo, error) } // NewCachedTargetsManager constructs new cachedTargetsManager @@ -117,7 +118,7 @@ func (m *cachedTargetsManager) DeregisterTargets(ctx context.Context, tgARN stri return nil } -func (m *cachedTargetsManager) ListTargets(ctx context.Context, tgARN string) ([]TargetInfo, error) { +func (m *cachedTargetsManager) ListTargets(ctx context.Context, tgARN string, targetType elbv2api.TargetType) ([]TargetInfo, error) { m.targetsCacheMutex.Lock() defer m.targetsCacheMutex.Unlock() @@ -125,7 +126,12 @@ func (m *cachedTargetsManager) ListTargets(ctx context.Context, tgARN string) ([ targetsCacheItem := rawTargetsCacheItem.(*targetsCacheItem) targetsCacheItem.mutex.Lock() defer targetsCacheItem.mutex.Unlock() - refreshedTargets, err := m.refreshUnhealthyTargets(ctx, tgARN, targetsCacheItem.targets) + cachedTargets := targetsCacheItem.targets + // Per AWS docs, AvailabilityZone parameter is not supported if the target type is alb + if targetType == elbv2api.TargetTypeALB { + cachedTargets = stripAvailabilityZonesFromTargets(cachedTargets) + } + refreshedTargets, err := m.refreshUnhealthyTargets(ctx, tgARN, cachedTargets) if err != nil { return nil, err } diff --git a/pkg/targetgroupbinding/utils.go b/pkg/targetgroupbinding/utils.go index 65f386096..efae026cc 100644 --- a/pkg/targetgroupbinding/utils.go +++ b/pkg/targetgroupbinding/utils.go @@ -35,3 +35,13 @@ func buildServiceReferenceKey(tgb *elbv2api.TargetGroupBinding, svcRef elbv2api. Name: svcRef.Name, } } + +func stripAvailabilityZonesFromTargets(targets []TargetInfo) []TargetInfo { + var strippedTargets []TargetInfo + for _, target := range targets { + strippedTarget := target + strippedTarget.Target.AvailabilityZone = nil + strippedTargets = append(strippedTargets, strippedTarget) + } + return strippedTargets +} From f63c68a12420a6211c27e4948b9f97114d4fc5c3 Mon Sep 17 00:00:00 2001 From: Jon Fearer Date: Wed, 15 Dec 2021 10:30:47 -0700 Subject: [PATCH 9/9] fix(alb-target): use networking v1 instead of v1beta1 --- pkg/targetgroupbinding/resource_manager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index da6fa194c..7e16018a3 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "inet.af/netaddr" - "k8s.io/api/networking/v1beta1" "time" "k8s.io/client-go/tools/record" @@ -18,6 +17,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/strategicpatch" @@ -240,7 +240,7 @@ func (m *defaultResourceManager) reconcileWithALBTargetType(ctx context.Context, func (m *defaultResourceManager) buildTargetLoadBalancerArn(ctx context.Context, tgb *elbv2api.TargetGroupBinding) (string, error) { ingressName := tgb.Spec.IngressRef.Name ingressKey := types.NamespacedName{Namespace: tgb.Namespace, Name: ingressName} - ingress := &v1beta1.Ingress{} + ingress := &networkingv1.Ingress{} err := m.k8sClient.Get(ctx, ingressKey, ingress) if err != nil { return "", err