From b2a63ce924602be64a156361f3ee85498843026a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6hn?= <christian@ckoehn.de>
Date: Wed, 17 Apr 2019 21:51:49 +0200
Subject: [PATCH] pkg/util: Don't expose unready nodes via client service
 (#2063)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Previously unready etcd nodes were already receiving client connections
although they are still in the initiation phase and not able to accept
any traffic. This caused connection failure or high latency.

Fixes #2030

Signed-off-by: Christian Köhn <christian.koehn@figo.io>
---
 CHANGELOG.md                |  2 ++
 pkg/util/k8sutil/k8sutil.go | 14 ++++++++------
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e674f87f1..2b370d044 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,8 @@
 
 ### Fixed
 
+- Don't expose unready nodes via client service. [#2063](https://github.com/coreos/etcd-operator/pull/2063)
+
 ### Deprecated
 
 ### Security
diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go
index 5b3dc95ae..8db36f18a 100644
--- a/pkg/util/k8sutil/k8sutil.go
+++ b/pkg/util/k8sutil/k8sutil.go
@@ -20,6 +20,7 @@ import (
 	"net"
 	"net/url"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -158,7 +159,7 @@ func CreateClientService(kubecli kubernetes.Interface, clusterName, ns string, o
 		TargetPort: intstr.FromInt(EtcdClientPort),
 		Protocol:   v1.ProtocolTCP,
 	}}
-	return createService(kubecli, ClientServiceName(clusterName), clusterName, ns, "", ports, owner)
+	return createService(kubecli, ClientServiceName(clusterName), clusterName, ns, "", ports, owner, false)
 }
 
 func ClientServiceName(clusterName string) string {
@@ -178,11 +179,11 @@ func CreatePeerService(kubecli kubernetes.Interface, clusterName, ns string, own
 		Protocol:   v1.ProtocolTCP,
 	}}
 
-	return createService(kubecli, clusterName, clusterName, ns, v1.ClusterIPNone, ports, owner)
+	return createService(kubecli, clusterName, clusterName, ns, v1.ClusterIPNone, ports, owner, true)
 }
 
-func createService(kubecli kubernetes.Interface, svcName, clusterName, ns, clusterIP string, ports []v1.ServicePort, owner metav1.OwnerReference) error {
-	svc := newEtcdServiceManifest(svcName, clusterName, clusterIP, ports)
+func createService(kubecli kubernetes.Interface, svcName, clusterName, ns, clusterIP string, ports []v1.ServicePort, owner metav1.OwnerReference, publishNotReadyAddresses bool) error {
+	svc := newEtcdServiceManifest(svcName, clusterName, clusterIP, ports, publishNotReadyAddresses)
 	addOwnerRefToObject(svc.GetObjectMeta(), owner)
 	_, err := kubecli.CoreV1().Services(ns).Create(svc)
 	if err != nil && !apierrors.IsAlreadyExists(err) {
@@ -225,20 +226,21 @@ func CreateAndWaitPod(kubecli kubernetes.Interface, ns string, pod *v1.Pod, time
 	return retPod, nil
 }
 
-func newEtcdServiceManifest(svcName, clusterName, clusterIP string, ports []v1.ServicePort) *v1.Service {
+func newEtcdServiceManifest(svcName, clusterName, clusterIP string, ports []v1.ServicePort, publishNotReadyAddresses bool) *v1.Service {
 	labels := LabelsForCluster(clusterName)
 	svc := &v1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   svcName,
 			Labels: labels,
 			Annotations: map[string]string{
-				TolerateUnreadyEndpointsAnnotation: "true",
+				TolerateUnreadyEndpointsAnnotation: strconv.FormatBool(publishNotReadyAddresses),
 			},
 		},
 		Spec: v1.ServiceSpec{
 			Ports:     ports,
 			Selector:  labels,
 			ClusterIP: clusterIP,
+			// PublishNotReadyAddresses: publishNotReadyAddresses, // TODO(ckoehn): Activate once TolerateUnreadyEndpointsAnnotation is deprecated.
 		},
 	}
 	return svc