Merge pull request #71 from steven-sheehy/health-check

Add health check

.gitignore

@@ -5,6 +5,7 @@ chart/kube-keepalived-vip/Chart.yaml
 chart/kube-keepalived-vip/values.yaml
 # Ignore backup files
 *~
+*.bak
 .DS_Store
 .idea/
 

Makefile

@@ -4,7 +4,7 @@ all: push
 TAG = 0.30
 HAPROXY_TAG = 0.1
 # Helm uses SemVer2 versioning
-CHART_VERSION = 0.1.1
+CHART_VERSION = 0.2.0
 PREFIX = aledbf/kube-keepalived-vip
 BUILD_IMAGE = build-keepalived
 PKG = github.com/aledbf/kube-keepalived-vip
@@ -35,8 +35,9 @@ chart: chart/kube-keepalived-vip-$(CHART_VERSION).tgz
 .PHONY: chart-subst
 chart-subst: chart/kube-keepalived-vip/Chart.yaml.tmpl chart/kube-keepalived-vip/values.yaml.tmpl
 	for file in Chart.yaml values.yaml; do cp -f "chart/kube-keepalived-vip/$$file.tmpl" "chart/kube-keepalived-vip/$$file"; done
-	sed -i -e 's|%%TAG%%|$(TAG)|g' -e 's|%%HAPROXY_TAG%%|$(HAPROXY_TAG)|g' chart/kube-keepalived-vip/values.yaml
-	sed -i -e 's|%%CHART_VERSION%%|$(CHART_VERSION)|g' chart/kube-keepalived-vip/Chart.yaml
+	sed -i'.bak' -e 's|%%TAG%%|$(TAG)|g' -e 's|%%HAPROXY_TAG%%|$(HAPROXY_TAG)|g' chart/kube-keepalived-vip/values.yaml
+	sed -i'.bak' -e 's|%%CHART_VERSION%%|$(CHART_VERSION)|g' chart/kube-keepalived-vip/Chart.yaml
+	rm -f chart/kube-keepalived-vip/*.bak
 
 # Requires helm
 chart/kube-keepalived-vip-$(CHART_VERSION).tgz: chart-subst $(shell which helm) $(shell find chart/kube-keepalived-vip -type f)
@@ -74,3 +75,4 @@ dep-ensure:
 	dep ensure -v
 	dep prune -v
 	find vendor -name '*_test.go' -delete
+

chart/kube-keepalived-vip/templates/daemonset.yaml

@@ -37,6 +37,12 @@ spec:
         - name: {{ template "kube-keepalived-vip.name" . }}-{{ .Values.keepalived.name }}
           image: "{{ .Values.keepalived.image.repository }}:{{ .Values.keepalived.image.tag }}"
           imagePullPolicy: "{{ .Values.keepalived.image.pullPolicy }}"
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: {{ .Values.httpPort }}
+            initialDelaySeconds: 15
+            timeoutSeconds: 3
           securityContext:
             privileged: true
           volumeMounts:
@@ -64,6 +70,7 @@ spec:
             - --use-unicast={{ .Values.keepalived.useUnicast }}
             - --vrid={{ .Values.keepalived.vrid }}
             - --logtostderr
+            - --http-port={{ .Values.httpPort }}
 {{- if .Values.haproxy.enabled }}
             - --proxy-protocol-mode=true
 {{- end }}

chart/kube-keepalived-vip/values.yaml.tmpl

@@ -30,6 +30,9 @@ haproxy:
   # Resource allocations for the HAProxy container
   resources: {}
 
+# Health check port
+httpPort: 8080
+
 fullnameOverride: ""
 nameOverride: ""
 

pkg/cmd/main.go

@@ -79,6 +79,8 @@ var (
 
 	iface = flags.String("iface", "", `network interface to listen on. If undefined, the nodes
                  default interface will be used instead`)
+
+	httpPort = flags.Int("http-port", 8080, `The HTTP port to use for health checks`)
 )
 
 func main() {
@@ -95,6 +97,10 @@ func main() {
 		glog.Fatalf("Please specify --services-configmap")
 	}
 
+	if *httpPort < 0 || *httpPort > 65535 {
+		glog.Fatalf("Invalid HTTP port %d, only values between 0 and 65535 are allowed.", httpPort)
+	}
+
 	if *vrid < 0 || *vrid > 255 {
 		glog.Fatalf("Error using VRID %d, only values between 0 and 255 are allowed.", vrid)
 	}
@@ -131,7 +137,7 @@ func main() {
 	}
 
 	glog.Info("starting LVS configuration")
-	ipvsc := controller.NewIPVSController(kubeClient, *watchNamespace, *useUnicast, *configMapName, *vrid, *proxyMode, *iface)
+	ipvsc := controller.NewIPVSController(kubeClient, *watchNamespace, *useUnicast, *configMapName, *vrid, *proxyMode, *iface, *httpPort)
 
 	ipvsc.Start()
 }

pkg/controller/keepalived.go

@@ -17,10 +17,13 @@ limitations under the License.
 package controller
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
+	"strings"
 	"syscall"
 	"text/template"
 
@@ -31,9 +34,10 @@ import (
 )
 
 const (
-	iptablesChain = "KUBE-KEEPALIVED-VIP"
-	keepalivedCfg = "/etc/keepalived/keepalived.conf"
-	haproxyCfg    = "/etc/haproxy/haproxy.cfg"
+	iptablesChain   = "KUBE-KEEPALIVED-VIP"
+	keepalivedCfg   = "/etc/keepalived/keepalived.conf"
+	haproxyCfg      = "/etc/haproxy/haproxy.cfg"
+	keepalivedState = "/var/run/keepalived.state"
 )
 
 var (
@@ -175,6 +179,50 @@ func (k *keepalived) Reload() error {
 	return nil
 }
 
+// Whether keepalived child process is currently running
+func (k *keepalived) Healthy() error {
+	b, err := ioutil.ReadFile(keepalivedState)
+	if err != nil {
+		return err
+	}
+
+	master := false
+	state := strings.TrimSpace(string(b))
+	if strings.Contains(state, "MASTER") {
+		master = true
+	}
+
+	var out bytes.Buffer
+	cmd := exec.Command("ip", "-brief", "address", "show", k.iface, "up")
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = &out
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+		Pgid:    0,
+	}
+
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+
+	ips := out.String()
+	glog.V(3).Infof("Status of %s interface: %s", state, ips)
+
+	for _, vip := range k.vips {
+		containsVip := strings.Contains(ips, fmt.Sprintf(" %s/32 ", vip))
+
+		if master && !containsVip {
+			return fmt.Errorf("Missing VIP %s on %s", vip, state)
+		} else if !master && containsVip {
+			return fmt.Errorf("%s should not contain VIP %s", state, vip)
+		}
+	}
+
+	// All checks successful
+        return nil
+}
+
 func (k *keepalived) Cleanup() {
 	glog.Infof("Cleanup: %s", k.vips)
 	for _, vip := range k.vips {

pkg/controller/main.go

@@ -21,6 +21,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"os/signal"
 	"reflect"
@@ -125,6 +126,8 @@ type ipvsControllerController struct {
 	configMapName string
 	configMapResourceVersion string
 
+	httpPort int
+
 	ruMD5 string
 
 	// stopLock is used to enforce only a single call to Stop is active.
@@ -308,6 +311,14 @@ func (ipvsc *ipvsControllerController) Start() {
 		runtime.HandleError(fmt.Errorf("timed out waiting for caches to sync"))
 	}
 
+	go func() {
+		glog.Infof("Starting HTTP server on port %d", ipvsc.httpPort)
+		err := http.ListenAndServe(fmt.Sprintf(":%d", ipvsc.httpPort), nil)
+		if err != nil {
+			glog.Error(err.Error())
+		}
+	}()
+
 	glog.Info("starting keepalived to announce VIPs")
 	ipvsc.keepalived.Start()
 }
@@ -347,11 +358,12 @@ func (ipvsc *ipvsControllerController) Stop() error {
 }
 
 // NewIPVSController creates a new controller from the given config.
-func NewIPVSController(kubeClient *kubernetes.Clientset, namespace string, useUnicast bool, configMapName string, vrid int, proxyMode bool, iface string) *ipvsControllerController {
+func NewIPVSController(kubeClient *kubernetes.Clientset, namespace string, useUnicast bool, configMapName string, vrid int, proxyMode bool, iface string, httpPort int) *ipvsControllerController {
 	ipvsc := ipvsControllerController{
 		client:            kubeClient,
 		reloadRateLimiter: flowcontrol.NewTokenBucketRateLimiter(0.5, 1),
 		configMapName:     configMapName,
+		httpPort:          httpPort,
 		stopCh:            make(chan struct{}),
 	}
 
@@ -441,6 +453,18 @@ func NewIPVSController(kubeClient *kubernetes.Clientset, namespace string, useUn
 		cache.NewListWatchFromClient(ipvsc.client.CoreV1().RESTClient(), "configmaps", namespace, fields.Everything()),
 		&apiv1.ConfigMap{}, resyncPeriod, mapEventHandler)
 
+	http.HandleFunc("/health", func(rw http.ResponseWriter, req *http.Request) {
+		err := ipvsc.keepalived.Healthy()
+		if err != nil {
+			glog.Errorf("Health check unsuccessful: %v", err)
+			http.Error(rw, fmt.Sprintf("keepalived not healthy: %v", err), 500)
+			return
+		}
+
+		glog.V(3).Info("Health check successful")
+		fmt.Fprint(rw, "OK")
+	})
+
 	return &ipvsc
 }
 

rootfs/keepalived-check.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+TYPE="$1"
+NAME="$2"
+STATE="$3"
+
+echo -n "${STATE}" > /var/run/keepalived.state
+exit 0
+

rootfs/keepalived.tmpl

@@ -40,6 +40,8 @@ vrrp_instance vips {
     {{ . }}{{ end }}
   }
 
+  notify /keepalived-check.sh
+
 {{ if .proxyMode }}
   # In proxy mode there is no need to create virtual servers
   track_script {

vip-daemonset-proxy.yaml

@@ -24,6 +24,12 @@ spec:
               mountPath: /etc/haproxy
         - image: aledbf/kube-keepalived-vip:0.30
           name: kube-keepalived-vip
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            initialDelaySeconds: 15
+            timeoutSeconds: 3
           securityContext:
             privileged: true
           volumeMounts:

vip-daemonset.yaml

@@ -12,7 +12,13 @@ spec:
       containers:
         - image: aledbf/kube-keepalived-vip:0.30
           name: kube-keepalived-vip
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            initialDelaySeconds: 15
+            timeoutSeconds: 3
           securityContext:
             privileged: true
           volumeMounts: