From 8b2a4bd9294af1574e8188edc7a56578ea9c0102 Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Wed, 5 Dec 2018 12:54:26 -0600 Subject: [PATCH] Add health check Signed-off-by: Steven Sheehy --- .gitignore | 1 + Makefile | 8 +-- .../templates/daemonset.yaml | 7 +++ chart/kube-keepalived-vip/values.yaml.tmpl | 3 ++ pkg/cmd/main.go | 8 ++- pkg/controller/keepalived.go | 54 +++++++++++++++++-- pkg/controller/main.go | 26 ++++++++- rootfs/keepalived-check.sh | 9 ++++ rootfs/keepalived.tmpl | 2 + vip-daemonset-proxy.yaml | 6 +++ vip-daemonset.yaml | 8 ++- 11 files changed, 123 insertions(+), 9 deletions(-) create mode 100755 rootfs/keepalived-check.sh diff --git a/.gitignore b/.gitignore index aa357392..c72a8641 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ chart/kube-keepalived-vip/Chart.yaml chart/kube-keepalived-vip/values.yaml # Ignore backup files *~ +*.bak .DS_Store .idea/ diff --git a/Makefile b/Makefile index 1ced1e66..50008695 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: push TAG = 0.30 HAPROXY_TAG = 0.1 # Helm uses SemVer2 versioning -CHART_VERSION = 0.1.1 +CHART_VERSION = 0.2.0 PREFIX = aledbf/kube-keepalived-vip BUILD_IMAGE = build-keepalived PKG = github.com/aledbf/kube-keepalived-vip @@ -35,8 +35,9 @@ chart: chart/kube-keepalived-vip-$(CHART_VERSION).tgz .PHONY: chart-subst chart-subst: chart/kube-keepalived-vip/Chart.yaml.tmpl chart/kube-keepalived-vip/values.yaml.tmpl for file in Chart.yaml values.yaml; do cp -f "chart/kube-keepalived-vip/$$file.tmpl" "chart/kube-keepalived-vip/$$file"; done - sed -i -e 's|%%TAG%%|$(TAG)|g' -e 's|%%HAPROXY_TAG%%|$(HAPROXY_TAG)|g' chart/kube-keepalived-vip/values.yaml - sed -i -e 's|%%CHART_VERSION%%|$(CHART_VERSION)|g' chart/kube-keepalived-vip/Chart.yaml + sed -i'.bak' -e 's|%%TAG%%|$(TAG)|g' -e 's|%%HAPROXY_TAG%%|$(HAPROXY_TAG)|g' chart/kube-keepalived-vip/values.yaml + sed -i'.bak' -e 's|%%CHART_VERSION%%|$(CHART_VERSION)|g' chart/kube-keepalived-vip/Chart.yaml + rm -f chart/kube-keepalived-vip/*.bak # Requires helm chart/kube-keepalived-vip-$(CHART_VERSION).tgz: chart-subst $(shell which helm) $(shell find chart/kube-keepalived-vip -type f) @@ -74,3 +75,4 @@ dep-ensure: dep ensure -v dep prune -v find vendor -name '*_test.go' -delete + diff --git a/chart/kube-keepalived-vip/templates/daemonset.yaml b/chart/kube-keepalived-vip/templates/daemonset.yaml index 85398a5c..db3f63f0 100644 --- a/chart/kube-keepalived-vip/templates/daemonset.yaml +++ b/chart/kube-keepalived-vip/templates/daemonset.yaml @@ -37,6 +37,12 @@ spec: - name: {{ template "kube-keepalived-vip.name" . }}-{{ .Values.keepalived.name }} image: "{{ .Values.keepalived.image.repository }}:{{ .Values.keepalived.image.tag }}" imagePullPolicy: "{{ .Values.keepalived.image.pullPolicy }}" + livenessProbe: + httpGet: + path: /health + port: {{ .Values.httpPort }} + initialDelaySeconds: 15 + timeoutSeconds: 3 securityContext: privileged: true volumeMounts: @@ -64,6 +70,7 @@ spec: - --use-unicast={{ .Values.keepalived.useUnicast }} - --vrid={{ .Values.keepalived.vrid }} - --logtostderr + - --http-port={{ .Values.httpPort }} {{- if .Values.haproxy.enabled }} - --proxy-protocol-mode=true {{- end }} diff --git a/chart/kube-keepalived-vip/values.yaml.tmpl b/chart/kube-keepalived-vip/values.yaml.tmpl index 978bb69d..05470596 100644 --- a/chart/kube-keepalived-vip/values.yaml.tmpl +++ b/chart/kube-keepalived-vip/values.yaml.tmpl @@ -30,6 +30,9 @@ haproxy: # Resource allocations for the HAProxy container resources: {} +# Health check port +httpPort: 8080 + fullnameOverride: "" nameOverride: "" diff --git a/pkg/cmd/main.go b/pkg/cmd/main.go index 38c5a57d..24f36eb8 100644 --- a/pkg/cmd/main.go +++ b/pkg/cmd/main.go @@ -79,6 +79,8 @@ var ( iface = flags.String("iface", "", `network interface to listen on. If undefined, the nodes default interface will be used instead`) + + httpPort = flags.Int("http-port", 8080, `The HTTP port to use for health checks`) ) func main() { @@ -95,6 +97,10 @@ func main() { glog.Fatalf("Please specify --services-configmap") } + if *httpPort < 0 || *httpPort > 65535 { + glog.Fatalf("Invalid HTTP port %d, only values between 0 and 65535 are allowed.", httpPort) + } + if *vrid < 0 || *vrid > 255 { glog.Fatalf("Error using VRID %d, only values between 0 and 255 are allowed.", vrid) } @@ -131,7 +137,7 @@ func main() { } glog.Info("starting LVS configuration") - ipvsc := controller.NewIPVSController(kubeClient, *watchNamespace, *useUnicast, *configMapName, *vrid, *proxyMode, *iface) + ipvsc := controller.NewIPVSController(kubeClient, *watchNamespace, *useUnicast, *configMapName, *vrid, *proxyMode, *iface, *httpPort) ipvsc.Start() } diff --git a/pkg/controller/keepalived.go b/pkg/controller/keepalived.go index 5acfadd9..771efde2 100644 --- a/pkg/controller/keepalived.go +++ b/pkg/controller/keepalived.go @@ -17,10 +17,13 @@ limitations under the License. package controller import ( + "bytes" "encoding/json" "fmt" + "io/ioutil" "os" "os/exec" + "strings" "syscall" "text/template" @@ -31,9 +34,10 @@ import ( ) const ( - iptablesChain = "KUBE-KEEPALIVED-VIP" - keepalivedCfg = "/etc/keepalived/keepalived.conf" - haproxyCfg = "/etc/haproxy/haproxy.cfg" + iptablesChain = "KUBE-KEEPALIVED-VIP" + keepalivedCfg = "/etc/keepalived/keepalived.conf" + haproxyCfg = "/etc/haproxy/haproxy.cfg" + keepalivedState = "/var/run/keepalived.state" ) var ( @@ -175,6 +179,50 @@ func (k *keepalived) Reload() error { return nil } +// Whether keepalived child process is currently running +func (k *keepalived) Healthy() error { + b, err := ioutil.ReadFile(keepalivedState) + if err != nil { + return err + } + + master := false + state := strings.TrimSpace(string(b)) + if strings.Contains(state, "MASTER") { + master = true + } + + var out bytes.Buffer + cmd := exec.Command("ip", "-brief", "address", "show", k.iface, "up") + cmd.Stderr = os.Stderr + cmd.Stdout = &out + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + Pgid: 0, + } + + err = cmd.Run() + if err != nil { + return err + } + + ips := out.String() + glog.V(3).Infof("Status of %s interface: %s", state, ips) + + for _, vip := range k.vips { + containsVip := strings.Contains(ips, fmt.Sprintf(" %s/32 ", vip)) + + if master && !containsVip { + return fmt.Errorf("Missing VIP %s on %s", vip, state) + } else if !master && containsVip { + return fmt.Errorf("%s should not contain VIP %s", state, vip) + } + } + + // All checks successful + return nil +} + func (k *keepalived) Cleanup() { glog.Infof("Cleanup: %s", k.vips) for _, vip := range k.vips { diff --git a/pkg/controller/main.go b/pkg/controller/main.go index 12952756..45eb434e 100644 --- a/pkg/controller/main.go +++ b/pkg/controller/main.go @@ -21,6 +21,7 @@ import ( "encoding/hex" "fmt" "io" + "net/http" "os" "os/signal" "reflect" @@ -125,6 +126,8 @@ type ipvsControllerController struct { configMapName string configMapResourceVersion string + httpPort int + ruMD5 string // stopLock is used to enforce only a single call to Stop is active. @@ -308,6 +311,14 @@ func (ipvsc *ipvsControllerController) Start() { runtime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) } + go func() { + glog.Infof("Starting HTTP server on port %d", ipvsc.httpPort) + err := http.ListenAndServe(fmt.Sprintf(":%d", ipvsc.httpPort), nil) + if err != nil { + glog.Error(err.Error()) + } + }() + glog.Info("starting keepalived to announce VIPs") ipvsc.keepalived.Start() } @@ -347,11 +358,12 @@ func (ipvsc *ipvsControllerController) Stop() error { } // NewIPVSController creates a new controller from the given config. -func NewIPVSController(kubeClient *kubernetes.Clientset, namespace string, useUnicast bool, configMapName string, vrid int, proxyMode bool, iface string) *ipvsControllerController { +func NewIPVSController(kubeClient *kubernetes.Clientset, namespace string, useUnicast bool, configMapName string, vrid int, proxyMode bool, iface string, httpPort int) *ipvsControllerController { ipvsc := ipvsControllerController{ client: kubeClient, reloadRateLimiter: flowcontrol.NewTokenBucketRateLimiter(0.5, 1), configMapName: configMapName, + httpPort: httpPort, stopCh: make(chan struct{}), } @@ -441,6 +453,18 @@ func NewIPVSController(kubeClient *kubernetes.Clientset, namespace string, useUn cache.NewListWatchFromClient(ipvsc.client.CoreV1().RESTClient(), "configmaps", namespace, fields.Everything()), &apiv1.ConfigMap{}, resyncPeriod, mapEventHandler) + http.HandleFunc("/health", func(rw http.ResponseWriter, req *http.Request) { + err := ipvsc.keepalived.Healthy() + if err != nil { + glog.Errorf("Health check unsuccessful: %v", err) + http.Error(rw, fmt.Sprintf("keepalived not healthy: %v", err), 500) + return + } + + glog.V(3).Info("Health check successful") + fmt.Fprint(rw, "OK") + }) + return &ipvsc } diff --git a/rootfs/keepalived-check.sh b/rootfs/keepalived-check.sh new file mode 100755 index 00000000..5a739f35 --- /dev/null +++ b/rootfs/keepalived-check.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +TYPE="$1" +NAME="$2" +STATE="$3" + +echo -n "${STATE}" > /var/run/keepalived.state +exit 0 + diff --git a/rootfs/keepalived.tmpl b/rootfs/keepalived.tmpl index bc3a610c..90e17f83 100644 --- a/rootfs/keepalived.tmpl +++ b/rootfs/keepalived.tmpl @@ -40,6 +40,8 @@ vrrp_instance vips { {{ . }}{{ end }} } + notify /keepalived-check.sh + {{ if .proxyMode }} # In proxy mode there is no need to create virtual servers track_script { diff --git a/vip-daemonset-proxy.yaml b/vip-daemonset-proxy.yaml index 2ac407a5..597aac9d 100644 --- a/vip-daemonset-proxy.yaml +++ b/vip-daemonset-proxy.yaml @@ -24,6 +24,12 @@ spec: mountPath: /etc/haproxy - image: aledbf/kube-keepalived-vip:0.30 name: kube-keepalived-vip + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 3 securityContext: privileged: true volumeMounts: diff --git a/vip-daemonset.yaml b/vip-daemonset.yaml index a820fed6..54351328 100644 --- a/vip-daemonset.yaml +++ b/vip-daemonset.yaml @@ -12,7 +12,13 @@ spec: containers: - image: aledbf/kube-keepalived-vip:0.30 name: kube-keepalived-vip - imagePullPolicy: Always + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 3 securityContext: privileged: true volumeMounts: