feat sicedar: added readiness prober

Signed-off-by: Martin Chodur <m.chodur@seznam.cz>

CHANGELOG.md

@@ -15,6 +15,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 - [#1358](https://github.com/thanos-io/thanos/pull/1358) Added `part_size` configuration option for HTTP multipart requests minimum part size for S3 storage type
 - [#1363](https://github.com/thanos-io/thanos/pull/1363) Thanos Receive now exposes `thanos_receive_hashring_nodes` and `thanos_receive_hashring_tenants` metrics to monitor status of hash-rings
+- [TODO](https://github.com/improbable-eng/thanos/pull/) Added `/-/ready` and `/-/healthy` endpoints to Thanos sidecar.
 
 ### Changed
 

cmd/thanos/main.go

@@ -70,7 +70,7 @@ func main() {
 	tracingConfig := regCommonTracingFlags(app)
 
 	cmds := map[string]setupFunc{}
-	registerSidecar(cmds, app, "sidecar")
+	registerSidecar(cmds, app)
 	registerStore(cmds, app, "store")
 	registerQuery(cmds, app, "query")
 	registerRule(cmds, app, "rule")

cmd/thanos/sidecar.go

@@ -8,10 +8,12 @@ import (
 	"sync"
 	"time"
 
+	"github.com/thanos-io/thanos/pkg/prober"
+
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/oklog/run"
-	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
@@ -26,13 +28,14 @@ import (
 	"github.com/thanos-io/thanos/pkg/store"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 	"google.golang.org/grpc"
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"gopkg.in/alecthomas/kingpin.v2"
 )
 
 const waitForExternalLabelsTimeout = 10 * time.Minute
 
-func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name string) {
-	cmd := app.Command(name, "sidecar for Prometheus server")
+func registerSidecar(m map[string]setupFunc, app *kingpin.Application) {
+	comp := component.Sidecar
+	cmd := app.Command(comp.String(), "sidecar for Prometheus server")
 
 	grpcBindAddr, httpBindAddr, cert, key, clientCA := regCommonServerFlags(cmd)
 
@@ -54,7 +57,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri
 
 	uploadCompacted := cmd.Flag("shipper.upload-compacted", "[Experimental] If true sidecar will try to upload compacted blocks as well. Useful for migration purposes. Works only if compaction is disabled on Prometheus.").Default("false").Hidden().Bool()
 
-	m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error {
+	m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error {
 		rl := reloader.New(
 			log.With(logger, "component", "reloader"),
 			reloader.ReloadURLFromBase(*promURL),
@@ -77,6 +80,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri
 			objStoreConfig,
 			rl,
 			*uploadCompacted,
+			comp,
 		)
 	}
 }
@@ -96,6 +100,7 @@ func runSidecar(
 	objStoreConfig *pathOrContent,
 	reloader *reloader.Reloader,
 	uploadCompacted bool,
+	comp component.Component,
 ) error {
 	var m = &promMetadata{
 		promURL: promURL,
@@ -117,6 +122,12 @@ func runSidecar(
 		uploads = false
 	}
 
+	readinessProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
+	// Initiate default HTTP listener providing metrics endpoint and readiness/liveness probes.
+	if err := defaultHTTPListener(g, logger, reg, httpBindAddr, readinessProber); err != nil {
+		return errors.Wrap(err, "create readiness prober")
+	}
+
 	// Setup all the concurrent groups.
 	{
 		promUp := prometheus.NewGauge(prometheus.GaugeOpts{
@@ -148,6 +159,7 @@ func runSidecar(
 						"err", err,
 					)
 					promUp.Set(0)
+					readinessProber.SetNotReady(err)
 					return err
 				}
 
@@ -156,6 +168,7 @@ func runSidecar(
 					"external_labels", m.Labels().String(),
 				)
 				promUp.Set(1)
+				readinessProber.SetReady()
 				lastHeartbeat.Set(float64(time.Now().UnixNano()) / 1e9)
 				return nil
 			})
@@ -176,8 +189,10 @@ func runSidecar(
 				if err := m.UpdateLabels(iterCtx, logger); err != nil {
 					level.Warn(logger).Log("msg", "heartbeat failed", "err", err)
 					promUp.Set(0)
+					readinessProber.SetNotReady(err)
 				} else {
 					promUp.Set(1)
+					readinessProber.SetReady()
 					lastHeartbeat.Set(float64(time.Now().UnixNano()) / 1e9)
 				}
 
@@ -195,9 +210,7 @@ func runSidecar(
 			cancel()
 		})
 	}
-	if err := metricHTTPListenGroup(g, logger, reg, httpBindAddr); err != nil {
-		return err
-	}
+
 	{
 		l, err := net.Listen("tcp", grpcBindAddr)
 		if err != nil {

tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml

@@ -145,6 +145,14 @@ spec:
             containerPort: 10902
           - name: grpc
             containerPort: 10901
+        livenessProbe:
+            httpGet:
+              port: 10902
+              path: /-/healthy
+        readinessProbe:
+          httpGet:
+            port: 10902
+            path: /-/ready
         volumeMounts:
           - name: prometheus
             mountPath: /var/prometheus

tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml

@@ -129,6 +129,14 @@ spec:
             containerPort: 10902
           - name: grpc
             containerPort: 10901
+        livenessProbe:
+            httpGet:
+              port: 10902
+              path: /-/healthy
+        readinessProbe:
+          httpGet:
+            port: 10902
+            path: /-/ready
         volumeMounts:
           - name: prometheus
             mountPath: /var/prometheus