From 64cc727fc08dbe065ba05c891de2f787089ceeb7 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Wed, 31 Jul 2019 16:31:12 +0200 Subject: [PATCH 1/4] Expose hashring metrics --- pkg/receive/config.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/pkg/receive/config.go b/pkg/receive/config.go index 7fcd2c7c5b..6908d110d4 100644 --- a/pkg/receive/config.go +++ b/pkg/receive/config.go @@ -33,9 +33,11 @@ type ConfigWatcher struct { logger log.Logger watcher *fsnotify.Watcher - changesCounter prometheus.Counter - errorCounter prometheus.Counter - refreshCounter prometheus.Counter + changesCounter prometheus.Counter + errorCounter prometheus.Counter + refreshCounter prometheus.Counter + hashringNodesGauge *prometheus.GaugeVec + hashringTenantsGauge *prometheus.GaugeVec // last is the last known configuration. last []HashringConfig @@ -75,6 +77,18 @@ func NewConfigWatcher(logger log.Logger, r prometheus.Registerer, path string, i Name: "thanos_receive_hashrings_file_refreshes_total", Help: "The number of refreshes of the hashrings configuration file.", }), + hashringNodesGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "thanos_receive_hashrings_nodes", + Help: "The number of nodes per hashring.", + }, + []string{"name"}), + hashringTenantsGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "thanos_receive_hashrings_tenants", + Help: "The number of tenants per hashring.", + }, + []string{"name"}), } if r != nil { @@ -172,6 +186,11 @@ func (cw *ConfigWatcher) refresh(ctx context.Context) { // Save the last known configuration. cw.last = config + for _, c := range config { + cw.hashringNodesGauge.WithLabelValues(c.Hashring).Set(float64(len(c.Endpoints))) + cw.hashringTenantsGauge.WithLabelValues(c.Hashring).Set(float64(len(c.Tenants))) + } + select { case <-ctx.Done(): return From 384f18e314a4b6e1566b408601a029b6f6684b94 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Wed, 31 Jul 2019 16:44:45 +0200 Subject: [PATCH 2/4] Fix metric names --- pkg/receive/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/receive/config.go b/pkg/receive/config.go index 6908d110d4..0b3678b5b0 100644 --- a/pkg/receive/config.go +++ b/pkg/receive/config.go @@ -79,13 +79,13 @@ func NewConfigWatcher(logger log.Logger, r prometheus.Registerer, path string, i }), hashringNodesGauge: prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "thanos_receive_hashrings_nodes", + Name: "thanos_receive_hashring_nodes", Help: "The number of nodes per hashring.", }, []string{"name"}), hashringTenantsGauge: prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "thanos_receive_hashrings_tenants", + Name: "thanos_receive_hashring_tenants", Help: "The number of tenants per hashring.", }, []string{"name"}), From 6ab03e19c6f300df024380ea0a3a8223115009df Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Wed, 31 Jul 2019 16:51:51 +0200 Subject: [PATCH 3/4] Update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c485a31c4b..1f30e91fff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1358](https://github.com/thanos-io/thanos/pull/1358) Added `part_size` configuration option for HTTP multipart requests minimum part size for S3 storage type +- [#1363](https://github.com/thanos-io/thanos/pull/1363) Thanos Receive now exposes `thanos_receive_hashring_nodes` and `thanos_receive_hashring_tenants` metrics to monitor status of hash-rings + ### Changed - [#1338](https://github.com/thanos-io/thanos/pull/1338) Querier still warns on store API duplicate, but allows a single one from duplicated set. This is gracefully warn about the problematic logic and not disrupt immediately. @@ -65,7 +67,7 @@ The other `type` you can use is `JAEGER` now. The `config` keys and values are J ### Changed -- [#1284](https://github.com/thanos-io/thanos/pull/1284) Add support for multiple label-sets in Info gRPC service. +- [#1284](https://github.com/thanos-io/thanos/pull/1284) Add support for multiple label-sets in Info gRPC service. This deprecates the single `Labels` slice of the `InfoResponse`, in a future release backward compatible handling for the single set of Labels will be removed. Upgrading to v0.6.0 or higher is advised. *breaking* If you run have duplicate queries in your Querier configuration with hierarchical federation of multiple Queries this PR makes Thanos Querier to detect this case and block all duplicates. Refer to 0.6.1 which at least allows for single replica to work. From 4c28e0f712dfb61ac3ad3be8c48c3df27ac58dbe Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Mon, 5 Aug 2019 14:48:35 +0200 Subject: [PATCH 4/4] Register new gauges --- pkg/receive/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/receive/config.go b/pkg/receive/config.go index 0b3678b5b0..61f9f0d05b 100644 --- a/pkg/receive/config.go +++ b/pkg/receive/config.go @@ -96,6 +96,8 @@ func NewConfigWatcher(logger log.Logger, r prometheus.Registerer, path string, i c.changesCounter, c.errorCounter, c.refreshCounter, + c.hashringNodesGauge, + c.hashringTenantsGauge, ) }