Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature: Add option to exclude consumer from Stream metrics #906

Merged
merged 6 commits into from
May 30, 2024
Merged

Feature: Add option to exclude consumer from Stream metrics #906

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a78079c
Feature: Add option to exclude consumer from Stream metrics
nantiferov May 18, 2024
1b2825c
Clarify exclude consumer metrics from streams
nantiferov May 20, 2024
d97b9bb
Merge branch 'oliver006:master' into feature-exclude-consumer-stream
nantiferov May 21, 2024
60ca0d6
Add TestStreamsExtractStreamMetricsExcludeConsumer to cover new option
nantiferov May 21, 2024
7ee9fed
Update Readme for new streams-exclude-consumer-metrics option
nantiferov May 25, 2024
cec04c8
Add check for excluded metrics to TestStreamsExtractStreamMetricsExcl…
nantiferov May 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ Prometheus uses file watches and all changes to the json file are applied immedi
| check-single-keys | REDIS_EXPORTER_CHECK_SINGLE_KEYS | Comma separated list of keys to export value and length/size, eg: `db3=user_count` will export key `user_count` from db `3`. db defaults to `0` if omitted. The keys specified with this flag will be looked up directly without any glob pattern matching. Use this option if you don't need glob pattern matching; it is faster than `check-keys`. |
| check-streams | REDIS_EXPORTER_CHECK_STREAMS | Comma separated list of stream-patterns to export info about streams, groups and consumers. Syntax is the same as `check-keys`. |
| check-single-streams | REDIS_EXPORTER_CHECK_SINGLE_STREAMS | Comma separated list of streams to export info about streams, groups and consumers. The streams specified with this flag will be looked up directly without any glob pattern matching. Use this option if you don't need glob pattern matching; it is faster than `check-streams`. |
| streams-exclude-consumer-metrics | REDIS_EXPORTER_STREAMS_EXCLUDE_CONSUMER_METRICS | Don't collect per consumer metrics for streams (decreases amount of metrics and cardinality). |
| check-keys-batch-size | REDIS_EXPORTER_CHECK_KEYS_BATCH_SIZE | Approximate number of keys to process in each execution. This is basically the COUNT option that will be passed into the SCAN command as part of the execution of the key or key group metrics, see [COUNT option](https://redis.io/commands/scan#the-count-option). Larger value speeds up scanning. Still Redis is a single-threaded app, huge `COUNT` can affect production environment. |
| count-keys | REDIS_EXPORTER_COUNT_KEYS | Comma separated list of patterns to count, eg: `db3=sessions:*` will count all keys with prefix `sessions:` from db `3`. db defaults to `0` if omitted. Warning: The exporter runs SCAN to count the keys. This might not perform well on large databases. |
| script | REDIS_EXPORTER_SCRIPT | Comma separated list of path(s) to Redis Lua script(s) for gathering extra metrics. |
Expand Down
75 changes: 38 additions & 37 deletions exporter/exporter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,40 +46,41 @@ type Exporter struct {
}

type Options struct {
User string
Password string
Namespace string
PasswordMap map[string]string
ConfigCommandName string
CheckKeys string
CheckSingleKeys string
CheckStreams string
CheckSingleStreams string
CheckKeysBatchSize int64
CheckKeyGroups string
MaxDistinctKeyGroups int64
CountKeys string
LuaScript map[string][]byte
ClientCertFile string
ClientKeyFile string
CaCertFile string
InclConfigMetrics bool
DisableExportingKeyValues bool
RedactConfigMetrics bool
InclSystemMetrics bool
SkipTLSVerification bool
SetClientName bool
IsTile38 bool
IsCluster bool
ExportClientList bool
ExportClientsInclPort bool
ConnectionTimeouts time.Duration
MetricsPath string
RedisMetricsOnly bool
PingOnConnect bool
RedisPwdFile string
Registry *prometheus.Registry
BuildInfo BuildInfo
User string
Password string
Namespace string
PasswordMap map[string]string
ConfigCommandName string
CheckKeys string
CheckSingleKeys string
CheckStreams string
CheckSingleStreams string
StreamsExcludeConsumerMetrics bool
CheckKeysBatchSize int64
CheckKeyGroups string
MaxDistinctKeyGroups int64
CountKeys string
LuaScript map[string][]byte
ClientCertFile string
ClientKeyFile string
CaCertFile string
InclConfigMetrics bool
DisableExportingKeyValues bool
RedactConfigMetrics bool
InclSystemMetrics bool
SkipTLSVerification bool
SetClientName bool
IsTile38 bool
IsCluster bool
ExportClientList bool
ExportClientsInclPort bool
ConnectionTimeouts time.Duration
MetricsPath string
RedisMetricsOnly bool
PingOnConnect bool
RedisPwdFile string
Registry *prometheus.Registry
BuildInfo BuildInfo
}

// NewRedisExporter returns a new exporter of Redis metrics.
Expand Down Expand Up @@ -189,10 +190,10 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) {
"rdb_last_bgsave_status": "rdb_last_bgsave_status",
"rdb_last_bgsave_time_sec": "rdb_last_bgsave_duration_sec",
"rdb_current_bgsave_time_sec": "rdb_current_bgsave_duration_sec",
"rdb_saves": "rdb_saves_total",
"rdb_saves": "rdb_saves_total",
"rdb_last_cow_size": "rdb_last_cow_size_bytes",
"rdb_last_load_keys_expired": "rdb_last_load_expired_keys",
"rdb_last_load_keys_loaded": "rdb_last_load_loaded_keys",
"rdb_last_load_keys_expired": "rdb_last_load_expired_keys",
"rdb_last_load_keys_loaded": "rdb_last_load_loaded_keys",
"aof_enabled": "aof_enabled",
"aof_rewrite_in_progress": "aof_rewrite_in_progress",
"aof_rewrite_scheduled": "aof_rewrite_scheduled",
Expand Down
8 changes: 5 additions & 3 deletions exporter/streams.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,9 +205,11 @@ func (e *Exporter) extractStreamMetrics(ch chan<- prometheus.Metric, c redis.Con
e.registerConstMetricGauge(ch, "stream_group_last_delivered_id", parseStreamItemId(g.LastDeliveredId), dbLabel, k.key, g.Name)
e.registerConstMetricGauge(ch, "stream_group_entries_read", float64(g.EntriesRead), dbLabel, k.key, g.Name)
e.registerConstMetricGauge(ch, "stream_group_lag", float64(g.Lag), dbLabel, k.key, g.Name)
for _, c := range g.StreamGroupConsumersInfo {
e.registerConstMetricGauge(ch, "stream_group_consumer_messages_pending", float64(c.Pending), dbLabel, k.key, g.Name, c.Name)
e.registerConstMetricGauge(ch, "stream_group_consumer_idle_seconds", float64(c.Idle)/1e3, dbLabel, k.key, g.Name, c.Name)
if !e.options.StreamsExcludeConsumerMetrics {
for _, c := range g.StreamGroupConsumersInfo {
e.registerConstMetricGauge(ch, "stream_group_consumer_messages_pending", float64(c.Pending), dbLabel, k.key, g.Name, c.Name)
e.registerConstMetricGauge(ch, "stream_group_consumer_idle_seconds", float64(c.Idle)/1e3, dbLabel, k.key, g.Name, c.Name)
}
}
}
}
Expand Down
74 changes: 74 additions & 0 deletions exporter/streams_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,3 +544,77 @@ func TestStreamsExtractStreamMetrics(t *testing.T) {

}
}

func TestStreamsExtractStreamMetricsExcludeConsumer(t *testing.T) {
oliver006 marked this conversation as resolved.
Show resolved Hide resolved
if os.Getenv("TEST_REDIS_URI") == "" {
t.Skipf("TEST_REDIS_URI not set - skipping")
}
addr := os.Getenv("TEST_REDIS_URI")
e, _ := NewRedisExporter(
addr,
Options{Namespace: "test", CheckSingleStreams: dbNumStrFull + "=" + TestStreamName, StreamsExcludeConsumerMetrics: true},
)
c, err := redis.DialURL(addr)
if err != nil {
t.Fatalf("Couldn't connect to %#v: %#v", addr, err)
}

setupDBKeys(t, addr)
defer deleteKeysFromDB(t, addr)

chM := make(chan prometheus.Metric)
go func() {
e.extractStreamMetrics(chM, c)
close(chM)
}()
want := map[string]bool{
"stream_length": false,
"stream_radix_tree_keys": false,
"stream_radix_tree_nodes": false,
"stream_last_generated_id": false,
"stream_groups": false,
"stream_max_deleted_entry_id": false,
"stream_first_entry_id": false,
"stream_last_entry_id": false,
"stream_group_consumers": false,
"stream_group_messages_pending": false,
"stream_group_last_delivered_id": false,
"stream_group_entries_read": false,
"stream_group_lag": false,
}

dont_want := map[string]bool{
"stream_group_consumer_messages_pending": false,
"stream_group_consumer_idle_seconds": false,
}

for m := range chM {
for k := range want {
log.Debugf("metric: %s", m.Desc().String())
log.Debugf("want: %s", k)

if strings.Contains(m.Desc().String(), k) {
want[k] = true
}
}
for k := range dont_want {
log.Debugf("metric: %s", m.Desc().String())
log.Debugf("don't want: %s", k)

if strings.Contains(m.Desc().String(), k) {
dont_want[k] = true
}
}
}

for k, found := range want {
if !found {
t.Errorf("didn't find %s metric, which should be collected", k)
}
}
for k, found := range dont_want {
if found {
t.Errorf("found %s metric, which shouldn't be collected", k)
}
}
}
Loading
Loading