From bd5dfa22716496b8ea5490cc1e948ab7c1234e44 Mon Sep 17 00:00:00 2001 From: Krishna Teja Puttagunta Date: Tue, 5 Sep 2023 21:43:40 -0700 Subject: [PATCH] Making GCInterval Configurable Signed-off-by: Krishna Teja Puttagunta --- CHANGELOG.md | 1 + docs/configuration/config-file-reference.md | 4 ++++ pkg/alertmanager/alertmanager.go | 4 ++-- pkg/alertmanager/alertmanager_test.go | 1 + pkg/alertmanager/multitenant.go | 8 +++++--- pkg/alertmanager/multitenant_test.go | 2 +- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98981dcb5e..5918e4df52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## master / unreleased +* [FEATURE] Added the flag `-alertmanager.alerts-gc-interval` to configure alert manager alerts Garbage collection interval. #5550 * [FEATURE] Ruler: Add support for Limit field on RuleGroup. #5528 * [FEATURE] AlertManager: Add support for Webex, Discord and Telegram Receiver. #5493 * [FEATURE] Ingester: added `-admin-limit-message` to customize the message contained in limit errors.#5460 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index d2e776d462..ff64668a33 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -433,6 +433,10 @@ cluster: # CLI flag: -alertmanager.api-concurrency [api_concurrency: | default = 0] +# Alertmanager alerts Garbage collection interval. +# CLI flag: -alertmanager.alerts-gc-interval +[gc_interval: | default = 30m] + alertmanager_client: # Timeout for downstream alertmanagers. # CLI flag: -alertmanager.alertmanager-client.remote-timeout diff --git a/pkg/alertmanager/alertmanager.go b/pkg/alertmanager/alertmanager.go index 497bff5174..d67f26eefd 100644 --- a/pkg/alertmanager/alertmanager.go +++ b/pkg/alertmanager/alertmanager.go @@ -85,6 +85,7 @@ type Config struct { Store alertstore.AlertStore PersisterConfig PersisterConfig APIConcurrency int + GCInterval time.Duration } // An Alertmanager manages the alerts for one user. @@ -254,8 +255,7 @@ func New(cfg *Config, reg *prometheus.Registry) (*Alertmanager, error) { if am.cfg.Limits != nil { callback = newAlertsLimiter(am.cfg.UserID, am.cfg.Limits, reg) } - - am.alerts, err = mem.NewAlerts(context.Background(), am.marker, 30*time.Minute, callback, am.logger, am.registry) + am.alerts, err = mem.NewAlerts(context.Background(), am.marker, am.cfg.GCInterval, callback, am.logger, am.registry) if err != nil { return nil, fmt.Errorf("failed to create alerts: %v", err) } diff --git a/pkg/alertmanager/alertmanager_test.go b/pkg/alertmanager/alertmanager_test.go index 52aa186c39..6859fb5086 100644 --- a/pkg/alertmanager/alertmanager_test.go +++ b/pkg/alertmanager/alertmanager_test.go @@ -46,6 +46,7 @@ func createAlertmanagerAndSendAlerts(t *testing.T, alertGroups, groupsLimit, exp TenantDataDir: t.TempDir(), ExternalURL: &url.URL{Path: "/am"}, ShardingEnabled: false, + GCInterval: 30 * time.Minute, }, reg) require.NoError(t, err) defer am.StopAndWait() diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index 3eca5a1e2b..8ce007abc1 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -78,8 +78,9 @@ type MultitenantAlertmanagerConfig struct { Cluster ClusterConfig `yaml:"cluster"` - EnableAPI bool `yaml:"enable_api"` - APIConcurrency int `yaml:"api_concurrency"` + EnableAPI bool `yaml:"enable_api"` + APIConcurrency int `yaml:"api_concurrency"` + GCInterval time.Duration `yaml:"gc_interval"` // For distributor. AlertmanagerClient ClientConfig `yaml:"alertmanager_client"` @@ -119,7 +120,7 @@ func (cfg *MultitenantAlertmanagerConfig) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.EnableAPI, "experimental.alertmanager.enable-api", false, "Enable the experimental alertmanager config api.") f.IntVar(&cfg.APIConcurrency, "alertmanager.api-concurrency", 0, "Maximum number of concurrent GET API requests before returning an error.") - + f.DurationVar(&cfg.GCInterval, "alertmanager.alerts-gc-interval", 30*time.Minute, "Alertmanager alerts Garbage collection interval.") f.BoolVar(&cfg.ShardingEnabled, "alertmanager.sharding-enabled", false, "Shard tenants across multiple alertmanager instances.") f.Var(&cfg.EnabledTenants, "alertmanager.enabled-tenants", "Comma separated list of tenants whose alerts this alertmanager can process. If specified, only these tenants will be handled by alertmanager, otherwise this alertmanager can process alerts from all tenants.") f.Var(&cfg.DisabledTenants, "alertmanager.disabled-tenants", "Comma separated list of tenants whose alerts this alertmanager cannot process. If specified, a alertmanager that would normally pick the specified tenant(s) for processing will ignore them instead.") @@ -969,6 +970,7 @@ func (am *MultitenantAlertmanager) newAlertmanager(userID string, amConfig *amco PersisterConfig: am.cfg.Persister, Limits: am.limits, APIConcurrency: am.cfg.APIConcurrency, + GCInterval: am.cfg.GCInterval, }, reg) if err != nil { return nil, fmt.Errorf("unable to start Alertmanager for user %v: %v", userID, err) diff --git a/pkg/alertmanager/multitenant_test.go b/pkg/alertmanager/multitenant_test.go index 7c17b80838..38057bb144 100644 --- a/pkg/alertmanager/multitenant_test.go +++ b/pkg/alertmanager/multitenant_test.go @@ -83,7 +83,7 @@ func mockAlertmanagerConfig(t *testing.T) *MultitenantAlertmanagerConfig { cfg.ShardingRing.InstanceAddr = "127.0.0.1" cfg.PollInterval = time.Minute cfg.ShardingRing.FinalSleep = 0 - + cfg.GCInterval = 30 * time.Minute return cfg }