From ecd5fda470d67ea44865199957a1b164173c5dbc Mon Sep 17 00:00:00 2001 From: Daniel Deluiggi Date: Mon, 12 Aug 2024 16:10:02 -0700 Subject: [PATCH] Using highest value for ruler delay Signed-off-by: Daniel Deluiggi --- CHANGELOG.md | 2 +- pkg/util/validation/limits.go | 12 ++++++++++-- pkg/util/validation/limits_test.go | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b29fa5076..71a49d23f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ * [CHANGE] Querier: Deprecate and enable by default `querier.ingester-metadata-streaming` flag. #6147 * [CHANGE] QueryFrontend/QueryScheduler: Deprecate `-querier.max-outstanding-requests-per-tenant` and `-query-scheduler.max-outstanding-requests-per-tenant` flags. Use frontend.max-outstanding-requests-per-tenant instead. #6146 * [CHANGE] Ingesters: Enable 'snappy-block' compression on ingester clients by default. #6148 -* [CHANGE] Ruler: Mark `evaluation-delay-duration` flag as deprecated. #6149 +* [CHANGE] Ruler: Scheduling `evaluation-delay-duration` to be deprecated. Use the highest value between `evaluation-delay-duration` and `ruler.query_offset` #6149 * [FEATURE] Ingester/Distributor: Experimental: Enable native histogram ingestion via `-blocks-storage.tsdb.enable-native-histograms` flag. #5986 #6010 #6020 * [FEATURE] Querier: Enable querying native histogram chunks. #5944 #6031 * [FEATURE] Query Frontend: Support native histogram in query frontend response. #5996 #6043 diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index cf1ef481fa..4c120d8f58 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "flag" + "github.com/go-kit/log/level" "math" "regexp" "strings" @@ -174,6 +175,7 @@ type Limits struct { QueryRejection QueryRejection `yaml:"query_rejection" json:"query_rejection" doc:"nocli|description=Configuration for query rejection."` // Ruler defaults and limits. + RulerEvaluationDelay model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"` RulerTenantShardSize int `yaml:"ruler_tenant_shard_size" json:"ruler_tenant_shard_size"` RulerMaxRulesPerRuleGroup int `yaml:"ruler_max_rules_per_rule_group" json:"ruler_max_rules_per_rule_group"` RulerMaxRuleGroupsPerTenant int `yaml:"ruler_max_rule_groups_per_tenant" json:"ruler_max_rule_groups_per_tenant"` @@ -212,7 +214,6 @@ type Limits struct { // RegisterFlags adds the flags required to config this to the given FlagSet func (l *Limits) RegisterFlags(f *flag.FlagSet) { flagext.DeprecatedFlag(f, "ingester.max-series-per-query", "Deprecated: The maximum number of series for which a query can fetch samples from each ingester. This limit is enforced only in the ingesters (when querying samples not flushed to the storage yet) and it's a per-instance limit. This limit is ignored when running the Cortex blocks storage. When running Cortex with blocks storage use -querier.max-fetched-series-per-query limit instead.", util_log.Logger) - flagext.DeprecatedFlag(f, "ruler.evaluation-delay-duration", "Deprecated: Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex. This limit was replaced by ruler.query-offset", util_log.Logger) f.IntVar(&l.IngestionTenantShardSize, "distributor.ingestion-tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used. Must be set both on ingesters and distributors. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") f.Float64Var(&l.IngestionRate, "distributor.ingestion-rate-limit", 25000, "Per-user ingestion rate limit in samples per second.") @@ -265,6 +266,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxOutstandingPerTenant, "frontend.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per request queue (either query frontend or query scheduler); requests beyond this error with HTTP 429.") + f.Var(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", "Deprecated(use ruler.query-offset instead) and will be removed in v1.19.0: Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.") f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") f.IntVar(&l.RulerMaxRulesPerRuleGroup, "ruler.max-rules-per-rule-group", 0, "Maximum number of rules per rule group per-tenant. 0 to disable.") f.IntVar(&l.RulerMaxRuleGroupsPerTenant, "ruler.max-rule-groups-per-tenant", 0, "Maximum number of rule groups per-tenant. 0 to disable.") @@ -789,7 +791,13 @@ func (o *Overrides) RulerMaxRuleGroupsPerTenant(userID string) int { // RulerQueryOffset returns the rule query offset for a given user. func (o *Overrides) RulerQueryOffset(userID string) time.Duration { - return time.Duration(o.GetOverridesForUser(userID).RulerQueryOffset) + ruleOffset := time.Duration(o.GetOverridesForUser(userID).RulerQueryOffset) + evaluationDelay := time.Duration(o.GetOverridesForUser(userID).RulerEvaluationDelay) + if evaluationDelay > ruleOffset { + level.Warn(util_log.Logger).Log("msg", "ruler.query-offset was overridden by highest value in [Deprecated]ruler.evaluation-delay-duration", "ruler.query-offset", ruleOffset, "ruler.evaluation-delay-duration", evaluationDelay) + return evaluationDelay + } + return ruleOffset } // StoreGatewayTenantShardSize returns the store-gateway shard size for a given user. diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index 05807f63c9..997988ada9 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -764,3 +764,21 @@ func TestCompileQueryPriorityRegex(t *testing.T) { require.NoError(t, err) require.Nil(t, l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) } + +func TestEvaluationDelayHigherThanRulerQueryOffset(t *testing.T) { + tenant := "tenant" + evaluationDelay := time.Duration(10) + tenantLimits := map[string]*Limits{ + tenant: { + RulerQueryOffset: 5, + RulerEvaluationDelay: model.Duration(evaluationDelay), + }, + } + + defaults := Limits{} + ov, err := NewOverrides(defaults, newMockTenantLimits(tenantLimits)) + require.NoError(t, err) + + rulerQueryOffset := ov.RulerQueryOffset(tenant) + assert.Equal(t, evaluationDelay, rulerQueryOffset) +}