From 820170a2783323f89a4a4ceaf09a4331b486337b Mon Sep 17 00:00:00 2001 From: Mustafain Ali Khan Date: Fri, 12 Jul 2024 12:55:09 -0700 Subject: [PATCH] Use per-tenant limit for global query offset Signed-off-by: Mustafain Ali Khan --- CHANGELOG.md | 1 + docs/configuration/config-file-reference.md | 8 +- pkg/ruler/compat.go | 3 +- pkg/ruler/ruler.go | 6 +- pkg/ruler/ruler_test.go | 11 ++- pkg/ruler/rulespb/compat.go | 10 ++- pkg/ruler/rulespb/compat_test.go | 2 +- pkg/ruler/rulespb/rules.pb.go | 93 ++++++++++++--------- pkg/ruler/rulespb/rules.proto | 2 +- pkg/ruler/store_mock_test.go | 3 +- pkg/util/validation/limits.go | 8 +- 11 files changed, 89 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb01005e0d3..9042c8be3ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ * [FEATURE] Ingester: Add `ingester.instance-limits.max-inflight-query-requests` to allow limiting ingester concurrent queries. #6081 * [FEATURE] Distributor: Add `validation.max-native-histogram-buckets` to limit max number of bucket count. Distributor will try to automatically reduce histogram resolution until it is within the bucket limit or resolution cannot be reduced anymore. #6104 * [FEATURE] Store Gateway: Token bucket limiter. #6016 +* [FEATURE] Ruler: Add support for `query_offset` field on RuleGroup and new `ruler_query_offset` per-tenant limit. #6085 * [ENHANCEMENT] rulers: Add support to persist tokens in rulers. #5987 * [ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. #5892 * [ENHANCEMENT] Query Frontend/Querier: Returns `warnings` on prometheus query responses. #5916 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index f4f7865552a..0e646f77881 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3349,6 +3349,10 @@ query_rejection: # CLI flag: -ruler.max-rule-groups-per-tenant [ruler_max_rule_groups_per_tenant: | default = 0] +# Duration to offset all rule evaluation queries per-tenant. +# CLI flag: -ruler.query-offset +[ruler_query_offset: | default = 0s] + # The default tenant's shard size when the shuffle-sharding strategy is used. # Must be set when the store-gateway sharding is enabled with the # shuffle-sharding strategy. When this setting is specified in the per-tenant @@ -4147,10 +4151,6 @@ ruler_client: # CLI flag: -ruler.rule-path [rule_path: | default = "/rules"] -# Default offset for all rule evaluation queries -# CLI flag: -ruler.rule-query-offset -[rule_query_offset: | default = 0s] - # Comma-separated list of URL(s) of the Alertmanager(s) to send notifications # to. Each Alertmanager URL is treated as a separate group in the configuration. # Multiple Alertmanagers in HA per group can be supported by using DNS diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index 71fc80de1c1..9c3fd2f0f91 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -178,6 +178,7 @@ type RulesLimits interface { RulerTenantShardSize(userID string) int RulerMaxRuleGroupsPerTenant(userID string) int RulerMaxRulesPerRuleGroup(userID string) int + RulerQueryOffset(userID string) time.Duration DisabledRuleGroups(userID string) validation.DisabledRuleGroups } @@ -359,7 +360,7 @@ func DefaultTenantManagerFactory(cfg Config, p Pusher, q storage.Queryable, engi ConcurrentEvalsEnabled: cfg.ConcurrentEvalsEnabled, MaxConcurrentEvals: cfg.MaxConcurrentEvals, DefaultRuleQueryOffset: func() time.Duration { - return cfg.RuleQueryOffset + return overrides.RulerQueryOffset(userID) }, }) } diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 0d6f8a57cb1..e87b9b07009 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -105,8 +105,6 @@ type Config struct { PollInterval time.Duration `yaml:"poll_interval"` // Path to store rule files for prom manager. RulePath string `yaml:"rule_path"` - // Default offset for all rule evaluation queries. - RuleQueryOffset time.Duration `yaml:"rule_query_offset"` // URL of the Alertmanager to send notifications to. // If you are configuring the ruler to send to a Cortex Alertmanager, @@ -196,7 +194,6 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.Var(&cfg.ExternalURL, "ruler.external.url", "URL of alerts return path.") f.DurationVar(&cfg.EvaluationInterval, "ruler.evaluation-interval", 1*time.Minute, "How frequently to evaluate rules") f.DurationVar(&cfg.PollInterval, "ruler.poll-interval", 1*time.Minute, "How frequently to poll for rule changes") - f.DurationVar(&cfg.RuleQueryOffset, "ruler.rule-query-offset", 0*time.Minute, "Default offset for all rule evaluation queries") f.StringVar(&cfg.AlertmanagerURL, "ruler.alertmanager-url", "", "Comma-separated list of URL(s) of the Alertmanager(s) to send notifications to. Each Alertmanager URL is treated as a separate group in the configuration. Multiple Alertmanagers in HA per group can be supported by using DNS resolution via -ruler.alertmanager-discovery.") f.BoolVar(&cfg.AlertmanagerDiscovery, "ruler.alertmanager-discovery", false, "Use DNS SRV records to discover Alertmanager hosts.") @@ -914,6 +911,7 @@ func (r *Ruler) getLocalRules(userID string, rulesRequest RulesRequest, includeB } interval := group.Interval() + queryOffset := group.QueryOffset() groupDesc := &GroupStateDesc{ Group: &rulespb.RuleGroupDesc{ Name: group.Name(), @@ -921,7 +919,7 @@ func (r *Ruler) getLocalRules(userID string, rulesRequest RulesRequest, includeB Interval: interval, User: userID, Limit: int64(group.Limit()), - QueryOffset: group.QueryOffset(), + QueryOffset: &queryOffset, }, EvaluationTimestamp: group.GetLastEvaluation(), diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 9fc391fb729..befe7d01f44 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -88,6 +88,7 @@ type ruleLimits struct { maxRuleGroups int disabledRuleGroups validation.DisabledRuleGroups maxQueryLength time.Duration + queryOffset time.Duration } func (r ruleLimits) EvaluationDelay(_ string) time.Duration { @@ -112,6 +113,10 @@ func (r ruleLimits) DisabledRuleGroups(userID string) validation.DisabledRuleGro func (r ruleLimits) MaxQueryLength(_ string) time.Duration { return r.maxQueryLength } +func (r ruleLimits) RulerQueryOffset(_ string) time.Duration { + return r.queryOffset +} + func newEmptyQueryable() storage.Queryable { return storage.QueryableFunc(func(mint, maxt int64) (storage.Querier, error) { return emptyQuerier{}, nil @@ -2550,7 +2555,8 @@ func TestRuler_QueryOffset(t *testing.T) { compareRuleGroupDescToStateDesc(t, expectedRg, rg) // test default query offset=0 when not defined at group level - require.Equal(t, time.Duration(0), rg.GetGroup().QueryOffset) + gotOffset := rg.GetGroup().QueryOffset + require.Equal(t, time.Duration(0), *gotOffset) ctx = user.InjectOrgID(context.Background(), "user2") rls, err = r.Rules(ctx, &RulesRequest{}) @@ -2561,5 +2567,6 @@ func TestRuler_QueryOffset(t *testing.T) { compareRuleGroupDescToStateDesc(t, expectedRg, rg) // test group query offset is set - require.Equal(t, time.Minute*2, rg.GetGroup().QueryOffset) + gotOffset = rg.GetGroup().QueryOffset + require.Equal(t, time.Minute*2, *gotOffset) } diff --git a/pkg/ruler/rulespb/compat.go b/pkg/ruler/rulespb/compat.go index 4f916d09c4b..16bf075e27f 100644 --- a/pkg/ruler/rulespb/compat.go +++ b/pkg/ruler/rulespb/compat.go @@ -13,9 +13,10 @@ import ( // ToProto transforms a formatted prometheus rulegroup to a rule group protobuf func ToProto(user string, namespace string, rl rulefmt.RuleGroup) *RuleGroupDesc { - queryOffset := time.Duration(0) + var queryOffset *time.Duration if rl.QueryOffset != nil { - queryOffset = time.Duration(*rl.QueryOffset) + offset := time.Duration(*rl.QueryOffset) + queryOffset = &offset } rg := RuleGroupDesc{ Name: rl.Name, @@ -48,7 +49,10 @@ func formattedRuleToProto(rls []rulefmt.RuleNode) []*RuleDesc { // FromProto generates a rulefmt RuleGroup func FromProto(rg *RuleGroupDesc) rulefmt.RuleGroup { - queryOffset := model.Duration(rg.QueryOffset) + var queryOffset model.Duration + if rg.QueryOffset != nil { + queryOffset = model.Duration(*rg.QueryOffset) + } formattedRuleGroup := rulefmt.RuleGroup{ Name: rg.GetName(), Interval: model.Duration(rg.Interval), diff --git a/pkg/ruler/rulespb/compat_test.go b/pkg/ruler/rulespb/compat_test.go index b66fb3c3877..736366714dc 100644 --- a/pkg/ruler/rulespb/compat_test.go +++ b/pkg/ruler/rulespb/compat_test.go @@ -40,7 +40,7 @@ func TestProto(t *testing.T) { desc := ToProto("test", "namespace", rg) assert.Equal(t, len(rules), len(desc.Rules)) - assert.Equal(t, 30*time.Second, desc.QueryOffset) + assert.Equal(t, 30*time.Second, *desc.QueryOffset) ruleDesc := desc.Rules[0] diff --git a/pkg/ruler/rulespb/rules.pb.go b/pkg/ruler/rulespb/rules.pb.go index 5ac1eb30806..8f09b2cb40b 100644 --- a/pkg/ruler/rulespb/rules.pb.go +++ b/pkg/ruler/rulespb/rules.pb.go @@ -43,9 +43,9 @@ type RuleGroupDesc struct { // having to repeatedly redefine the proto description. It can also be leveraged // to create custom `ManagerOpts` based on rule configs which can then be passed // to the Prometheus Manager. - Options []*types.Any `protobuf:"bytes,9,rep,name=options,proto3" json:"options,omitempty"` - Limit int64 `protobuf:"varint,10,opt,name=limit,proto3" json:"limit,omitempty"` - QueryOffset time.Duration `protobuf:"bytes,11,opt,name=queryOffset,proto3,stdduration" json:"queryOffset"` + Options []*types.Any `protobuf:"bytes,9,rep,name=options,proto3" json:"options,omitempty"` + Limit int64 `protobuf:"varint,10,opt,name=limit,proto3" json:"limit,omitempty"` + QueryOffset *time.Duration `protobuf:"bytes,11,opt,name=queryOffset,proto3,stdduration" json:"queryOffset,omitempty"` } func (m *RuleGroupDesc) Reset() { *m = RuleGroupDesc{} } @@ -129,11 +129,11 @@ func (m *RuleGroupDesc) GetLimit() int64 { return 0 } -func (m *RuleGroupDesc) GetQueryOffset() time.Duration { +func (m *RuleGroupDesc) GetQueryOffset() *time.Duration { if m != nil { return m.QueryOffset } - return 0 + return nil } // RuleDesc is a proto representation of a Prometheus Rule @@ -222,13 +222,13 @@ func init() { func init() { proto.RegisterFile("rules.proto", fileDescriptor_8e722d3e922f0937) } var fileDescriptor_8e722d3e922f0937 = []byte{ - // 545 bytes of a gzipped FileDescriptorProto + // 548 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x52, 0x41, 0x8b, 0xd3, 0x4c, 0x18, 0xce, 0x6c, 0xd3, 0x6c, 0x3a, 0xa1, 0xec, 0x32, 0x5f, 0xf9, 0xc8, 0xae, 0x32, 0x2d, 0x0b, 0x42, 0x4f, 0x29, 0xac, 0x78, 0xf0, 0x20, 0xd2, 0xb2, 0xae, 0x50, 0x04, 0x25, 0x47, 0x11, 0x96, 0x49, 0x3a, 0x89, 0x71, 0xd3, 0x4c, 0x9c, 0x4c, 0x64, 0x7b, 0xf3, 0x27, 0x78, 0xf4, 0x27, 0xf8, 0x53, 0xf6, 0x58, 0x6f, 0x8b, 0x48, 0xb5, 0xe9, 0x45, 0x3c, 0xed, 0x4f, 0x90, 0x99, 0x49, 0x74, - 0xd5, 0x83, 0xeb, 0xc1, 0x53, 0xde, 0x67, 0x9e, 0x3c, 0xf3, 0x3e, 0xf3, 0xbc, 0x2f, 0x74, 0x78, + 0xd5, 0x83, 0xeb, 0xc1, 0x53, 0xde, 0x67, 0x9e, 0x3c, 0xf3, 0x3e, 0xef, 0x33, 0x2f, 0x74, 0x78, 0x99, 0xd2, 0xc2, 0xcb, 0x39, 0x13, 0x0c, 0xb5, 0x15, 0xd8, 0xef, 0xc5, 0x2c, 0x66, 0xea, 0x64, 0x24, 0x2b, 0x4d, 0xee, 0xe3, 0x98, 0xb1, 0x38, 0xa5, 0x23, 0x85, 0x82, 0x32, 0x1a, 0xcd, 0x4a, 0x4e, 0x44, 0xc2, 0xb2, 0x9a, 0xdf, 0xfb, 0x95, 0x27, 0xd9, 0xa2, 0xa6, 0xee, 0xc6, 0x89, 0x78, @@ -237,27 +237,27 @@ var fileDescriptor_8e722d3e922f0937 = []byte{ 0xe9, 0x43, 0xce, 0xca, 0xfc, 0x88, 0x16, 0x21, 0x42, 0xd0, 0xcc, 0xc8, 0x9c, 0xba, 0x60, 0x00, 0x86, 0x1d, 0x5f, 0xd5, 0xe8, 0x26, 0xec, 0xc8, 0x6f, 0x91, 0x93, 0x90, 0xba, 0x5b, 0x8a, 0xf8, 0x71, 0x80, 0xee, 0x43, 0x3b, 0xc9, 0x04, 0xe5, 0xaf, 0x48, 0xea, 0xb6, 0x06, 0x60, 0xe8, 0x1c, - 0xee, 0x79, 0xda, 0xac, 0xd7, 0x98, 0xf5, 0x8e, 0xea, 0xc7, 0x4c, 0xec, 0xf3, 0x55, 0xdf, 0x78, - 0xfb, 0xa9, 0x0f, 0xfc, 0xef, 0x22, 0x74, 0x0b, 0xea, 0x64, 0x5c, 0x73, 0xd0, 0x1a, 0x3a, 0x87, - 0x3b, 0x9e, 0x0e, 0x4d, 0xfa, 0x92, 0x96, 0x7c, 0xcd, 0x4a, 0x67, 0x65, 0x41, 0xb9, 0x6b, 0x69, - 0x67, 0xb2, 0x46, 0x1e, 0xdc, 0x66, 0xb9, 0xbc, 0xb8, 0x70, 0x3b, 0x4a, 0xdc, 0xfb, 0xad, 0xf5, - 0x38, 0x5b, 0xf8, 0xcd, 0x4f, 0xa8, 0x07, 0xdb, 0x69, 0x32, 0x4f, 0x84, 0x0b, 0x07, 0x60, 0xd8, - 0xf2, 0x35, 0x40, 0x0f, 0xa0, 0xf3, 0xb2, 0xa4, 0x7c, 0xf1, 0x38, 0x8a, 0x0a, 0x2a, 0x5c, 0xe7, - 0xfa, 0x8f, 0xb8, 0xaa, 0x9b, 0x9a, 0x76, 0x7b, 0xd7, 0x9a, 0x9a, 0xf6, 0xf6, 0xae, 0x3d, 0x35, - 0x6d, 0x7b, 0xb7, 0x73, 0xf0, 0xbe, 0x05, 0xed, 0xe6, 0x19, 0xd2, 0xbf, 0x9c, 0x4c, 0x93, 0xac, - 0xac, 0xd1, 0xff, 0xd0, 0xe2, 0x34, 0x64, 0x7c, 0x56, 0xc7, 0x5a, 0x23, 0xe9, 0x93, 0xa4, 0x94, - 0x0b, 0x15, 0x68, 0xc7, 0xd7, 0x00, 0xdd, 0x81, 0xad, 0x88, 0x71, 0xd7, 0xbc, 0xbe, 0x3f, 0xf9, - 0x3f, 0xca, 0xa0, 0x95, 0x92, 0x80, 0xa6, 0x85, 0xdb, 0x56, 0x19, 0xfd, 0xe7, 0x35, 0xcb, 0xe0, - 0x3d, 0x92, 0xe7, 0x4f, 0x48, 0xc2, 0x27, 0x63, 0xa9, 0xf9, 0xb0, 0xea, 0xff, 0xd5, 0x32, 0x69, - 0xfd, 0x78, 0x46, 0x72, 0x41, 0xb9, 0x5f, 0x77, 0x41, 0x67, 0xd0, 0x21, 0x59, 0xc6, 0x04, 0xd1, - 0x83, 0xb1, 0xfe, 0x69, 0xd3, 0xab, 0xad, 0xd0, 0x33, 0xd8, 0x3d, 0xa5, 0x34, 0x3f, 0x4e, 0x78, - 0x92, 0xc5, 0xc7, 0x8c, 0xbb, 0xdd, 0x3f, 0x45, 0x75, 0x43, 0x3a, 0xf8, 0xba, 0xea, 0xef, 0x48, - 0xdd, 0x49, 0xa4, 0x84, 0x27, 0x11, 0xe3, 0x2a, 0xbd, 0x9f, 0x2f, 0x53, 0x93, 0xed, 0x4e, 0xee, - 0x2d, 0xd7, 0xd8, 0xb8, 0x58, 0x63, 0xe3, 0x72, 0x8d, 0xc1, 0xeb, 0x0a, 0x83, 0x77, 0x15, 0x06, - 0xe7, 0x15, 0x06, 0xcb, 0x0a, 0x83, 0xcf, 0x15, 0x06, 0x5f, 0x2a, 0x6c, 0x5c, 0x56, 0x18, 0xbc, - 0xd9, 0x60, 0x63, 0xb9, 0xc1, 0xc6, 0xc5, 0x06, 0x1b, 0x4f, 0xb7, 0xd5, 0x0e, 0xe7, 0x41, 0x60, - 0x29, 0x0f, 0xb7, 0xbf, 0x05, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xaa, 0xf7, 0xf4, 0x1a, 0x04, 0x00, - 0x00, + 0xee, 0x79, 0xda, 0xac, 0xd7, 0x98, 0xf5, 0x8e, 0xea, 0x61, 0x26, 0xf6, 0xf9, 0xaa, 0x6f, 0xbc, + 0xfd, 0xd4, 0x07, 0xfe, 0x77, 0x11, 0xba, 0x05, 0x75, 0x32, 0xae, 0x39, 0x68, 0x0d, 0x9d, 0xc3, + 0x1d, 0x4f, 0x87, 0x26, 0x7d, 0x49, 0x4b, 0xbe, 0x66, 0xa5, 0xb3, 0xb2, 0xa0, 0xdc, 0xb5, 0xb4, + 0x33, 0x59, 0x23, 0x0f, 0x6e, 0xb3, 0x5c, 0x5e, 0x5c, 0xb8, 0x1d, 0x25, 0xee, 0xfd, 0xd6, 0x7a, + 0x9c, 0x2d, 0xfc, 0xe6, 0x27, 0xd4, 0x83, 0xed, 0x34, 0x99, 0x27, 0xc2, 0x85, 0x03, 0x30, 0x6c, + 0xf9, 0x1a, 0xa0, 0x07, 0xd0, 0x79, 0x59, 0x52, 0xbe, 0x78, 0x1c, 0x45, 0x05, 0x15, 0xae, 0x73, + 0x9d, 0x21, 0x80, 0x1a, 0xe2, 0xaa, 0x6e, 0x6a, 0xda, 0xed, 0x5d, 0x6b, 0x6a, 0xda, 0xdb, 0xbb, + 0xf6, 0xd4, 0xb4, 0xed, 0xdd, 0xce, 0xc1, 0xfb, 0x16, 0xb4, 0x9b, 0x31, 0xa4, 0x7f, 0xf9, 0x32, + 0x4d, 0xb2, 0xb2, 0x46, 0xff, 0x43, 0x8b, 0xd3, 0x90, 0xf1, 0x59, 0x1d, 0x6b, 0x8d, 0xa4, 0x4f, + 0x92, 0x52, 0x2e, 0x54, 0xa0, 0x1d, 0x5f, 0x03, 0x74, 0x07, 0xb6, 0x22, 0xc6, 0x5d, 0xf3, 0xfa, + 0x21, 0xcb, 0xff, 0x51, 0x06, 0xad, 0x94, 0x04, 0x34, 0x2d, 0xdc, 0xb6, 0xca, 0xe8, 0x3f, 0xaf, + 0x59, 0x06, 0xef, 0x91, 0x3c, 0x7f, 0x42, 0x12, 0x3e, 0x19, 0x4b, 0xcd, 0x87, 0x55, 0xff, 0xaf, + 0x96, 0x49, 0xeb, 0xc7, 0x33, 0x92, 0x0b, 0xca, 0xfd, 0xba, 0x0b, 0x3a, 0x83, 0x0e, 0xc9, 0x32, + 0x26, 0x88, 0x7e, 0x18, 0xeb, 0x9f, 0x36, 0xbd, 0xda, 0x0a, 0x3d, 0x83, 0xdd, 0x53, 0x4a, 0xf3, + 0xe3, 0x84, 0x27, 0x59, 0x7c, 0xcc, 0xb8, 0xdb, 0xfd, 0x53, 0x54, 0x37, 0xa4, 0x83, 0xaf, 0xab, + 0xfe, 0x8e, 0xd4, 0x9d, 0x44, 0x4a, 0x78, 0x12, 0x31, 0xae, 0xd2, 0xfb, 0xf9, 0x32, 0xf5, 0xb2, + 0xdd, 0xc9, 0xbd, 0xe5, 0x1a, 0x1b, 0x17, 0x6b, 0x6c, 0x5c, 0xae, 0x31, 0x78, 0x5d, 0x61, 0xf0, + 0xae, 0xc2, 0xe0, 0xbc, 0xc2, 0x60, 0x59, 0x61, 0xf0, 0xb9, 0xc2, 0xe0, 0x4b, 0x85, 0x8d, 0xcb, + 0x0a, 0x83, 0x37, 0x1b, 0x6c, 0x2c, 0x37, 0xd8, 0xb8, 0xd8, 0x60, 0xe3, 0xe9, 0xb6, 0xda, 0xe1, + 0x3c, 0x08, 0x2c, 0xe5, 0xe1, 0xf6, 0xb7, 0x00, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x1c, 0xe8, 0x17, + 0x1a, 0x04, 0x00, 0x00, } func (this *RuleGroupDesc) Equal(that interface{}) bool { @@ -310,7 +310,13 @@ func (this *RuleGroupDesc) Equal(that interface{}) bool { if this.Limit != that1.Limit { return false } - if this.QueryOffset != that1.QueryOffset { + if this.QueryOffset != nil && that1.QueryOffset != nil { + if *this.QueryOffset != *that1.QueryOffset { + return false + } + } else if this.QueryOffset != nil { + return false + } else if that1.QueryOffset != nil { return false } return true @@ -432,14 +438,16 @@ func (m *RuleGroupDesc) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - n1, err1 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.QueryOffset, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.QueryOffset):]) - if err1 != nil { - return 0, err1 + if m.QueryOffset != nil { + n1, err1 := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.QueryOffset, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(*m.QueryOffset):]) + if err1 != nil { + return 0, err1 + } + i -= n1 + i = encodeVarintRules(dAtA, i, uint64(n1)) + i-- + dAtA[i] = 0x5a } - i -= n1 - i = encodeVarintRules(dAtA, i, uint64(n1)) - i-- - dAtA[i] = 0x5a if m.Limit != 0 { i = encodeVarintRules(dAtA, i, uint64(m.Limit)) i-- @@ -639,8 +647,10 @@ func (m *RuleGroupDesc) Size() (n int) { if m.Limit != 0 { n += 1 + sovRules(uint64(m.Limit)) } - l = github_com_gogo_protobuf_types.SizeOfStdDuration(m.QueryOffset) - n += 1 + l + sovRules(uint64(l)) + if m.QueryOffset != nil { + l = github_com_gogo_protobuf_types.SizeOfStdDuration(*m.QueryOffset) + n += 1 + l + sovRules(uint64(l)) + } return n } @@ -709,7 +719,7 @@ func (this *RuleGroupDesc) String() string { `User:` + fmt.Sprintf("%v", this.User) + `,`, `Options:` + repeatedStringForOptions + `,`, `Limit:` + fmt.Sprintf("%v", this.Limit) + `,`, - `QueryOffset:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.QueryOffset), "Duration", "duration.Duration", 1), `&`, ``, 1) + `,`, + `QueryOffset:` + strings.Replace(fmt.Sprintf("%v", this.QueryOffset), "Duration", "duration.Duration", 1) + `,`, `}`, }, "") return s @@ -1012,7 +1022,10 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if err := github_com_gogo_protobuf_types.StdDurationUnmarshal(&m.QueryOffset, dAtA[iNdEx:postIndex]); err != nil { + if m.QueryOffset == nil { + m.QueryOffset = new(time.Duration) + } + if err := github_com_gogo_protobuf_types.StdDurationUnmarshal(m.QueryOffset, dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex diff --git a/pkg/ruler/rulespb/rules.proto b/pkg/ruler/rulespb/rules.proto index 189c050eb47..ed4e98a76b2 100644 --- a/pkg/ruler/rulespb/rules.proto +++ b/pkg/ruler/rulespb/rules.proto @@ -29,7 +29,7 @@ message RuleGroupDesc { repeated google.protobuf.Any options = 9; int64 limit =10; google.protobuf.Duration queryOffset = 11 - [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; + [(gogoproto.nullable) = true, (gogoproto.stdduration) = true]; } // RuleDesc is a proto representation of a Prometheus Rule diff --git a/pkg/ruler/store_mock_test.go b/pkg/ruler/store_mock_test.go index 5bd6025f01a..30a53fdab16 100644 --- a/pkg/ruler/store_mock_test.go +++ b/pkg/ruler/store_mock_test.go @@ -132,6 +132,7 @@ var ( }, }, } + queryOffset = 2 * time.Minute mockRulesQueryOffset = map[string]rulespb.RuleGroupList{ "user1": { &rulespb.RuleGroupDesc{ @@ -164,7 +165,7 @@ var ( }, }, Interval: interval, - QueryOffset: 2 * time.Minute, + QueryOffset: &queryOffset, }, }, } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index fc9faab0780..e54217cd445 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -178,6 +178,7 @@ type Limits struct { RulerTenantShardSize int `yaml:"ruler_tenant_shard_size" json:"ruler_tenant_shard_size"` RulerMaxRulesPerRuleGroup int `yaml:"ruler_max_rules_per_rule_group" json:"ruler_max_rules_per_rule_group"` RulerMaxRuleGroupsPerTenant int `yaml:"ruler_max_rule_groups_per_tenant" json:"ruler_max_rule_groups_per_tenant"` + RulerQueryOffset model.Duration `yaml:"ruler_query_offset" json:"ruler_query_offset"` // Store-gateway. StoreGatewayTenantShardSize float64 `yaml:"store_gateway_tenant_shard_size" json:"store_gateway_tenant_shard_size"` @@ -264,10 +265,10 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxOutstandingPerTenant, "frontend.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per request queue (either query frontend or query scheduler); requests beyond this error with HTTP 429.") - f.Var(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.") f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") f.IntVar(&l.RulerMaxRulesPerRuleGroup, "ruler.max-rules-per-rule-group", 0, "Maximum number of rules per rule group per-tenant. 0 to disable.") f.IntVar(&l.RulerMaxRuleGroupsPerTenant, "ruler.max-rule-groups-per-tenant", 0, "Maximum number of rule groups per-tenant. 0 to disable.") + f.Var(&l.RulerQueryOffset, "ruler.query-offset", "Duration to offset all rule evaluation queries per-tenant.") f.Var(&l.CompactorBlocksRetentionPeriod, "compactor.blocks-retention-period", "Delete blocks containing samples older than the specified retention period. 0 to disable.") f.IntVar(&l.CompactorTenantShardSize, "compactor.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by the compactor. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") @@ -791,6 +792,11 @@ func (o *Overrides) RulerMaxRuleGroupsPerTenant(userID string) int { return o.GetOverridesForUser(userID).RulerMaxRuleGroupsPerTenant } +// RulerQueryOffset returns the rule query offset for a given user. +func (o *Overrides) RulerQueryOffset(userID string) time.Duration { + return time.Duration(o.GetOverridesForUser(userID).RulerQueryOffset) +} + // StoreGatewayTenantShardSize returns the store-gateway shard size for a given user. func (o *Overrides) StoreGatewayTenantShardSize(userID string) float64 { return o.GetOverridesForUser(userID).StoreGatewayTenantShardSize