From 3b7d8204ec048de028dcac69a776b52711f7d80f Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Tue, 17 May 2022 11:22:57 +0200 Subject: [PATCH 1/3] query-scheduler: Now proper support for `querier_forget_delay` Since now shuffle sharding is supported on the `query-path` in Loki, it makes sense to enable this config. Signed-off-by: Kaviraj --- docs/sources/configuration/_index.md | 6 ++++++ pkg/scheduler/scheduler.go | 6 ++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/sources/configuration/_index.md b/docs/sources/configuration/_index.md index b91b3f7be1b2..f69c1269c3c3 100644 --- a/docs/sources/configuration/_index.md +++ b/docs/sources/configuration/_index.md @@ -328,6 +328,12 @@ The `query_scheduler` block configures the Loki query scheduler. # CLI flag: -query-scheduler.max-outstanding-requests-per-tenant [max_outstanding_requests_per_tenant: | default = 100] +# If a querier disconnects without sending notification about graceful shutdown, +# the query-frontend will keep the querier in the tenant's shard until the forget delay has passed. +# This feature is useful to reduce the blast radius when shuffle-sharding is enabled. +# CLI flag: -query-scheduler.querier-forget-delay +[querier_forget_delay: | default = 0] + # This configures the gRPC client used to report errors back to the # query-frontend. [grpc_client_config: ] diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 5c5604685773..cbf90b833d7a 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -120,7 +120,7 @@ type connectedFrontend struct { type Config struct { MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant"` - QuerierForgetDelay time.Duration `yaml:"-"` + QuerierForgetDelay time.Duration `yaml:"querier_forget_delay"` GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config" doc:"description=This configures the gRPC client used to report errors back to the query-frontend."` // Schedulers ring UseSchedulerRing bool `yaml:"use_scheduler_ring"` @@ -129,9 +129,7 @@ type Config struct { func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.IntVar(&cfg.MaxOutstandingPerTenant, "query-scheduler.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per query scheduler. In-flight requests above this limit will fail with HTTP response status code 429.") - // Loki doesn't have query shuffle sharding yet for which this config is intended - // use the default value of 0 until someday when this config may be needed. - cfg.QuerierForgetDelay = 0 + f.DurationVar(&cfg.QuerierForgetDelay, "query-scheduler.querier-forget-delay", 0, "If a querier disconnects without sending notification about graceful shutdown, the query-frontend will keep the querier in the tenant's shard until the forget delay has passed. This feature is useful to reduce the blast radius when shuffle-sharding is enabled.") cfg.GRPCClientConfig.RegisterFlagsWithPrefix("query-scheduler.grpc-client-config", f) f.BoolVar(&cfg.UseSchedulerRing, "query-scheduler.use-scheduler-ring", false, "Set to true to have the query scheduler create a ring and the frontend and frontend_worker use this ring to get the addresses of the query schedulers. If frontend_address and scheduler_address are not present in the config this value will be toggle by Loki to true") cfg.SchedulerRing.RegisterFlagsWithPrefix("query-scheduler.", "collectors/", f) From af8258fa2acfcfb124f472ee7a61bd45ebd85193 Mon Sep 17 00:00:00 2001 From: Kaviraj Kanagaraj Date: Tue, 17 May 2022 11:28:22 +0200 Subject: [PATCH 2/3] Update pkg/scheduler/scheduler.go Co-authored-by: Christian Simon --- pkg/scheduler/scheduler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index cbf90b833d7a..7a433b3fc1ff 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -129,7 +129,7 @@ type Config struct { func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.IntVar(&cfg.MaxOutstandingPerTenant, "query-scheduler.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per query scheduler. In-flight requests above this limit will fail with HTTP response status code 429.") - f.DurationVar(&cfg.QuerierForgetDelay, "query-scheduler.querier-forget-delay", 0, "If a querier disconnects without sending notification about graceful shutdown, the query-frontend will keep the querier in the tenant's shard until the forget delay has passed. This feature is useful to reduce the blast radius when shuffle-sharding is enabled.") + f.DurationVar(&cfg.QuerierForgetDelay, "query-scheduler.querier-forget-delay", 0, "If a querier disconnects without sending notification about graceful shutdown, the query-scheduler will keep the querier in the tenant's shard until the forget delay has passed. This feature is useful to reduce the blast radius when shuffle-sharding is enabled.") cfg.GRPCClientConfig.RegisterFlagsWithPrefix("query-scheduler.grpc-client-config", f) f.BoolVar(&cfg.UseSchedulerRing, "query-scheduler.use-scheduler-ring", false, "Set to true to have the query scheduler create a ring and the frontend and frontend_worker use this ring to get the addresses of the query schedulers. If frontend_address and scheduler_address are not present in the config this value will be toggle by Loki to true") cfg.SchedulerRing.RegisterFlagsWithPrefix("query-scheduler.", "collectors/", f) From 3dcb4c1e095615827c9990d086ea1c13651d851e Mon Sep 17 00:00:00 2001 From: Kaviraj Kanagaraj Date: Tue, 17 May 2022 11:28:27 +0200 Subject: [PATCH 3/3] Update docs/sources/configuration/_index.md Co-authored-by: Christian Simon --- docs/sources/configuration/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/configuration/_index.md b/docs/sources/configuration/_index.md index f69c1269c3c3..24d37c8631ca 100644 --- a/docs/sources/configuration/_index.md +++ b/docs/sources/configuration/_index.md @@ -329,7 +329,7 @@ The `query_scheduler` block configures the Loki query scheduler. [max_outstanding_requests_per_tenant: | default = 100] # If a querier disconnects without sending notification about graceful shutdown, -# the query-frontend will keep the querier in the tenant's shard until the forget delay has passed. +# the query-scheduler will keep the querier in the tenant's shard until the forget delay has passed. # This feature is useful to reduce the blast radius when shuffle-sharding is enabled. # CLI flag: -query-scheduler.querier-forget-delay [querier_forget_delay: | default = 0]