diff --git a/production/ksonnet/loki/config.libsonnet b/production/ksonnet/loki/config.libsonnet index 006143db7c90..a21162e9487f 100644 --- a/production/ksonnet/loki/config.libsonnet +++ b/production/ksonnet/loki/config.libsonnet @@ -147,12 +147,9 @@ frontend: { compress_responses: true, log_queries_longer_than: '5s', - max_outstanding_per_tenant: if $._config.queryFrontend.sharded_queries_enabled then 1024 else 256, }, frontend_worker: { - frontend_address: 'query-frontend.%s.svc.cluster.local:9095' % $._config.namespace, - // Limit to N/2 worker threads per frontend, as we have two frontends. - parallelism: std.floor($._config.querier.concurrency / $._config.queryFrontend.replicas), + match_max_concurrent: true, grpc_client_config: { max_send_msg_size: $._config.grpc_server_max_msg_size, }, @@ -178,6 +175,7 @@ parallelise_shardable_queries: true, } else {}, querier: { + max_concurrent: $._config.querier.concurrency, query_ingesters_within: '2h', // twice the max-chunk age (1h default) for safety buffer }, limits_config: { diff --git a/production/ksonnet/loki/loki.libsonnet b/production/ksonnet/loki/loki.libsonnet index db311f304297..fc5978cfb42d 100644 --- a/production/ksonnet/loki/loki.libsonnet +++ b/production/ksonnet/loki/loki.libsonnet @@ -15,7 +15,7 @@ // Query scheduler support // must be mixed in after frontend and querier so it can override their configuration. -(import 'query-scheduler.libsonnet') + +(import 'query-scheduler.libsonnet') + // Supporting services (import 'memcached.libsonnet') + diff --git a/production/ksonnet/loki/query-scheduler.libsonnet b/production/ksonnet/loki/query-scheduler.libsonnet index fbab6129b6a4..dcaf54a91292 100644 --- a/production/ksonnet/loki/query-scheduler.libsonnet +++ b/production/ksonnet/loki/query-scheduler.libsonnet @@ -1,18 +1,28 @@ - local k = import 'ksonnet-util/kausal.libsonnet'; { + local max_outstanding = if $._config.queryFrontend.sharded_queries_enabled then 1024 else 256, + // Override frontend and querier configuration - _config +:: { + _config+:: { loki+: if $._config.query_scheduler_enabled then { frontend+: { - scheduler_address: 'query-scheduler.%s.svc.cluster.local:9095' % $._config.namespace, + scheduler_address: 'query-scheduler-discovery.%s.svc.cluster.local:9095' % $._config.namespace, }, frontend_worker+: { - frontend_address: '', - scheduler_address: 'query-scheduler.%s.svc.cluster.local:9095' % $._config.namespace, + scheduler_address: 'query-scheduler-discovery.%s.svc.cluster.local:9095' % $._config.namespace, + }, + query_scheduler+: { + max_outstanding_requests_per_tenant: max_outstanding, + }, + } else { + frontend+: { + max_outstanding_per_tenant: max_outstanding, }, - } else {}, + frontend_worker+: { + frontend_address: 'query-frontend.%s.svc.cluster.local:9095' % $._config.namespace, + }, + }, }, query_scheduler_args:: if $._config.query_scheduler_enabled then @@ -45,7 +55,11 @@ local k = import 'ksonnet-util/kausal.libsonnet'; else {}, local service = k.core.v1.service, - query_scheduler_service: if $._config.query_scheduler_enabled then - k.util.serviceFor($.query_scheduler_deployment) - else {}, + + // Headless to make sure resolution gets IP address of target pods, and not service IP. + query_scheduler_discovery_service: if !$._config.query_scheduler_enabled then {} else + k.util.serviceFor($.query_scheduler_deployment) + + service.mixin.spec.withPublishNotReadyAddresses(true) + + service.mixin.spec.withClusterIp('None') + + service.mixin.metadata.withName('query-scheduler-discovery'), }