From 623f96f536f9b4cc625181b2eda54a690ef02edf Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Sun, 9 Jan 2022 19:32:13 +0200 Subject: [PATCH 1/7] Add memory ballast --- cmd/loki/main.go | 4 ++++ pkg/loki/config_wrapper.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/cmd/loki/main.go b/cmd/loki/main.go index 228b1335f44a..82ffc63b547d 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "reflect" + "runtime" "github.com/go-kit/log/level" "github.com/prometheus/common/version" @@ -87,6 +88,9 @@ func main() { }() } + ballast := make([]byte, config.BallastBytes) + runtime.KeepAlive(ballast) + // Start Loki t, err := loki.New(config.Config) util_log.CheckFatal("initialising loki", err) diff --git a/pkg/loki/config_wrapper.go b/pkg/loki/config_wrapper.go index c6def028f793..f3acd1d9c3ad 100644 --- a/pkg/loki/config_wrapper.go +++ b/pkg/loki/config_wrapper.go @@ -35,6 +35,7 @@ type ConfigWrapper struct { LogConfig bool ConfigFile string ConfigExpandEnv bool + BallastBytes int } func (c *ConfigWrapper) RegisterFlags(f *flag.FlagSet) { @@ -46,6 +47,8 @@ func (c *ConfigWrapper) RegisterFlags(f *flag.FlagSet) { "level with the order reversed, reversing the order makes viewing the entries easier in Grafana.") f.StringVar(&c.ConfigFile, "config.file", "", "yaml file to load") f.BoolVar(&c.ConfigExpandEnv, "config.expand-env", false, "Expands ${var} in config according to the values of the environment variables.") + f.IntVar(&c.BallastBytes, "config.ballast-bytes", 0, "The amount of virtual memory to reserve as a ballast in order to optimise "+ + "garbage collection.") c.Config.RegisterFlags(f) } From 360df57fb44efd6095cdb14e568e220750d3d640 Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Sun, 9 Jan 2022 19:44:29 +0200 Subject: [PATCH 2/7] Add changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8bd3df81cbf..751df6c49f05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## Main - +* [5081](https://github.com/grafana/loki/pull/5081) **SasSwart**: Add the option to configure memory ballast for Loki * [5067](https://github.com/grafana/loki/pull/5057) **cstyan**: Add a metric to Azure Blob Storage client to track total egress bytes * [4950](https://github.com/grafana/loki/pull/4950) **DylanGuedes**: Implement common instance addr/net interface * [4949](https://github.com/grafana/loki/pull/4949) **ssncferreira**: Add query `queueTime` metric to statistics and metrics.go From 1ff4222d96cf7672bdb45782a8d8128de42182e0 Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Sun, 9 Jan 2022 20:25:34 +0200 Subject: [PATCH 3/7] Add documentation Add additional information to Flag description --- cmd/loki/main.go | 3 +++ pkg/loki/config_wrapper.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/loki/main.go b/cmd/loki/main.go index 82ffc63b547d..d95f28d60361 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -88,6 +88,9 @@ func main() { }() } + // Allocate a block of memory to reduce the frequency of garbage collection. + // The larger the ballast, the lower the garbage collection frequency. + // https://github.com/grafana/loki/issues/781 ballast := make([]byte, config.BallastBytes) runtime.KeepAlive(ballast) diff --git a/pkg/loki/config_wrapper.go b/pkg/loki/config_wrapper.go index f3acd1d9c3ad..ee8c783eccc1 100644 --- a/pkg/loki/config_wrapper.go +++ b/pkg/loki/config_wrapper.go @@ -48,7 +48,7 @@ func (c *ConfigWrapper) RegisterFlags(f *flag.FlagSet) { f.StringVar(&c.ConfigFile, "config.file", "", "yaml file to load") f.BoolVar(&c.ConfigExpandEnv, "config.expand-env", false, "Expands ${var} in config according to the values of the environment variables.") f.IntVar(&c.BallastBytes, "config.ballast-bytes", 0, "The amount of virtual memory to reserve as a ballast in order to optimise "+ - "garbage collection.") + "garbage collection. Larger ballasts result in fewer garbage collection passes, reducing CPU overhead at the cost of heap size.") c.Config.RegisterFlags(f) } From 81024217c3eb2cafad287f4754639a7e133a3342 Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Mon, 10 Jan 2022 03:02:04 +0200 Subject: [PATCH 4/7] Ensure that ballast_bytes can be configured from config file Add documentation --- docs/sources/configuration/_index.md | 6 ++++++ pkg/loki/config_wrapper.go | 3 --- pkg/loki/loki.go | 7 ++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/sources/configuration/_index.md b/docs/sources/configuration/_index.md index c069c3070631..6a735580d47f 100644 --- a/docs/sources/configuration/_index.md +++ b/docs/sources/configuration/_index.md @@ -93,6 +93,12 @@ Pass the `-config.expand-env` flag at the command line to enable this way of set # if true. If false, the OrgID will always be set to "fake". [auth_enabled: | default = true] +# The amount of virtual memory to reserve as a ballast in order to optimise +# garbage collection. Larger ballasts result in fewer garbage collection passes, reducing CPU overhead at +# the cost of heap size. The ballast will not consume physical memory, because it is never read from. +# It will, however, distort metrics, because it is counted as live memory. +[ballast_bytes: | default = 0] + # Configures the server of the launched module(s). [server: ] diff --git a/pkg/loki/config_wrapper.go b/pkg/loki/config_wrapper.go index ee8c783eccc1..c6def028f793 100644 --- a/pkg/loki/config_wrapper.go +++ b/pkg/loki/config_wrapper.go @@ -35,7 +35,6 @@ type ConfigWrapper struct { LogConfig bool ConfigFile string ConfigExpandEnv bool - BallastBytes int } func (c *ConfigWrapper) RegisterFlags(f *flag.FlagSet) { @@ -47,8 +46,6 @@ func (c *ConfigWrapper) RegisterFlags(f *flag.FlagSet) { "level with the order reversed, reversing the order makes viewing the entries easier in Grafana.") f.StringVar(&c.ConfigFile, "config.file", "", "yaml file to load") f.BoolVar(&c.ConfigExpandEnv, "config.expand-env", false, "Expands ${var} in config according to the values of the environment variables.") - f.IntVar(&c.BallastBytes, "config.ballast-bytes", 0, "The amount of virtual memory to reserve as a ballast in order to optimise "+ - "garbage collection. Larger ballasts result in fewer garbage collection passes, reducing CPU overhead at the cost of heap size.") c.Config.RegisterFlags(f) } diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index a3372cb86afa..bb05e572d68c 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -53,9 +53,10 @@ import ( // Config is the root config for Loki. type Config struct { - Target flagext.StringSliceCSV `yaml:"target,omitempty"` - AuthEnabled bool `yaml:"auth_enabled,omitempty"` - HTTPPrefix string `yaml:"http_prefix"` + Target flagext.StringSliceCSV `yaml:"target,omitempty"` + AuthEnabled bool `yaml:"auth_enabled,omitempty"` + HTTPPrefix string `yaml:"http_prefix"` + BallastBytes int `yaml:"ballast_bytes"` Common common.Config `yaml:"common,omitempty"` Server server.Config `yaml:"server,omitempty"` From 6d43ca29239864b6d805ea722275271d5843d593 Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Mon, 10 Jan 2022 03:47:26 +0200 Subject: [PATCH 5/7] Add Operational scalability documentation related to memory ballast --- docs/sources/operations/scalability.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/sources/operations/scalability.md b/docs/sources/operations/scalability.md index 5c45330de49c..86edc21ae058 100644 --- a/docs/sources/operations/scalability.md +++ b/docs/sources/operations/scalability.md @@ -22,3 +22,8 @@ In order to run with the Query Scheduler, the frontend needs to be passed the sc It is not valid to start the querier with both a configured frontend and a scheduler address. The query scheduler process itself can be started via the `-target=query-scheduler` option of the Loki Docker image. For instance, `docker run grafana/loki:latest -config.file=/cortex/config/cortex.yaml -target=query-scheduler -server.http-listen-port=8009 -server.grpc-listen-port=9009` starts the query scheduler listening on ports `8009` and `9009`. + +## Memory Ballast +In compute constrained environments, garbage collection can become a significant factor. This can be optimised, at the expense of memory consumption, by configuring a memory ballast using the `ballast_bytes` configuration option. + +A larger memory ballast will mean that standard heap size volatility is less relatively significant, and therefore less likely to trigger garbage collection. This will result in lower compute overhead, but higher memory consumption, as the heap is cleaned less frequently. \ No newline at end of file From 37631af4779fd9c1e6317ca7651418ca8dc34ed9 Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Mon, 10 Jan 2022 09:55:36 +0200 Subject: [PATCH 6/7] Restore config.ballast-bytes cli flag --- pkg/loki/config_wrapper.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/loki/config_wrapper.go b/pkg/loki/config_wrapper.go index c6def028f793..03523d3b95a2 100644 --- a/pkg/loki/config_wrapper.go +++ b/pkg/loki/config_wrapper.go @@ -46,6 +46,8 @@ func (c *ConfigWrapper) RegisterFlags(f *flag.FlagSet) { "level with the order reversed, reversing the order makes viewing the entries easier in Grafana.") f.StringVar(&c.ConfigFile, "config.file", "", "yaml file to load") f.BoolVar(&c.ConfigExpandEnv, "config.expand-env", false, "Expands ${var} in config according to the values of the environment variables.") + f.IntVar(&c.BallastBytes, "config.ballast-bytes", 0, "The amount of virtual memory to reserve as a ballast in order to optimise "+ + "garbage collection. Larger ballasts result in fewer garbage collection passes, reducing compute overhead at the cost of memory usage.") c.Config.RegisterFlags(f) } From 87fb8cfc4fe3ee82b312e273182e8895151bc96c Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Mon, 10 Jan 2022 09:57:52 +0200 Subject: [PATCH 7/7] Move config.ballast-bytes cli flag declaration --- pkg/loki/config_wrapper.go | 2 -- pkg/loki/loki.go | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/loki/config_wrapper.go b/pkg/loki/config_wrapper.go index 03523d3b95a2..c6def028f793 100644 --- a/pkg/loki/config_wrapper.go +++ b/pkg/loki/config_wrapper.go @@ -46,8 +46,6 @@ func (c *ConfigWrapper) RegisterFlags(f *flag.FlagSet) { "level with the order reversed, reversing the order makes viewing the entries easier in Grafana.") f.StringVar(&c.ConfigFile, "config.file", "", "yaml file to load") f.BoolVar(&c.ConfigExpandEnv, "config.expand-env", false, "Expands ${var} in config according to the values of the environment variables.") - f.IntVar(&c.BallastBytes, "config.ballast-bytes", 0, "The amount of virtual memory to reserve as a ballast in order to optimise "+ - "garbage collection. Larger ballasts result in fewer garbage collection passes, reducing compute overhead at the cost of memory usage.") c.Config.RegisterFlags(f) } diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index bb05e572d68c..cbc74935e4ae 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -91,6 +91,8 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { "The alias 'all' can be used in the list to load a number of core modules and will enable single-binary mode. "+ "The aliases 'read' and 'write' can be used to only run components related to the read path or write path, respectively.") f.BoolVar(&c.AuthEnabled, "auth.enabled", true, "Set to false to disable auth.") + f.IntVar(&c.BallastBytes, "config.ballast-bytes", 0, "The amount of virtual memory to reserve as a ballast in order to optimise "+ + "garbage collection. Larger ballasts result in fewer garbage collection passes, reducing compute overhead at the cost of memory usage.") c.registerServerFlagsWithChangedDefaultValues(f) c.Common.RegisterFlags(f)