From f06eabbf0e2c3db3ec899c224d6c947c5edd7d6a Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Fri, 21 Jun 2024 11:56:26 +0200 Subject: [PATCH] feat: Add settings for cpu/mutex/block profiling options (#13278) The HTTP endpoints for scraping CPU, mutex and block profiles are already exposed by the server, however, block and mutex profiling must be explicitly enabled by setting a non-zero value for `SetBlockProfileRate` and `SetMutexProfileFraction` respectively. For more information about the available options consult the Go documentation: * https://pkg.go.dev/runtime#SetBlockProfileRate * https://pkg.go.dev/runtime#SetMutexProfileFraction * https://pkg.go.dev/runtime#SetCPUProfileRate Signed-off-by: Christian Haudum --- cmd/loki/main.go | 14 ++++++++++++++ docs/sources/shared/configuration.md | 21 +++++++++++++++++++++ pkg/loki/loki.go | 2 ++ pkg/loki/profiling_config.go | 21 +++++++++++++++++++++ tools/doc-generator/parse/root_blocks.go | 6 ++++++ 5 files changed, 64 insertions(+) create mode 100644 pkg/loki/profiling_config.go diff --git a/cmd/loki/main.go b/cmd/loki/main.go index 128efe6bc369..bb839c6cf3ec 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -106,6 +106,8 @@ func main() { }() } + setProfilingOptions(config.Profiling) + // Allocate a block of memory to reduce the frequency of garbage collection. // The larger the ballast, the lower the garbage collection frequency. // https://github.com/grafana/loki/issues/781 @@ -127,3 +129,15 @@ func main() { err = t.Run(loki.RunOpts{StartTime: startTime}) util_log.CheckFatal("running loki", err, util_log.Logger) } + +func setProfilingOptions(cfg loki.ProfilingConfig) { + if cfg.BlockProfileRate > 0 { + runtime.SetBlockProfileRate(cfg.BlockProfileRate) + } + if cfg.CPUProfileRate > 0 { + runtime.SetCPUProfileRate(cfg.CPUProfileRate) + } + if cfg.MutexProfileFraction > 0 { + runtime.SetMutexProfileFraction(cfg.MutexProfileFraction) + } +} diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index 00ccded5ef90..89d461541845 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -567,6 +567,9 @@ compactor_grpc_client: # Configuration for analytics. [analytics: ] +# Configuration for profiling options. +[profiling: ] + # Common configuration to be shared between multiple modules. If a more specific # configuration is given in other sections, the related configuration within # this section will be ignored. @@ -3850,6 +3853,24 @@ chunks: [row_shards: | default = 16] ``` +### profiling + +Configuration for `profiling` options. + +```yaml +# Sets the value for runtime.SetBlockProfilingRate +# CLI flag: -profiling.block-profile-rate +[block_profile_rate: | default = 0] + +# Sets the value for runtime.SetCPUProfileRate +# CLI flag: -profiling.cpu-profile-rate +[cpu_profile_rate: | default = 0] + +# Sets the value for runtime.SetMutexProfileFraction +# CLI flag: -profiling.mutex-profile-fraction +[mutex_profile_fraction: | default = 0] +``` + ### querier Configures the `querier`. Only appropriate when running all modules or just the querier. diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 68b210de4a77..0b2f2a3c9105 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -108,6 +108,7 @@ type Config struct { OperationalConfig runtime.Config `yaml:"operational_config,omitempty"` Tracing tracing.Config `yaml:"tracing"` Analytics analytics.Config `yaml:"analytics"` + Profiling ProfilingConfig `yaml:"profiling,omitempty"` LegacyReadTarget bool `yaml:"legacy_read_target,omitempty" doc:"hidden|deprecated"` @@ -179,6 +180,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.QueryScheduler.RegisterFlags(f) c.Analytics.RegisterFlags(f) c.OperationalConfig.RegisterFlags(f) + c.Profiling.RegisterFlags(f) } func (c *Config) registerServerFlagsWithChangedDefaultValues(fs *flag.FlagSet) { diff --git a/pkg/loki/profiling_config.go b/pkg/loki/profiling_config.go new file mode 100644 index 000000000000..30162f2b00bd --- /dev/null +++ b/pkg/loki/profiling_config.go @@ -0,0 +1,21 @@ +package loki + +import "flag" + +type ProfilingConfig struct { + BlockProfileRate int `yaml:"block_profile_rate"` + CPUProfileRate int `yaml:"cpu_profile_rate"` + MutexProfileFraction int `yaml:"mutex_profile_fraction"` +} + +// RegisterFlags registers flag. +func (c *ProfilingConfig) RegisterFlags(f *flag.FlagSet) { + c.RegisterFlagsWithPrefix("profiling.", f) +} + +// RegisterFlagsWithPrefix registers flag with a common prefix. +func (c *ProfilingConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.IntVar(&c.BlockProfileRate, prefix+"block-profile-rate", 0, "Sets the value for runtime.SetBlockProfilingRate") + f.IntVar(&c.CPUProfileRate, prefix+"cpu-profile-rate", 0, "Sets the value for runtime.SetCPUProfileRate") + f.IntVar(&c.MutexProfileFraction, prefix+"mutex-profile-fraction", 0, "Sets the value for runtime.SetMutexProfileFraction") +} diff --git a/tools/doc-generator/parse/root_blocks.go b/tools/doc-generator/parse/root_blocks.go index fa3da5ab28b7..85e81705848c 100644 --- a/tools/doc-generator/parse/root_blocks.go +++ b/tools/doc-generator/parse/root_blocks.go @@ -23,6 +23,7 @@ import ( "github.com/grafana/loki/v3/pkg/ingester" ingester_client "github.com/grafana/loki/v3/pkg/ingester/client" "github.com/grafana/loki/v3/pkg/loghttp/push" + "github.com/grafana/loki/v3/pkg/loki" "github.com/grafana/loki/v3/pkg/loki/common" frontend "github.com/grafana/loki/v3/pkg/lokifrontend" "github.com/grafana/loki/v3/pkg/querier" @@ -168,6 +169,11 @@ var ( StructType: []reflect.Type{reflect.TypeOf(analytics.Config{})}, Desc: "Configuration for analytics.", }, + { + Name: "profiling", + StructType: []reflect.Type{reflect.TypeOf(loki.ProfilingConfig{})}, + Desc: "Configuration for profiling options.", + }, { Name: "common",