From ce24ea94b2f3022b6c9382670132e2943b465efa Mon Sep 17 00:00:00 2001 From: Quentin McGaw Date: Fri, 19 Nov 2021 16:40:41 +0100 Subject: [PATCH] feat(pprof): Pprof HTTP server service (#1991) - Add a pprof HTTP server service to dot node - `internal/httpserver` package - `internal/pprof` package - Configurable server listening address from flags and toml config file - Configurable pprof server on/off from flags and toml config file - Configurable mutex profile fraction from flags and toml config file - Configurable block profile rate from flags and toml config file - Pprof documentation - Remove pprof to file operation and `--cpuprof` and `--memprof` flags --- chain/dev/config.toml | 6 ++ chain/dev/defaults.go | 18 ++++ chain/gssmr/config.toml | 6 ++ chain/gssmr/defaults.go | 18 ++++ chain/kusama/config.toml | 6 ++ chain/kusama/defaults.go | 18 ++++ chain/polkadot/config.toml | 8 +- chain/polkadot/defaults.go | 18 ++++ cmd/gossamer/config.go | 47 ++++++++++ cmd/gossamer/config_test.go | 3 + cmd/gossamer/export.go | 9 +- cmd/gossamer/export_test.go | 9 +- cmd/gossamer/flags.go | 28 ++++-- cmd/gossamer/main.go | 8 +- cmd/gossamer/profile.go | 80 ---------------- cmd/gossamer/utils.go | 1 + docs/docs/testing-and-debugging/pprof.md | 61 +++++++++++++ docs/docs/usage/command-line.md | 6 +- docs/docs/usage/configuration.md | 6 +- dot/config.go | 48 ++++++++++ dot/config/toml/config.go | 9 ++ dot/node.go | 10 +- dot/node_test.go | 33 +------ dot/services.go | 7 ++ dot/services_test.go | 11 +++ go.mod | 1 + go.sum | 1 + internal/httpserver/address.go | 10 ++ internal/httpserver/logger.go | 12 +++ internal/httpserver/logger_mock_test.go | 70 ++++++++++++++ internal/httpserver/matchers_test.go | 34 +++++++ internal/httpserver/run.go | 65 +++++++++++++ internal/httpserver/run_test.go | 74 +++++++++++++++ internal/httpserver/server.go | 51 +++++++++++ internal/httpserver/server_test.go | 38 ++++++++ internal/pprof/logger_mock_test.go | 70 ++++++++++++++ internal/pprof/matchers_test.go | 34 +++++++ internal/pprof/runner_mock_test.go | 47 ++++++++++ internal/pprof/server.go | 27 ++++++ internal/pprof/server_test.go | 102 +++++++++++++++++++++ internal/pprof/service.go | 64 +++++++++++++ internal/pprof/service_test.go | 111 +++++++++++++++++++++++ internal/pprof/settings.go | 31 +++++++ 43 files changed, 1177 insertions(+), 139 deletions(-) delete mode 100644 cmd/gossamer/profile.go create mode 100644 docs/docs/testing-and-debugging/pprof.md create mode 100644 internal/httpserver/address.go create mode 100644 internal/httpserver/logger.go create mode 100644 internal/httpserver/logger_mock_test.go create mode 100644 internal/httpserver/matchers_test.go create mode 100644 internal/httpserver/run.go create mode 100644 internal/httpserver/run_test.go create mode 100644 internal/httpserver/server.go create mode 100644 internal/httpserver/server_test.go create mode 100644 internal/pprof/logger_mock_test.go create mode 100644 internal/pprof/matchers_test.go create mode 100644 internal/pprof/runner_mock_test.go create mode 100644 internal/pprof/server.go create mode 100644 internal/pprof/server_test.go create mode 100644 internal/pprof/service.go create mode 100644 internal/pprof/service_test.go create mode 100644 internal/pprof/settings.go diff --git a/chain/dev/config.toml b/chain/dev/config.toml index d6023875d8..9e30f02c91 100644 --- a/chain/dev/config.toml +++ b/chain/dev/config.toml @@ -38,3 +38,9 @@ port = 8545 host = "localhost" modules = ["system", "author", "chain", "state", "rpc", "grandpa", "offchain", "childstate", "syncstate", "payment"] ws-port = 8546 + +[pprof] +enabled = false +listening-address = "localhost:6060" +block-rate = 0 +mutex-rate = 0 diff --git a/chain/dev/defaults.go b/chain/dev/defaults.go index af4bfbd146..8e0d6a42bb 100644 --- a/chain/dev/defaults.go +++ b/chain/dev/defaults.go @@ -86,3 +86,21 @@ var ( // DefaultWSEnabled enables the WS server DefaultWSEnabled = true ) + +const ( + // PprofConfig + + // DefaultPprofEnabled to indicate the pprof http server should be enabled or not. + DefaultPprofEnabled = true + + // DefaultPprofListeningAddress default pprof HTTP server listening address. + DefaultPprofListeningAddress = "localhost:6060" + + // DefaultPprofBlockRate default block profile rate. + // Set to 0 to disable profiling. + DefaultPprofBlockRate = 0 + + // DefaultPprofMutexRate default mutex profile rate. + // Set to 0 to disable profiling. + DefaultPprofMutexRate = 0 +) diff --git a/chain/gssmr/config.toml b/chain/gssmr/config.toml index c335b5003e..d75a4b8acc 100644 --- a/chain/gssmr/config.toml +++ b/chain/gssmr/config.toml @@ -39,3 +39,9 @@ port = 8545 host = "localhost" modules = ["system", "author", "chain", "state", "rpc", "grandpa", "offchain", "childstate", "syncstate", "payment"] ws-port = 8546 + +[pprof] +enabled = false +listening-address = "localhost:6060" +block-rate = 0 +mutex-rate = 0 diff --git a/chain/gssmr/defaults.go b/chain/gssmr/defaults.go index c4cfccdae3..7c3717ffb1 100644 --- a/chain/gssmr/defaults.go +++ b/chain/gssmr/defaults.go @@ -92,3 +92,21 @@ var ( // DefaultRPCWSPort rpc websocket port DefaultRPCWSPort = uint32(8546) ) + +const ( + // PprofConfig + + // DefaultPprofEnabled to indicate the pprof http server should be enabled or not. + DefaultPprofEnabled = true + + // DefaultPprofListeningAddress default pprof HTTP server listening address. + DefaultPprofListeningAddress = "localhost:6060" + + // DefaultPprofBlockRate default block profile rate. + // Set to 0 to disable profiling. + DefaultPprofBlockRate = 0 + + // DefaultPprofMutexRate default mutex profile rate. + // Set to 0 to disable profiling. + DefaultPprofMutexRate = 0 +) diff --git a/chain/kusama/config.toml b/chain/kusama/config.toml index 9a1a17e76c..37aabc8165 100644 --- a/chain/kusama/config.toml +++ b/chain/kusama/config.toml @@ -39,3 +39,9 @@ modules = ["system", "author", "chain", "state", "rpc", "grandpa", "offchain", " ws-port = 8546 ws = false ws-external = false + +[pprof] +enabled = false +listening-address = "localhost:6060" +block-rate = 0 +mutex-rate = 0 diff --git a/chain/kusama/defaults.go b/chain/kusama/defaults.go index 75caf2e2a5..256e8e5f32 100644 --- a/chain/kusama/defaults.go +++ b/chain/kusama/defaults.go @@ -78,3 +78,21 @@ var ( // DefaultRPCWSPort rpc websocket port DefaultRPCWSPort = uint32(8546) ) + +const ( + // PprofConfig + + // DefaultPprofEnabled to indicate the pprof http server should be enabled or not. + DefaultPprofEnabled = false + + // DefaultPprofListeningAddress default pprof HTTP server listening address. + DefaultPprofListeningAddress = "localhost:6060" + + // DefaultPprofBlockRate default block profile rate. + // Set to 0 to disable profiling. + DefaultPprofBlockRate = 0 + + // DefaultPprofMutexRate default mutex profile rate. + // Set to 0 to disable profiling. + DefaultPprofMutexRate = 0 +) diff --git a/chain/polkadot/config.toml b/chain/polkadot/config.toml index b44cf55a94..86dc72ad6e 100644 --- a/chain/polkadot/config.toml +++ b/chain/polkadot/config.toml @@ -35,4 +35,10 @@ enabled = false port = 8545 host = "localhost" modules = ["system", "author", "chain", "state", "rpc", "grandpa", "offchain", "childstate", "syncstate", "payment"] -ws-port = 8546 \ No newline at end of file +ws-port = 8546 + +[pprof] +enabled = false +listening-address = "localhost:6060" +block-rate = 0 +mutex-rate = 0 diff --git a/chain/polkadot/defaults.go b/chain/polkadot/defaults.go index 342337fa01..a60888fe3c 100644 --- a/chain/polkadot/defaults.go +++ b/chain/polkadot/defaults.go @@ -79,3 +79,21 @@ var ( // DefaultRPCWSPort rpc websocket port DefaultRPCWSPort = uint32(8546) ) + +const ( + // PprofConfig + + // DefaultPprofEnabled to indicate the pprof http server should be enabled or not. + DefaultPprofEnabled = false + + // DefaultPprofListeningAddress default pprof HTTP server listening address. + DefaultPprofListeningAddress = "localhost:6060" + + // DefaultPprofBlockRate default block profile rate. + // Set to 0 to disable profiling. + DefaultPprofBlockRate = 0 + + // DefaultPprofMutexRate default mutex profile rate. + // Set to 0 to disable profiling. + DefaultPprofMutexRate = 0 +) diff --git a/cmd/gossamer/config.go b/cmd/gossamer/config.go index c381499059..be4f453d58 100644 --- a/cmd/gossamer/config.go +++ b/cmd/gossamer/config.go @@ -135,6 +135,7 @@ func createDotConfig(ctx *cli.Context) (*dot.Config, error) { setDotCoreConfig(ctx, tomlCfg.Core, &cfg.Core) setDotNetworkConfig(ctx, tomlCfg.Network, &cfg.Network) setDotRPCConfig(ctx, tomlCfg.RPC, &cfg.RPC) + setDotPprofConfig(ctx, tomlCfg.Pprof, &cfg.Pprof) if rewind := ctx.GlobalInt(RewindFlag.Name); rewind != 0 { cfg.State.Rewind = rewind @@ -849,3 +850,49 @@ func updateDotConfigFromGenesisData(ctx *cli.Context, cfg *dot.Config) error { return nil } + +func setDotPprofConfig(ctx *cli.Context, tomlCfg ctoml.PprofConfig, cfg *dot.PprofConfig) { + if !cfg.Enabled { + // only allow to enable pprof from the TOML configuration. + // If it is enabled by default, it cannot be disabled. + cfg.Enabled = tomlCfg.Enabled + } + + if tomlCfg.ListeningAddress != "" { + cfg.Settings.ListeningAddress = tomlCfg.ListeningAddress + } + + if tomlCfg.BlockRate > 0 { + // block rate must be 0 (disabled) by default, since we + // cannot disable it here. + cfg.Settings.BlockProfileRate = tomlCfg.BlockRate + } + + if tomlCfg.MutexRate > 0 { + // mutex rate must be 0 (disabled) by default, since we + // cannot disable it here. + cfg.Settings.MutexProfileRate = tomlCfg.MutexRate + } + + // check --pprofserver flag and update node configuration + if enabled := ctx.GlobalBool(PprofServerFlag.Name); enabled || cfg.Enabled { + cfg.Enabled = true + } else if ctx.IsSet(PprofServerFlag.Name) && !enabled { + cfg.Enabled = false + } + + // check --pprofaddress flag and update node configuration + if address := ctx.GlobalString(PprofAddressFlag.Name); address != "" { + cfg.Settings.ListeningAddress = address + } + + if rate := ctx.GlobalInt(PprofBlockRateFlag.Name); rate > 0 { + cfg.Settings.BlockProfileRate = rate + } + + if rate := ctx.GlobalInt(PprofMutexRateFlag.Name); rate > 0 { + cfg.Settings.MutexProfileRate = rate + } + + logger.Debug("pprof configuration: " + cfg.String()) +} diff --git a/cmd/gossamer/config_test.go b/cmd/gossamer/config_test.go index 14326b87d2..f9b0d1c692 100644 --- a/cmd/gossamer/config_test.go +++ b/cmd/gossamer/config_test.go @@ -782,6 +782,7 @@ func TestUpdateConfigFromGenesisJSON(t *testing.T) { Network: testCfg.Network, RPC: testCfg.RPC, System: testCfg.System, + Pprof: testCfg.Pprof, } cfg, err := createDotConfig(ctx) @@ -836,6 +837,7 @@ func TestUpdateConfigFromGenesisJSON_Default(t *testing.T) { Network: testCfg.Network, RPC: testCfg.RPC, System: testCfg.System, + Pprof: testCfg.Pprof, } cfg, err := createDotConfig(ctx) @@ -894,6 +896,7 @@ func TestUpdateConfigFromGenesisData(t *testing.T) { }, RPC: testCfg.RPC, System: testCfg.System, + Pprof: testCfg.Pprof, } cfg, err := createDotConfig(ctx) diff --git a/cmd/gossamer/export.go b/cmd/gossamer/export.go index bf0f60c202..c99c7b89dd 100644 --- a/cmd/gossamer/export.go +++ b/cmd/gossamer/export.go @@ -57,7 +57,14 @@ func exportAction(ctx *cli.Context) error { } func dotConfigToToml(dcfg *dot.Config) *ctoml.Config { - cfg := &ctoml.Config{} + cfg := &ctoml.Config{ + Pprof: ctoml.PprofConfig{ + Enabled: dcfg.Pprof.Enabled, + ListeningAddress: dcfg.Pprof.Settings.ListeningAddress, + BlockRate: dcfg.Pprof.Settings.BlockProfileRate, + MutexRate: dcfg.Pprof.Settings.MutexProfileRate, + }, + } cfg.Global = ctoml.GlobalConfig{ Name: dcfg.Global.Name, diff --git a/cmd/gossamer/export_test.go b/cmd/gossamer/export_test.go index 4cab5cb0c8..5a58c5f656 100644 --- a/cmd/gossamer/export_test.go +++ b/cmd/gossamer/export_test.go @@ -76,7 +76,8 @@ func TestExportCommand(t *testing.T) { DiscoveryInterval: testCfg.Network.DiscoveryInterval, MinPeers: testCfg.Network.MinPeers, }, - RPC: testCfg.RPC, + RPC: testCfg.RPC, + Pprof: testCfg.Pprof, }, }, { @@ -109,7 +110,8 @@ func TestExportCommand(t *testing.T) { DiscoveryInterval: testCfg.Network.DiscoveryInterval, MinPeers: testCfg.Network.MinPeers, }, - RPC: testCfg.RPC, + RPC: testCfg.RPC, + Pprof: testCfg.Pprof, }, }, { @@ -142,7 +144,8 @@ func TestExportCommand(t *testing.T) { DiscoveryInterval: testCfg.Network.DiscoveryInterval, MinPeers: testCfg.Network.MinPeers, }, - RPC: testCfg.RPC, + RPC: testCfg.RPC, + Pprof: testCfg.Pprof, }, }, } diff --git a/cmd/gossamer/flags.go b/cmd/gossamer/flags.go index 11dc3b05e7..e2690774b4 100644 --- a/cmd/gossamer/flags.go +++ b/cmd/gossamer/flags.go @@ -97,13 +97,23 @@ var ( Name: "basepath", Usage: "Data directory for the node", } - CPUProfFlag = cli.StringFlag{ - Name: "cpuprof", - Usage: "File to write CPU profile to", + PprofServerFlag = cli.StringFlag{ + Name: "pprofserver", + Usage: "enable or disable the pprof HTTP server", } - MemProfFlag = cli.StringFlag{ - Name: "memprof", - Usage: "File to write memory profile to", + PprofAddressFlag = cli.StringFlag{ + Name: "pprofaddress", + Usage: "pprof HTTP server listening address, if it is enabled.", + } + PprofBlockRateFlag = cli.IntFlag{ + Name: "pprofblockrate", + Value: -1, + Usage: "pprof block rate. See https://pkg.go.dev/runtime#SetBlockProfileRate.", + } + PprofMutexRateFlag = cli.IntFlag{ + Name: "pprofmutexrate", + Value: -1, + Usage: "profiling mutex rate. See https://pkg.go.dev/runtime#SetMutexProfileFraction.", } // PublishMetricsFlag publishes node metrics to prometheus. @@ -372,8 +382,10 @@ var ( ChainFlag, ConfigFlag, BasePathFlag, - CPUProfFlag, - MemProfFlag, + PprofServerFlag, + PprofAddressFlag, + PprofBlockRateFlag, + PprofMutexRateFlag, RewindFlag, DBPathFlag, BloomFilterSizeFlag, diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go index f8a968531b..e868f07bf7 100644 --- a/cmd/gossamer/main.go +++ b/cmd/gossamer/main.go @@ -200,12 +200,6 @@ func gossamerAction(ctx *cli.Context) error { return fmt.Errorf("failed to read command argument: %q", arguments[0]) } - // begin profiling, if set - stopFunc, err := beginProfile(ctx) - if err != nil { - return err - } - // setup gossamer logger lvl, err := setupLogger(ctx) if err != nil { @@ -283,7 +277,7 @@ func gossamerAction(ctx *cli.Context) error { return err } - node, err := dot.NewNode(cfg, ks, stopFunc) + node, err := dot.NewNode(cfg, ks) if err != nil { logger.Errorf("failed to create node services: %s", err) return err diff --git a/cmd/gossamer/profile.go b/cmd/gossamer/profile.go deleted file mode 100644 index b3f224c9c4..0000000000 --- a/cmd/gossamer/profile.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package main - -import ( - "os" - "runtime" - "runtime/pprof" - - "github.com/urfave/cli" -) - -func beginProfile(ctx *cli.Context) (func(), error) { - cpuStopFunc, err := cpuProfile(ctx) - if err != nil { - return nil, err - } - - memStopFunc, err := memProfile(ctx) - if err != nil { - return nil, err - } - - return func() { - if cpuStopFunc != nil { - cpuStopFunc() - } - - if memStopFunc != nil { - memStopFunc() - } - }, nil -} - -func cpuProfile(ctx *cli.Context) (func(), error) { - cpuProfFile := ctx.GlobalString(CPUProfFlag.Name) - if cpuProfFile == "" { - return nil, nil - } - - cpuFile, err := os.Create(cpuProfFile) - if err != nil { - return nil, err - } - - if err := pprof.StartCPUProfile(cpuFile); err != nil { - return nil, err - } - - return func() { - pprof.StopCPUProfile() - if err := cpuFile.Close(); err != nil { - logger.Error("failed to close file " + cpuFile.Name()) - } - }, nil -} - -func memProfile(ctx *cli.Context) (func(), error) { - memProfFile := ctx.GlobalString(MemProfFlag.Name) - if memProfFile == "" { - return nil, nil - } - - memFile, err := os.Create(memProfFile) - if err != nil { - return nil, err - } - - return func() { - runtime.GC() - if err := pprof.WriteHeapProfile(memFile); err != nil { - logger.Errorf("could not write memory profile: %s", err) - } - - if err := memFile.Close(); err != nil { - logger.Error("failed to close file " + memFile.Name()) - } - }, nil -} diff --git a/cmd/gossamer/utils.go b/cmd/gossamer/utils.go index 8c3567f479..d464e9d43c 100644 --- a/cmd/gossamer/utils.go +++ b/cmd/gossamer/utils.go @@ -98,6 +98,7 @@ func newTestConfig(t *testing.T) *dot.Config { Network: dot.GssmrConfig().Network, RPC: dot.GssmrConfig().RPC, System: dot.GssmrConfig().System, + Pprof: dot.GssmrConfig().Pprof, } return cfg diff --git a/docs/docs/testing-and-debugging/pprof.md b/docs/docs/testing-and-debugging/pprof.md new file mode 100644 index 0000000000..68464e84b1 --- /dev/null +++ b/docs/docs/testing-and-debugging/pprof.md @@ -0,0 +1,61 @@ +--- +layout: default +title: Pprof +permalink: /testing-and-debugging/pprof +--- + +## Pprof + +There is a built-in pprof server to faciliate profiling the program. +You can enable it with the flag `--pprofserver` or by modifying the TOML configuration file. + +Note it does not affect performance unless the server is queried. + +We assume Gossamer runs on `localhost` and the Pprof server is listening +on the default `6060` port. You can configure the Pprof server listening address with the pprof TOML key `listening-address` or the flag `--pprofaddress`. + +You need to have [Go](https://golang.org/dl/) installed to profile the program. + +### Browser + +The easiest way to visualize profiling data is through your browser. + +For example, the following commands will show interactive results at [http://localhost:8000](http://localhost:8000): + +```sh +# Check the heap +go tool pprof -http=localhost:8000 http://localhost:6060/debug/pprof/heap +# Check the CPU time spent for 10 seconds +go tool pprof -http=localhost:8000 http://localhost:6060/debug/pprof/profile?seconds=10 +``` + +### Compare heaps + +You can compare heaps with Go's pprof, this is especially useful to find memory leaks. + +1. Download the first heap profile `wget -qO heap.1.out http://localhost:6060/debug/pprof/heap` +1. Download the second heap profile `wget -qO heap.2.out http://localhost:6060/debug/pprof/heap` +1. Compare the second heap profile with the first one using `go tool pprof -base ./heap.1.out heap.2.out` + +### More routes + +More routes are available in the HTTP pprof server. You can also list them at [http://localhost:6060/debug/pprof/](http://localhost:6060/debug/pprof/). +Notable ones are written below: + +#### Goroutine blocking profile + +The route `/debug/pprof/block` is available but requires to set the block profile rate, using either the toml value `block-rate` or the flag `--pprofblockrate`. + +#### Mutex contention profile + +The route `/debug/pprof/mutex` is available but requires to set the mutex profile rate, using either the toml value `mutex-rate` or the flag `--pprofmutexrate`. + +#### Other routes + +The other routes are listed below, if you need them: + +- `/debug/pprof/cmdline` +- `/debug/pprof/symbol` +- `/debug/pprof/trace` +- `/debug/pprof/goroutine` +- `/debug/pprof/threadcreate` diff --git a/docs/docs/usage/command-line.md b/docs/docs/usage/command-line.md index 7adef34ed7..4ccb57d7b1 100644 --- a/docs/docs/usage/command-line.md +++ b/docs/docs/usage/command-line.md @@ -26,11 +26,13 @@ The global flags can be used in conjunction with any Gossamer command --basepath value Data directory for the node --chain value Node implementation id used to load default node configuration --config value TOML configuration file ---cpuprof File to write CPU profile to --log value Supports levels crit (silent) to trce (trace) (default: "info") ---memprof File to write memory profile to --name value Node implementation name --rewind value Rewind head of chain by given number of blocks +--pprofserver Enable or disable the pprof HTTP server +--pprofaddress pprof HTTP server listening address, if it is enabled. +--pprofblockrate pprof block rate. See https://pkg.go.dev/runtime#SetBlockProfileRate. +--pprofmutexrate profiling mutex rate. See https://pkg.go.dev/runtime#SetMutexProfileFraction. ``` ### Local flags diff --git a/docs/docs/usage/configuration.md b/docs/docs/usage/configuration.md index 664b745f1e..99c0230d18 100644 --- a/docs/docs/usage/configuration.md +++ b/docs/docs/usage/configuration.md @@ -14,9 +14,11 @@ Gossamer consumes a `.toml` file containing predefined settings for the node fro [global] basepath = "~/.gossamer/gssmr" log = " | trace | debug | info | warn | error | crit" -cpuprof = "~/cpuprof.out" -memprof = "~/memprof.out" name = "gssmr" +pprofserver = false +pprofaddress = ":6060" +pprofblockrate = 0 +pprofmutexrate = 0 [log] core = " | trace | debug | info | warn | error | crit" diff --git a/dot/config.go b/dot/config.go index cb2a600397..9fde0fb4a1 100644 --- a/dot/config.go +++ b/dot/config.go @@ -15,6 +15,7 @@ import ( "github.com/ChainSafe/gossamer/dot/state/pruner" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/log" + "github.com/ChainSafe/gossamer/internal/pprof" "github.com/ChainSafe/gossamer/lib/genesis" ) @@ -32,6 +33,7 @@ type Config struct { RPC RPCConfig System types.SystemInfo State StateConfig + Pprof PprofConfig } // GlobalConfig is used for every node command @@ -146,6 +148,20 @@ func networkServiceEnabled(cfg *Config) bool { return cfg.Core.Roles != byte(0) } +// PprofConfig is the configuration for the pprof HTTP server. +type PprofConfig struct { + Enabled bool + Settings pprof.Settings +} + +func (p PprofConfig) String() string { + if !p.Enabled { + return "disabled" + } + + return p.Settings.String() +} + // GssmrConfig returns a new test configuration using the provided basepath func GssmrConfig() *Config { return &Config{ @@ -197,6 +213,14 @@ func GssmrConfig() *Config { Modules: gssmr.DefaultRPCModules, WSPort: gssmr.DefaultRPCWSPort, }, + Pprof: PprofConfig{ + Enabled: gssmr.DefaultPprofEnabled, + Settings: pprof.Settings{ + ListeningAddress: gssmr.DefaultPprofListeningAddress, + BlockProfileRate: gssmr.DefaultPprofBlockRate, + MutexProfileRate: gssmr.DefaultPprofMutexRate, + }, + }, } } @@ -246,6 +270,14 @@ func KusamaConfig() *Config { Modules: kusama.DefaultRPCModules, WSPort: kusama.DefaultRPCWSPort, }, + Pprof: PprofConfig{ + Enabled: kusama.DefaultPprofEnabled, + Settings: pprof.Settings{ + ListeningAddress: kusama.DefaultPprofListeningAddress, + BlockProfileRate: kusama.DefaultPprofBlockRate, + MutexProfileRate: kusama.DefaultPprofMutexRate, + }, + }, } } @@ -295,6 +327,14 @@ func PolkadotConfig() *Config { Modules: polkadot.DefaultRPCModules, WSPort: polkadot.DefaultRPCWSPort, }, + Pprof: PprofConfig{ + Enabled: polkadot.DefaultPprofEnabled, + Settings: pprof.Settings{ + ListeningAddress: polkadot.DefaultPprofListeningAddress, + BlockProfileRate: polkadot.DefaultPprofBlockRate, + MutexProfileRate: polkadot.DefaultPprofMutexRate, + }, + }, } } @@ -349,5 +389,13 @@ func DevConfig() *Config { Enabled: dev.DefaultRPCEnabled, WS: dev.DefaultWSEnabled, }, + Pprof: PprofConfig{ + Enabled: dev.DefaultPprofEnabled, + Settings: pprof.Settings{ + ListeningAddress: dev.DefaultPprofListeningAddress, + BlockProfileRate: dev.DefaultPprofBlockRate, + MutexProfileRate: dev.DefaultPprofMutexRate, + }, + }, } } diff --git a/dot/config/toml/config.go b/dot/config/toml/config.go index 7ec4cd6b69..7d5287f53e 100644 --- a/dot/config/toml/config.go +++ b/dot/config/toml/config.go @@ -12,6 +12,7 @@ type Config struct { Core CoreConfig `toml:"core,omitempty"` Network NetworkConfig `toml:"network,omitempty"` RPC RPCConfig `toml:"rpc,omitempty"` + Pprof PprofConfig `toml:"pprof,omitempty"` } // GlobalConfig is to marshal/unmarshal toml global config vars @@ -88,3 +89,11 @@ type RPCConfig struct { WSUnsafe bool `toml:"ws-unsafe,omitempty"` WSUnsafeExternal bool `toml:"ws-unsafe-external,omitempty"` } + +// PprofConfig contains the configuration for Pprof. +type PprofConfig struct { + Enabled bool `toml:"enabled,omitempty"` + ListeningAddress string `toml:"listening-address,omitempty"` + BlockRate int `toml:"block-rate,omitempty"` + MutexRate int `toml:"mutex-rate,omitempty"` +} diff --git a/dot/node.go b/dot/node.go index 421d51e073..caaed8fa64 100644 --- a/dot/node.go +++ b/dot/node.go @@ -37,7 +37,6 @@ var logger = log.NewFromGlobal(log.AddContext("pkg", "dot")) type Node struct { Name string Services *services.ServiceRegistry // registry of all node services - StopFunc func() // func to call when node stops, currently used for profiling wg sync.WaitGroup started chan struct{} } @@ -172,7 +171,7 @@ func LoadGlobalNodeName(basepath string) (nodename string, err error) { } // NewNode creates a new dot node from a dot node configuration -func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, error) { +func NewNode(cfg *Config, ks *keystore.GlobalKeystore) (*Node, error) { // set garbage collection percent to 10% // can be overwritten by setting the GOGC env variable, which defaults to 100 prev := debug.SetGCPercent(10) @@ -196,6 +195,8 @@ func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, networkSrvc *network.Service ) + nodeSrvcs = append(nodeSrvcs, createPprofService(cfg.Pprof.Settings)) + stateSrvc, err := createStateService(cfg) if err != nil { return nil, fmt.Errorf("failed to create state service: %s", err) @@ -289,7 +290,6 @@ func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, serviceRegistryLogger := logger.New(log.AddContext("pkg", "services")) node := &Node{ Name: cfg.Global.Name, - StopFunc: stopFunc, Services: services.NewServiceRegistry(serviceRegistryLogger), started: make(chan struct{}), } @@ -395,10 +395,6 @@ func (n *Node) Start() error { // Stop stops all dot node services func (n *Node) Stop() { - if n.StopFunc != nil { - n.StopFunc() - } - // stop all node services n.Services.StopAll() n.wg.Done() diff --git a/dot/node_test.go b/dot/node_test.go index 8ad4e55367..c3df7258b5 100644 --- a/dot/node_test.go +++ b/dot/node_test.go @@ -4,10 +4,8 @@ package dot import ( - "io" "math/big" "reflect" - "sync" "testing" "github.com/ChainSafe/gossamer/dot/core" @@ -19,7 +17,6 @@ import ( "github.com/ChainSafe/gossamer/lib/genesis" "github.com/ChainSafe/gossamer/lib/grandpa" "github.com/ChainSafe/gossamer/lib/keystore" - "github.com/ChainSafe/gossamer/lib/services" "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" @@ -102,7 +99,7 @@ func TestNewNode(t *testing.T) { cfg.Core.Roles = types.FullNodeRole - node, err := NewNode(cfg, ks, nil) + node, err := NewNode(cfg, ks) require.NoError(t, err) bp := node.Services.Get(&babe.Service{}) @@ -135,7 +132,7 @@ func TestNewNode_Authority(t *testing.T) { cfg.Core.Roles = types.AuthorityRole - node, err := NewNode(cfg, ks, nil) + node, err := NewNode(cfg, ks) require.NoError(t, err) bp := node.Services.Get(&babe.Service{}) @@ -168,7 +165,7 @@ func TestStartNode(t *testing.T) { cfg.Core.Roles = types.FullNodeRole - node, err := NewNode(cfg, ks, nil) + node, err := NewNode(cfg, ks) require.NoError(t, err) go func() { @@ -270,7 +267,7 @@ func TestInitNode_LoadStorageRoot(t *testing.T) { ks.Gran.Insert(ed25519Keyring.Alice()) sr25519Keyring, _ := keystore.NewSr25519Keyring() ks.Babe.Insert(sr25519Keyring.Alice()) - node, err := NewNode(cfg, ks, nil) + node, err := NewNode(cfg, ks) require.NoError(t, err) if reflect.TypeOf(node) != reflect.TypeOf(&Node{}) { @@ -328,7 +325,7 @@ func TestInitNode_LoadBalances(t *testing.T) { ed25519Keyring, _ := keystore.NewEd25519Keyring() ks.Gran.Insert(ed25519Keyring.Alice()) - node, err := NewNode(cfg, ks, nil) + node, err := NewNode(cfg, ks) require.NoError(t, err) if reflect.TypeOf(node) != reflect.TypeOf(&Node{}) { @@ -356,26 +353,6 @@ func TestInitNode_LoadBalances(t *testing.T) { require.Equal(t, expected, bal) } -func TestNode_StopFunc(t *testing.T) { - testvar := "before" - stopFunc := func() { - testvar = "after" - } - - serviceRegistryLogger := log.New(log.SetWriter(io.Discard)) - servicesRegistry := services.NewServiceRegistry(serviceRegistryLogger) - - node := &Node{ - Services: servicesRegistry, - StopFunc: stopFunc, - wg: sync.WaitGroup{}, - } - node.wg.Add(1) - - node.Stop() - require.Equal(t, testvar, "after") -} - func TestNode_PersistGlobalName_WhenInitialize(t *testing.T) { globalName := RandomNodeName() diff --git a/dot/services.go b/dot/services.go index 245dcc1271..2f1ae51064 100644 --- a/dot/services.go +++ b/dot/services.go @@ -19,6 +19,8 @@ import ( "github.com/ChainSafe/gossamer/dot/sync" "github.com/ChainSafe/gossamer/dot/system" "github.com/ChainSafe/gossamer/dot/types" + "github.com/ChainSafe/gossamer/internal/log" + "github.com/ChainSafe/gossamer/internal/pprof" "github.com/ChainSafe/gossamer/lib/babe" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/crypto" @@ -409,3 +411,8 @@ func newSyncService(cfg *Config, st *state.Service, fg sync.FinalityGadget, veri func createDigestHandler(st *state.Service) (*digest.Handler, error) { return digest.NewHandler(st.Block, st.Epoch, st.Grandpa) } + +func createPprofService(settings pprof.Settings) (service *pprof.Service) { + pprofLogger := log.NewFromGlobal(log.AddContext("pkg", "pprof")) + return pprof.NewService(settings, pprofLogger) +} diff --git a/dot/services_test.go b/dot/services_test.go index a2e34bec3b..1e4b7924b7 100644 --- a/dot/services_test.go +++ b/dot/services_test.go @@ -11,6 +11,7 @@ import ( "github.com/ChainSafe/gossamer/dot/network" "github.com/ChainSafe/gossamer/dot/types" + "github.com/ChainSafe/gossamer/internal/pprof" "github.com/ChainSafe/gossamer/lib/grandpa" "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/utils" @@ -355,3 +356,13 @@ func TestNewWebSocketServer(t *testing.T) { require.Equal(t, item.expected, message) } } + +func Test_createPprofService(t *testing.T) { + t.Parallel() + + settings := pprof.Settings{} + + service := createPprofService(settings) + + require.NotNil(t, service) +} diff --git a/go.mod b/go.mod index e484eee5bc..83a0567d4b 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/ethereum/go-ethereum v1.10.12 github.com/fatih/color v1.13.0 github.com/go-playground/validator/v10 v10.9.0 + github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.2 github.com/google/go-cmp v0.5.6 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index f4a5053f36..8ed3fc791d 100644 --- a/go.sum +++ b/go.sum @@ -351,6 +351,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= diff --git a/internal/httpserver/address.go b/internal/httpserver/address.go new file mode 100644 index 0000000000..1ce9d46483 --- /dev/null +++ b/internal/httpserver/address.go @@ -0,0 +1,10 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package httpserver + +// GetAddress obtains the address the HTTP server is listening on. +func (s *Server) GetAddress() (address string) { + <-s.addressSet + return s.address +} diff --git a/internal/httpserver/logger.go b/internal/httpserver/logger.go new file mode 100644 index 0000000000..9d2ef97897 --- /dev/null +++ b/internal/httpserver/logger.go @@ -0,0 +1,12 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package httpserver + +// Logger is the logger interface accepted by the +// HTTP server. +type Logger interface { + Info(msg string) + Warn(msg string) + Error(msg string) +} diff --git a/internal/httpserver/logger_mock_test.go b/internal/httpserver/logger_mock_test.go new file mode 100644 index 0000000000..790ad0a28f --- /dev/null +++ b/internal/httpserver/logger_mock_test.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/ChainSafe/gossamer/internal/httpserver (interfaces: Logger) + +// Package httpserver is a generated GoMock package. +package httpserver + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockLogger is a mock of Logger interface. +type MockLogger struct { + ctrl *gomock.Controller + recorder *MockLoggerMockRecorder +} + +// MockLoggerMockRecorder is the mock recorder for MockLogger. +type MockLoggerMockRecorder struct { + mock *MockLogger +} + +// NewMockLogger creates a new mock instance. +func NewMockLogger(ctrl *gomock.Controller) *MockLogger { + mock := &MockLogger{ctrl: ctrl} + mock.recorder = &MockLoggerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLogger) EXPECT() *MockLoggerMockRecorder { + return m.recorder +} + +// Error mocks base method. +func (m *MockLogger) Error(arg0 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Error", arg0) +} + +// Error indicates an expected call of Error. +func (mr *MockLoggerMockRecorder) Error(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Error", reflect.TypeOf((*MockLogger)(nil).Error), arg0) +} + +// Info mocks base method. +func (m *MockLogger) Info(arg0 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Info", arg0) +} + +// Info indicates an expected call of Info. +func (mr *MockLoggerMockRecorder) Info(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Info", reflect.TypeOf((*MockLogger)(nil).Info), arg0) +} + +// Warn mocks base method. +func (m *MockLogger) Warn(arg0 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Warn", arg0) +} + +// Warn indicates an expected call of Warn. +func (mr *MockLoggerMockRecorder) Warn(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Warn", reflect.TypeOf((*MockLogger)(nil).Warn), arg0) +} diff --git a/internal/httpserver/matchers_test.go b/internal/httpserver/matchers_test.go new file mode 100644 index 0000000000..ec3e9b6a64 --- /dev/null +++ b/internal/httpserver/matchers_test.go @@ -0,0 +1,34 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package httpserver + +import ( + "regexp" + + gomock "github.com/golang/mock/gomock" +) + +var _ gomock.Matcher = (*regexMatcher)(nil) + +type regexMatcher struct { + regexp *regexp.Regexp +} + +func (r *regexMatcher) Matches(x interface{}) bool { + s, ok := x.(string) + if !ok { + return false + } + return r.regexp.MatchString(s) +} + +func (r *regexMatcher) String() string { + return "regular expression " + r.regexp.String() +} + +func newRegexMatcher(regex string) *regexMatcher { + return ®exMatcher{ + regexp: regexp.MustCompile(regex), + } +} diff --git a/internal/httpserver/run.go b/internal/httpserver/run.go new file mode 100644 index 0000000000..2e5c8539ec --- /dev/null +++ b/internal/httpserver/run.go @@ -0,0 +1,65 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package httpserver + +import ( + "context" + "errors" + "net" + "net/http" + "time" +) + +// Run runs the HTTP server until ctx is canceled. +// The done channel has an error written to when the HTTP server +// is terminated, and can be nil or not nil. +func (s *Server) Run(ctx context.Context, ready chan<- struct{}, done chan<- error) { + server := http.Server{Addr: s.address, Handler: s.handler} + + crashed := make(chan struct{}) + shutdownDone := make(chan struct{}) + go func() { + defer close(shutdownDone) + select { + case <-ctx.Done(): + case <-crashed: + return + } + + s.logger.Warn(s.name + " http server shutting down: " + ctx.Err().Error()) + const shutdownGraceDuration = 3 * time.Second + shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownGraceDuration) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil { + s.logger.Error(s.name + " http server failed shutting down: " + err.Error()) + } + }() + + listener, err := net.Listen("tcp", s.address) + if err != nil { + close(s.addressSet) + close(crashed) // stop shutdown goroutine + <-shutdownDone + done <- err + return + } + + s.address = listener.Addr().String() + close(s.addressSet) + + // note: no further write so no need to mutex + s.logger.Info(s.name + " http server listening on " + s.address) + close(ready) + + err = server.Serve(listener) + + if err != nil && !errors.Is(ctx.Err(), context.Canceled) { + // server crashed + close(crashed) // stop shutdown goroutine + } else { + err = nil + } + <-shutdownDone + done <- err +} diff --git a/internal/httpserver/run_test.go b/internal/httpserver/run_test.go new file mode 100644 index 0000000000..424f8c0b0e --- /dev/null +++ b/internal/httpserver/run_test.go @@ -0,0 +1,74 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package httpserver + +import ( + "context" + "regexp" + "testing" + + gomock "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +func Test_Server_Run_success(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + + logger := NewMockLogger(ctrl) + logger.EXPECT().Info(newRegexMatcher("^test http server listening on 127.0.0.1:[1-9][0-9]{0,4}$")) + logger.EXPECT().Warn("test http server shutting down: context canceled") + + server := &Server{ + name: "test", + address: "127.0.0.1:0", + addressSet: make(chan struct{}), + logger: logger, + } + + ctx, cancel := context.WithCancel(context.Background()) + ready := make(chan struct{}) + done := make(chan error) + + go server.Run(ctx, ready, done) + + addressRegex := regexp.MustCompile(`^127.0.0.1:[1-9][0-9]{0,4}$`) + address := server.GetAddress() + assert.Regexp(t, addressRegex, address) + address = server.GetAddress() + assert.Regexp(t, addressRegex, address) + + <-ready + + cancel() + err := <-done + assert.NoError(t, err) +} + +func Test_Server_Run_failure(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + + logger := NewMockLogger(ctrl) + + server := &Server{ + name: "test", + address: "127.0.0.1:-1", + addressSet: make(chan struct{}), + logger: logger, + } + + ready := make(chan struct{}) + done := make(chan error) + + go server.Run(context.Background(), ready, done) + + select { + case <-ready: + t.Fatal("server should not be ready") + case err := <-done: + assert.EqualError(t, err, "listen tcp: address -1: invalid port") + } +} diff --git a/internal/httpserver/server.go b/internal/httpserver/server.go new file mode 100644 index 0000000000..1f26e135e2 --- /dev/null +++ b/internal/httpserver/server.go @@ -0,0 +1,51 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +// Package httpserver implements an HTTP server. +package httpserver + +import ( + "context" + "net/http" +) + +var _ Interface = (*Server)(nil) + +// Interface is the HTTP server interface +type Interface interface { + Runner + AddressGetter +} + +// Runner is the interface for an HTTP server with a Run method. +type Runner interface { + Run(ctx context.Context, ready chan<- struct{}, done chan<- error) +} + +// AddressGetter obtains the address the HTTP server is listening on. +type AddressGetter interface { + GetAddress() (address string) +} + +// Server is an HTTP server implementation, which uses +// the HTTP handler provided. +type Server struct { + name string + address string + addressSet chan struct{} + handler http.Handler + logger Logger +} + +// New creates a new HTTP server with a name, listening on +// the address specified and using the HTTP handler provided. +func New(name, address string, handler http.Handler, + logger Logger) *Server { + return &Server{ + name: name, + address: address, + addressSet: make(chan struct{}), + handler: handler, + logger: logger, + } +} diff --git a/internal/httpserver/server_test.go b/internal/httpserver/server_test.go new file mode 100644 index 0000000000..bfe2be6977 --- /dev/null +++ b/internal/httpserver/server_test.go @@ -0,0 +1,38 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package httpserver + +import ( + "net/http" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +//go:generate mockgen -destination=logger_mock_test.go -package $GOPACKAGE . Logger + +func Test_New(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + + const name = "name" + const address = "test" + handler := http.NewServeMux() + logger := NewMockLogger(ctrl) + + expectedServer := &Server{ + name: name, + address: address, + handler: handler, + logger: logger, + } + + server := New(name, address, handler, logger) + + assert.NotNil(t, server.addressSet) + server.addressSet = nil + + assert.Equal(t, expectedServer, server) +} diff --git a/internal/pprof/logger_mock_test.go b/internal/pprof/logger_mock_test.go new file mode 100644 index 0000000000..02edbbfbe3 --- /dev/null +++ b/internal/pprof/logger_mock_test.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/ChainSafe/gossamer/internal/httpserver (interfaces: Logger) + +// Package pprof is a generated GoMock package. +package pprof + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockLogger is a mock of Logger interface. +type MockLogger struct { + ctrl *gomock.Controller + recorder *MockLoggerMockRecorder +} + +// MockLoggerMockRecorder is the mock recorder for MockLogger. +type MockLoggerMockRecorder struct { + mock *MockLogger +} + +// NewMockLogger creates a new mock instance. +func NewMockLogger(ctrl *gomock.Controller) *MockLogger { + mock := &MockLogger{ctrl: ctrl} + mock.recorder = &MockLoggerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLogger) EXPECT() *MockLoggerMockRecorder { + return m.recorder +} + +// Error mocks base method. +func (m *MockLogger) Error(arg0 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Error", arg0) +} + +// Error indicates an expected call of Error. +func (mr *MockLoggerMockRecorder) Error(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Error", reflect.TypeOf((*MockLogger)(nil).Error), arg0) +} + +// Info mocks base method. +func (m *MockLogger) Info(arg0 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Info", arg0) +} + +// Info indicates an expected call of Info. +func (mr *MockLoggerMockRecorder) Info(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Info", reflect.TypeOf((*MockLogger)(nil).Info), arg0) +} + +// Warn mocks base method. +func (m *MockLogger) Warn(arg0 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Warn", arg0) +} + +// Warn indicates an expected call of Warn. +func (mr *MockLoggerMockRecorder) Warn(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Warn", reflect.TypeOf((*MockLogger)(nil).Warn), arg0) +} diff --git a/internal/pprof/matchers_test.go b/internal/pprof/matchers_test.go new file mode 100644 index 0000000000..4443e64d23 --- /dev/null +++ b/internal/pprof/matchers_test.go @@ -0,0 +1,34 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package pprof + +import ( + "regexp" + + gomock "github.com/golang/mock/gomock" +) + +var _ gomock.Matcher = (*regexMatcher)(nil) + +type regexMatcher struct { + regexp *regexp.Regexp +} + +func (r *regexMatcher) Matches(x interface{}) bool { + s, ok := x.(string) + if !ok { + return false + } + return r.regexp.MatchString(s) +} + +func (r *regexMatcher) String() string { + return "regular expression " + r.regexp.String() +} + +func newRegexMatcher(regex string) *regexMatcher { + return ®exMatcher{ + regexp: regexp.MustCompile(regex), + } +} diff --git a/internal/pprof/runner_mock_test.go b/internal/pprof/runner_mock_test.go new file mode 100644 index 0000000000..556715c5c6 --- /dev/null +++ b/internal/pprof/runner_mock_test.go @@ -0,0 +1,47 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/ChainSafe/gossamer/internal/httpserver (interfaces: Runner) + +// Package pprof is a generated GoMock package. +package pprof + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockRunner is a mock of Runner interface. +type MockRunner struct { + ctrl *gomock.Controller + recorder *MockRunnerMockRecorder +} + +// MockRunnerMockRecorder is the mock recorder for MockRunner. +type MockRunnerMockRecorder struct { + mock *MockRunner +} + +// NewMockRunner creates a new mock instance. +func NewMockRunner(ctrl *gomock.Controller) *MockRunner { + mock := &MockRunner{ctrl: ctrl} + mock.recorder = &MockRunnerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRunner) EXPECT() *MockRunnerMockRecorder { + return m.recorder +} + +// Run mocks base method. +func (m *MockRunner) Run(arg0 context.Context, arg1 chan<- struct{}, arg2 chan<- error) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Run", arg0, arg1, arg2) +} + +// Run indicates an expected call of Run. +func (mr *MockRunnerMockRecorder) Run(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockRunner)(nil).Run), arg0, arg1, arg2) +} diff --git a/internal/pprof/server.go b/internal/pprof/server.go new file mode 100644 index 0000000000..a097fd4d6d --- /dev/null +++ b/internal/pprof/server.go @@ -0,0 +1,27 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package pprof + +import ( + "net/http" + "net/http/pprof" + + "github.com/ChainSafe/gossamer/internal/httpserver" +) + +// NewServer creates a new Pprof server which will listen at +// the address specified. +func NewServer(address string, logger httpserver.Logger) *httpserver.Server { + handler := http.NewServeMux() + handler.HandleFunc("/debug/pprof/", pprof.Index) + handler.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + handler.HandleFunc("/debug/pprof/profile", pprof.Profile) + handler.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + handler.HandleFunc("/debug/pprof/trace", pprof.Trace) + handler.Handle("/debug/pprof/block", pprof.Handler("block")) + handler.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) + handler.Handle("/debug/pprof/heap", pprof.Handler("heap")) + handler.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) + return httpserver.New("pprof", address, handler, logger) +} diff --git a/internal/pprof/server_test.go b/internal/pprof/server_test.go new file mode 100644 index 0000000000..1f51309567 --- /dev/null +++ b/internal/pprof/server_test.go @@ -0,0 +1,102 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package pprof + +import ( + "context" + "io/ioutil" + "net/http" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +//go:generate mockgen -destination=logger_mock_test.go -package $GOPACKAGE github.com/ChainSafe/gossamer/internal/httpserver Logger + +func Test_Server(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + + const address = "127.0.0.1:0" + logger := NewMockLogger(ctrl) + + logger.EXPECT().Info(newRegexMatcher("^pprof http server listening on 127.0.0.1:[1-9][0-9]{0,4}$")) + logger.EXPECT().Warn("pprof http server shutting down: context canceled") + + server := NewServer(address, logger) + require.NotNil(t, server) + + ctx, cancel := context.WithCancel(context.Background()) + ready := make(chan struct{}) + done := make(chan error) + + go server.Run(ctx, ready, done) + + select { + case <-ready: + case err := <-done: + t.Fatalf("server crashed before being ready: %s", err) + } + + serverAddress := server.GetAddress() + + const clientTimeout = 2 * time.Second + httpClient := &http.Client{Timeout: clientTimeout} + + pathsToCheck := []string{ + "debug/pprof/", + "debug/pprof/cmdline", + "debug/pprof/profile?seconds=1", + "debug/pprof/symbol", + "debug/pprof/trace?seconds=0", + "debug/pprof/block", + "debug/pprof/goroutine", + "debug/pprof/heap", + "debug/pprof/threadcreate", + } + + type httpResult struct { + url string + response *http.Response + err error + } + results := make(chan httpResult) + + for _, pathToCheck := range pathsToCheck { + url := "http://" + serverAddress + "/" + pathToCheck + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + require.NoError(t, err) + + go func(client *http.Client, request *http.Request, results chan<- httpResult) { + response, err := client.Do(request) //nolint:bodyclose + results <- httpResult{ + url: request.URL.String(), + response: response, + err: err, + } + }(httpClient, request, results) + } + + for range pathsToCheck { + httpResult := <-results + + require.NoErrorf(t, httpResult.err, "unexpected error for URL %s: %s", httpResult.url, httpResult.err) + assert.Equalf(t, http.StatusOK, httpResult.response.StatusCode, + "unexpected status code for URL %s: %s", httpResult.url, http.StatusText(httpResult.response.StatusCode)) + + b, err := ioutil.ReadAll(httpResult.response.Body) + require.NoErrorf(t, err, "unexpected error for URL %s: %s", httpResult.url, err) + assert.NotEmptyf(t, b, "response body is empty for URL %s", httpResult.url) + + err = httpResult.response.Body.Close() + assert.NoErrorf(t, err, "unexpected error for URL %s: %s", httpResult.url, err) + } + + cancel() + <-done +} diff --git a/internal/pprof/service.go b/internal/pprof/service.go new file mode 100644 index 0000000000..932f3c2f8a --- /dev/null +++ b/internal/pprof/service.go @@ -0,0 +1,64 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package pprof + +import ( + "context" + "errors" + "runtime" + + "github.com/ChainSafe/gossamer/internal/httpserver" +) + +// Service is a pprof http server service compatible with the +// dot/service.go interface. +type Service struct { + settings Settings + server httpserver.Runner + cancel context.CancelFunc + done chan error +} + +// NewService creates a pprof server service compatible with the +// dot/service.go interface. +func NewService(settings Settings, logger httpserver.Logger) *Service { + settings.setDefaults() + + return &Service{ + settings: settings, + server: NewServer(settings.ListeningAddress, logger), + done: make(chan error), + } +} + +var ErrServerDoneBeforeReady = errors.New("server terminated before being ready") + +// Start starts the pprof server service. +func (s *Service) Start() (err error) { + runtime.SetBlockProfileRate(s.settings.BlockProfileRate) + runtime.SetMutexProfileFraction(s.settings.MutexProfileRate) + + ctx, cancel := context.WithCancel(context.Background()) + s.cancel = cancel + ready := make(chan struct{}) + + go s.server.Run(ctx, ready, s.done) + + select { + case <-ready: + return nil + case err := <-s.done: + close(s.done) + if err != nil { + return err + } + return ErrServerDoneBeforeReady + } +} + +// Stop stops the pprof server service. +func (s *Service) Stop() (err error) { + s.cancel() + return <-s.done +} diff --git a/internal/pprof/service_test.go b/internal/pprof/service_test.go new file mode 100644 index 0000000000..30a927705b --- /dev/null +++ b/internal/pprof/service_test.go @@ -0,0 +1,111 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package pprof + +import ( + "context" + "errors" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_NewService(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + + settings := Settings{} + logger := NewMockLogger(ctrl) + + service := NewService(settings, logger) + + expectedSettings := Settings{ + ListeningAddress: "localhost:6060", + } + assert.Equal(t, expectedSettings, service.settings) + assert.NotNil(t, service.server) + assert.NotNil(t, service.done) +} + +//go:generate mockgen -destination=runner_mock_test.go -package $GOPACKAGE github.com/ChainSafe/gossamer/internal/httpserver Runner + +func Test_Service_StartStop_success(t *testing.T) { + t.Parallel() + + errDummy := errors.New("dummy") + + testCases := map[string]struct { + startDone bool + startDoneErr error + startErr error + stopDoneErr error + stopErr error + }{ + "start nil error": { + startDone: true, + startErr: ErrServerDoneBeforeReady, + }, + "start error": { + startDone: true, + startDoneErr: errDummy, + startErr: errDummy, + }, + "stop error": { + stopDoneErr: errDummy, + stopErr: errDummy, + }, + "success": {}, + } + + for name, testCase := range testCases { + testCase := testCase + t.Run(name, func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + + server := NewMockRunner(ctrl) + ctxType, cancelType := context.WithCancel(context.Background()) + defer cancelType() + server.EXPECT().Run( + gomock.AssignableToTypeOf(ctxType), + gomock.AssignableToTypeOf(make(chan<- struct{})), + gomock.AssignableToTypeOf(make(chan<- error)), + ).Do(func(ctx context.Context, ready chan<- struct{}, done chan<- error) { + if testCase.startDone { + done <- testCase.startDoneErr + return // start failure + } + close(ready) + <-ctx.Done() + done <- testCase.startDoneErr + }) + + service := &Service{ + server: server, + done: make(chan error), + } + + err := service.Start() + + if testCase.startErr != nil { + require.EqualError(t, err, testCase.startErr.Error()) + } else { + assert.NoError(t, err) + } + + if testCase.startDone { + return // start failed, we won't stop + } + + err = service.Stop() + if testCase.startErr != nil { + require.EqualError(t, err, testCase.stopErr.Error()) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/internal/pprof/settings.go b/internal/pprof/settings.go new file mode 100644 index 0000000000..762b7f8153 --- /dev/null +++ b/internal/pprof/settings.go @@ -0,0 +1,31 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package pprof + +import "fmt" + +// Settings are the settings for the Pprof service. +type Settings struct { + // ListeningAddress is the HTTP pprof server + // listening address. + ListeningAddress string + // See runtime.SetBlockProfileRate + // Set to 0 to disable profiling. + BlockProfileRate int + // See runtime.SetMutexProfileFraction + // Set to 0 to disable profiling. + MutexProfileRate int +} + +func (s *Settings) setDefaults() { + if s.ListeningAddress == "" { + s.ListeningAddress = "localhost:6060" + } +} + +func (s Settings) String() string { + return fmt.Sprintf( + "listening on %s and setting block profile rate to %d, mutex profile rate to %d", + s.ListeningAddress, s.BlockProfileRate, s.MutexProfileRate) +}