feat(pprof): Pprof HTTP server service (ChainSafe#1991)

- Add a pprof HTTP server service to dot node - `internal/httpserver` package - `internal/pprof` package - Configurable server listening address from flags and toml config file - Configurable pprof server on/off from flags and toml config file - Configurable mutex profile fraction from flags and toml config file - Configurable block profile rate from flags and toml config file - Pprof documentation - Remove pprof to file operation and `--cpuprof` and `--memprof` flags
timwu20 · Nov 19, 2021 · ce24ea9 · ce24ea9
1 parent 8c2993d
commit ce24ea9
Show file tree

Hide file tree

Showing 43 changed files with 1,177 additions and 139 deletions.
diff --git a/chain/dev/config.toml b/chain/dev/config.toml
@@ -38,3 +38,9 @@ port = 8545
 host = "localhost"
 modules = ["system", "author", "chain", "state", "rpc", "grandpa", "offchain", "childstate", "syncstate", "payment"]
 ws-port = 8546
+
+[pprof]
+enabled = false
+listening-address = "localhost:6060"
+block-rate = 0
+mutex-rate = 0
diff --git a/chain/dev/defaults.go b/chain/dev/defaults.go
@@ -86,3 +86,21 @@ var (
 	// DefaultWSEnabled enables the WS server
 	DefaultWSEnabled = true
 )
+
+const (
+	// PprofConfig
+
+	// DefaultPprofEnabled to indicate the pprof http server should be enabled or not.
+	DefaultPprofEnabled = true
+
+	// DefaultPprofListeningAddress default pprof HTTP server listening address.
+	DefaultPprofListeningAddress = "localhost:6060"
+
+	// DefaultPprofBlockRate default block profile rate.
+	// Set to 0 to disable profiling.
+	DefaultPprofBlockRate = 0
+
+	// DefaultPprofMutexRate default mutex profile rate.
+	// Set to 0 to disable profiling.
+	DefaultPprofMutexRate = 0
+)
diff --git a/chain/gssmr/config.toml b/chain/gssmr/config.toml
@@ -39,3 +39,9 @@ port = 8545
 host = "localhost"
 modules = ["system", "author", "chain", "state", "rpc", "grandpa", "offchain", "childstate", "syncstate", "payment"]
 ws-port = 8546
+
+[pprof]
+enabled = false
+listening-address = "localhost:6060"
+block-rate = 0
+mutex-rate = 0
diff --git a/chain/gssmr/defaults.go b/chain/gssmr/defaults.go
@@ -92,3 +92,21 @@ var (
 	// DefaultRPCWSPort rpc websocket port
 	DefaultRPCWSPort = uint32(8546)
 )
+
+const (
+	// PprofConfig
+
+	// DefaultPprofEnabled to indicate the pprof http server should be enabled or not.
+	DefaultPprofEnabled = true
+
+	// DefaultPprofListeningAddress default pprof HTTP server listening address.
+	DefaultPprofListeningAddress = "localhost:6060"
+
+	// DefaultPprofBlockRate default block profile rate.
+	// Set to 0 to disable profiling.
+	DefaultPprofBlockRate = 0
+
+	// DefaultPprofMutexRate default mutex profile rate.
+	// Set to 0 to disable profiling.
+	DefaultPprofMutexRate = 0
+)
diff --git a/chain/kusama/config.toml b/chain/kusama/config.toml
@@ -39,3 +39,9 @@ modules = ["system", "author", "chain", "state", "rpc", "grandpa", "offchain", "
 ws-port = 8546
 ws = false
 ws-external = false
+
+[pprof]
+enabled = false
+listening-address = "localhost:6060"
+block-rate = 0
+mutex-rate = 0
diff --git a/chain/kusama/defaults.go b/chain/kusama/defaults.go
@@ -78,3 +78,21 @@ var (
 	// DefaultRPCWSPort rpc websocket port
 	DefaultRPCWSPort = uint32(8546)
 )
+
+const (
+	// PprofConfig
+
+	// DefaultPprofEnabled to indicate the pprof http server should be enabled or not.
+	DefaultPprofEnabled = false
+
+	// DefaultPprofListeningAddress default pprof HTTP server listening address.
+	DefaultPprofListeningAddress = "localhost:6060"
+
+	// DefaultPprofBlockRate default block profile rate.
+	// Set to 0 to disable profiling.
+	DefaultPprofBlockRate = 0
+
+	// DefaultPprofMutexRate default mutex profile rate.
+	// Set to 0 to disable profiling.
+	DefaultPprofMutexRate = 0
+)
diff --git a/chain/polkadot/config.toml b/chain/polkadot/config.toml
@@ -35,4 +35,10 @@ enabled = false
 port = 8545
 host = "localhost"
 modules = ["system", "author", "chain", "state", "rpc", "grandpa", "offchain", "childstate", "syncstate", "payment"]
-ws-port = 8546
+ws-port = 8546
+
+[pprof]
+enabled = false
+listening-address = "localhost:6060"
+block-rate = 0
+mutex-rate = 0
diff --git a/chain/polkadot/defaults.go b/chain/polkadot/defaults.go
@@ -79,3 +79,21 @@ var (
 	// DefaultRPCWSPort rpc websocket port
 	DefaultRPCWSPort = uint32(8546)
 )
+
+const (
+	// PprofConfig
+
+	// DefaultPprofEnabled to indicate the pprof http server should be enabled or not.
+	DefaultPprofEnabled = false
+
+	// DefaultPprofListeningAddress default pprof HTTP server listening address.
+	DefaultPprofListeningAddress = "localhost:6060"
+
+	// DefaultPprofBlockRate default block profile rate.
+	// Set to 0 to disable profiling.
+	DefaultPprofBlockRate = 0
+
+	// DefaultPprofMutexRate default mutex profile rate.
+	// Set to 0 to disable profiling.
+	DefaultPprofMutexRate = 0
+)
diff --git a/cmd/gossamer/config.go b/cmd/gossamer/config.go
@@ -135,6 +135,7 @@ func createDotConfig(ctx *cli.Context) (*dot.Config, error) {
 	setDotCoreConfig(ctx, tomlCfg.Core, &cfg.Core)
 	setDotNetworkConfig(ctx, tomlCfg.Network, &cfg.Network)
 	setDotRPCConfig(ctx, tomlCfg.RPC, &cfg.RPC)
+	setDotPprofConfig(ctx, tomlCfg.Pprof, &cfg.Pprof)
 
 	if rewind := ctx.GlobalInt(RewindFlag.Name); rewind != 0 {
 		cfg.State.Rewind = rewind
@@ -849,3 +850,49 @@ func updateDotConfigFromGenesisData(ctx *cli.Context, cfg *dot.Config) error {
 
 	return nil
 }
+
+func setDotPprofConfig(ctx *cli.Context, tomlCfg ctoml.PprofConfig, cfg *dot.PprofConfig) {
+	if !cfg.Enabled {
+		// only allow to enable pprof from the TOML configuration.
+		// If it is enabled by default, it cannot be disabled.
+		cfg.Enabled = tomlCfg.Enabled
+	}
+
+	if tomlCfg.ListeningAddress != "" {
+		cfg.Settings.ListeningAddress = tomlCfg.ListeningAddress
+	}
+
+	if tomlCfg.BlockRate > 0 {
+		// block rate must be 0 (disabled) by default, since we
+		// cannot disable it here.
+		cfg.Settings.BlockProfileRate = tomlCfg.BlockRate
+	}
+
+	if tomlCfg.MutexRate > 0 {
+		// mutex rate must be 0 (disabled) by default, since we
+		// cannot disable it here.
+		cfg.Settings.MutexProfileRate = tomlCfg.MutexRate
+	}
+
+	// check --pprofserver flag and update node configuration
+	if enabled := ctx.GlobalBool(PprofServerFlag.Name); enabled || cfg.Enabled {
+		cfg.Enabled = true
+	} else if ctx.IsSet(PprofServerFlag.Name) && !enabled {
+		cfg.Enabled = false
+	}
+
+	// check --pprofaddress flag and update node configuration
+	if address := ctx.GlobalString(PprofAddressFlag.Name); address != "" {
+		cfg.Settings.ListeningAddress = address
+	}
+
+	if rate := ctx.GlobalInt(PprofBlockRateFlag.Name); rate > 0 {
+		cfg.Settings.BlockProfileRate = rate
+	}
+
+	if rate := ctx.GlobalInt(PprofMutexRateFlag.Name); rate > 0 {
+		cfg.Settings.MutexProfileRate = rate
+	}
+
+	logger.Debug("pprof configuration: " + cfg.String())
+}
diff --git a/cmd/gossamer/config_test.go b/cmd/gossamer/config_test.go
@@ -782,6 +782,7 @@ func TestUpdateConfigFromGenesisJSON(t *testing.T) {
 		Network: testCfg.Network,
 		RPC:     testCfg.RPC,
 		System:  testCfg.System,
+		Pprof:   testCfg.Pprof,
 	}
 
 	cfg, err := createDotConfig(ctx)
@@ -836,6 +837,7 @@ func TestUpdateConfigFromGenesisJSON_Default(t *testing.T) {
 		Network: testCfg.Network,
 		RPC:     testCfg.RPC,
 		System:  testCfg.System,
+		Pprof:   testCfg.Pprof,
 	}
 
 	cfg, err := createDotConfig(ctx)
@@ -894,6 +896,7 @@ func TestUpdateConfigFromGenesisData(t *testing.T) {
 		},
 		RPC:    testCfg.RPC,
 		System: testCfg.System,
+		Pprof:  testCfg.Pprof,
 	}
 
 	cfg, err := createDotConfig(ctx)

diff --git a/cmd/gossamer/export.go b/cmd/gossamer/export.go
@@ -57,7 +57,14 @@ func exportAction(ctx *cli.Context) error {
 }
 
 func dotConfigToToml(dcfg *dot.Config) *ctoml.Config {
-	cfg := &ctoml.Config{}
+	cfg := &ctoml.Config{
+		Pprof: ctoml.PprofConfig{
+			Enabled:          dcfg.Pprof.Enabled,
+			ListeningAddress: dcfg.Pprof.Settings.ListeningAddress,
+			BlockRate:        dcfg.Pprof.Settings.BlockProfileRate,
+			MutexRate:        dcfg.Pprof.Settings.MutexProfileRate,
+		},
+	}
 
 	cfg.Global = ctoml.GlobalConfig{
 		Name:         dcfg.Global.Name,

diff --git a/cmd/gossamer/export_test.go b/cmd/gossamer/export_test.go
@@ -76,7 +76,8 @@ func TestExportCommand(t *testing.T) {
 					DiscoveryInterval: testCfg.Network.DiscoveryInterval,
 					MinPeers:          testCfg.Network.MinPeers,
 				},
-				RPC: testCfg.RPC,
+				RPC:   testCfg.RPC,
+				Pprof: testCfg.Pprof,
 			},
 		},
 		{
@@ -109,7 +110,8 @@ func TestExportCommand(t *testing.T) {
 					DiscoveryInterval: testCfg.Network.DiscoveryInterval,
 					MinPeers:          testCfg.Network.MinPeers,
 				},
-				RPC: testCfg.RPC,
+				RPC:   testCfg.RPC,
+				Pprof: testCfg.Pprof,
 			},
 		},
 		{
@@ -142,7 +144,8 @@ func TestExportCommand(t *testing.T) {
 					DiscoveryInterval: testCfg.Network.DiscoveryInterval,
 					MinPeers:          testCfg.Network.MinPeers,
 				},
-				RPC: testCfg.RPC,
+				RPC:   testCfg.RPC,
+				Pprof: testCfg.Pprof,
 			},
 		},
 	}

diff --git a/cmd/gossamer/flags.go b/cmd/gossamer/flags.go
@@ -97,13 +97,23 @@ var (
 		Name:  "basepath",
 		Usage: "Data directory for the node",
 	}
-	CPUProfFlag = cli.StringFlag{
-		Name:  "cpuprof",
-		Usage: "File to write CPU profile to",
+	PprofServerFlag = cli.StringFlag{
+		Name:  "pprofserver",
+		Usage: "enable or disable the pprof HTTP server",
 	}
-	MemProfFlag = cli.StringFlag{
-		Name:  "memprof",
-		Usage: "File to write memory profile to",
+	PprofAddressFlag = cli.StringFlag{
+		Name:  "pprofaddress",
+		Usage: "pprof HTTP server listening address, if it is enabled.",
+	}
+	PprofBlockRateFlag = cli.IntFlag{
+		Name:  "pprofblockrate",
+		Value: -1,
+		Usage: "pprof block rate. See https://pkg.go.dev/runtime#SetBlockProfileRate.",
+	}
+	PprofMutexRateFlag = cli.IntFlag{
+		Name:  "pprofmutexrate",
+		Value: -1,
+		Usage: "profiling mutex rate. See https://pkg.go.dev/runtime#SetMutexProfileFraction.",
 	}
 
 	// PublishMetricsFlag publishes node metrics to prometheus.
@@ -372,8 +382,10 @@ var (
 		ChainFlag,
 		ConfigFlag,
 		BasePathFlag,
-		CPUProfFlag,
-		MemProfFlag,
+		PprofServerFlag,
+		PprofAddressFlag,
+		PprofBlockRateFlag,
+		PprofMutexRateFlag,
 		RewindFlag,
 		DBPathFlag,
 		BloomFilterSizeFlag,

diff --git a/cmd/gossamer/main.go b/cmd/gossamer/main.go
@@ -200,12 +200,6 @@ func gossamerAction(ctx *cli.Context) error {
 		return fmt.Errorf("failed to read command argument: %q", arguments[0])
 	}
 
-	// begin profiling, if set
-	stopFunc, err := beginProfile(ctx)
-	if err != nil {
-		return err
-	}
-
 	// setup gossamer logger
 	lvl, err := setupLogger(ctx)
 	if err != nil {
@@ -283,7 +277,7 @@ func gossamerAction(ctx *cli.Context) error {
 		return err
 	}
 
-	node, err := dot.NewNode(cfg, ks, stopFunc)
+	node, err := dot.NewNode(cfg, ks)
 	if err != nil {
 		logger.Errorf("failed to create node services: %s", err)
 		return err