Add http endpoints to use profiling tools

Add an optional flag "--pprof" for snap daemon. When this flag is set
the rest API add profiling tool routes. The flag will also be passed
when running a plugin. Now, response from the plugin have a field for
passin port used for profiling tools.

Made the change on snapctl to display ports on running plugins.

cmd/snapctl/plugin.go

@@ -173,9 +173,9 @@ func listPlugins(ctx *cli.Context) error {
 			fmt.Println("No running plugins found. Have you started a task?")
 			return nil
 		}
-		printFields(w, false, 0, "NAME", "HIT COUNT", "LAST HIT", "TYPE")
+		printFields(w, false, 0, "NAME", "HIT COUNT", "LAST HIT", "TYPE", "PPROF PORT")
 		for _, rp := range plugins.AvailablePlugins {
-			printFields(w, false, 0, rp.Name, rp.HitCount, time.Unix(rp.LastHitTimestamp, 0).Format(timeFormat), rp.Type)
+			printFields(w, false, 0, rp.Name, rp.HitCount, time.Unix(rp.LastHitTimestamp, 0).Format(timeFormat), rp.Type, rp.PprofPort)
 		}
 	} else {
 		if len(plugins.LoadedPlugins) == 0 {

control/available_plugin.go

@@ -75,6 +75,7 @@ type availablePlugin struct {
 	exec               string
 	execPath           string
 	fromPackage        bool
+	pprofPort          string
 }
 
 // newAvailablePlugin returns an availablePlugin with information from a
@@ -92,6 +93,7 @@ func newAvailablePlugin(resp plugin.Response, emitter gomit.Emitter, ep executab
 		healthChan:  make(chan error, 1),
 		lastHitTime: time.Now(),
 		ePlugin:     ep,
+		pprofPort:   resp.PprofAddress,
 	}
 	ap.key = fmt.Sprintf("%s"+core.Separator+"%s"+core.Separator+"%d", ap.pluginType.String(), ap.name, ap.version)
 
@@ -167,6 +169,10 @@ func newAvailablePlugin(resp plugin.Response, emitter gomit.Emitter, ep executab
 	return ap, nil
 }
 
+func (a *availablePlugin) Port() string {
+	return a.pprofPort
+}
+
 func (a *availablePlugin) ID() uint32 {
 	return a.id
 }

control/config.go

@@ -45,6 +45,7 @@ const (
 	defaultAutoDiscoverPath  string        = ""
 	defaultKeyringPaths      string        = ""
 	defaultCacheExpiration   time.Duration = 500 * time.Millisecond
+	defaultPprof             bool          = false
 )
 
 type pluginConfig struct {
@@ -79,6 +80,7 @@ type Config struct {
 	Plugins           *pluginConfig     `json:"plugins"yaml:"plugins"`
 	ListenAddr        string            `json:"listen_addr,omitempty"yaml:"listen_addr"`
 	ListenPort        int               `json:"listen_port,omitempty"yaml:"listen_port"`
+	Pprof             bool              `json:"pprof"yaml:"pprof"`
 }
 
 const (
@@ -119,6 +121,9 @@ const (
 					},
 					"listen_port": {
 						"type": "integer"
+					},
+					"pprof": {
+						"type": "boolean"
 					}
 				},
 				"additionalProperties": false
@@ -138,6 +143,7 @@ func GetDefaultConfig() *Config {
 		KeyringPaths:      defaultKeyringPaths,
 		CacheExpiration:   jsonutil.Duration{defaultCacheExpiration},
 		Plugins:           newPluginConfig(),
+		Pprof:             defaultPprof,
 	}
 }
 
@@ -191,6 +197,10 @@ func (c *Config) UnmarshalJSON(data []byte) error {
 			if err := json.Unmarshal(v, &(c.ListenPort)); err != nil {
 				return err
 			}
+		case "pprof":
+			if err := json.Unmarshal(v, &(c.Pprof)); err != nil {
+				return err
+			}
 		default:
 			return fmt.Errorf("Unrecognized key '%v' in global config file while parsing 'control'", k)
 		}

control/control.go

@@ -199,7 +199,7 @@ func New(cfg *Config) *pluginControl {
 	}).Debug("metric catalog created")
 
 	// Plugin Manager
-	c.pluginManager = newPluginManager()
+	c.pluginManager = newPluginManager(OptSetPprof(cfg.Pprof))
 	controlLogger.WithFields(log.Fields{
 		"_block": "new",
 	}).Debug("plugin manager created")

control/plugin/plugin.go

@@ -140,19 +140,24 @@ type Arg struct {
 	NoDaemon bool
 	// The listen port
 	listenPort string
+
+	// enable pprof
+	Pprof bool
 }
 
-func NewArg(logLevel int) Arg {
+func NewArg(logLevel int, pprof bool) Arg {
 	return Arg{
 		LogLevel:            log.Level(logLevel),
 		PingTimeoutDuration: PingTimeoutDurationDefault,
+		Pprof:               pprof,
 	}
 }
 
 // Response from started plugin
 type Response struct {
 	Meta          PluginMeta
 	ListenAddress string
+	PprofAddress  string
 	Token         string
 	Type          PluginType
 	// State is a signal from plugin to control that it passed

control/plugin/plugin_test.go

@@ -61,7 +61,7 @@ func TestMetricType(t *testing.T) {
 
 func TestArg(t *testing.T) {
 	Convey("NewArg", t, func() {
-		arg := NewArg(int(log.InfoLevel))
+		arg := NewArg(int(log.InfoLevel), false)
 		So(arg, ShouldNotBeNil)
 	})
 }

control/plugin_manager.go

@@ -236,6 +236,7 @@ type pluginManager struct {
 	loadedPlugins     *loadedPlugins
 	logPath           string
 	pluginConfig      *pluginConfig
+	pprof             bool
 }
 
 func newPluginManager(opts ...pluginManagerOpt) *pluginManager {
@@ -259,6 +260,13 @@ func newPluginManager(opts ...pluginManagerOpt) *pluginManager {
 
 type pluginManagerOpt func(*pluginManager)
 
+// OptSetPprof sets the pprof flag on the plugin manager
+func OptSetPprof(pprof bool) pluginManagerOpt {
+	return func(p *pluginManager) {
+		p.pprof = pprof
+	}
+}
+
 // OptSetPluginConfig sets the config on the plugin manager
 func OptSetPluginConfig(cf *pluginConfig) pluginManagerOpt {
 	return func(p *pluginManager) {
@@ -570,7 +578,7 @@ func (p *pluginManager) UnloadPlugin(pl core.Plugin) (*loadedPlugin, serror.Snap
 
 // GenerateArgs generates the cli args to send when stating a plugin
 func (p *pluginManager) GenerateArgs(logLevel int) plugin.Arg {
-	return plugin.NewArg(logLevel)
+	return plugin.NewArg(logLevel, p.pprof)
 }
 
 func (p *pluginManager) teardown() {

control/strategy/fixtures/fixtures.go

@@ -42,6 +42,7 @@ const (
 	processorType      = plugin.ProcessorPluginType
 	version            = 1
 	name               = "mock"
+	port               = ""
 )
 
 var lastHit = time.Unix(1460027570, 0)
@@ -57,6 +58,7 @@ type MockAvailablePlugin struct {
 	strategy   plugin.RoutingStrategyType
 	pluginType plugin.PluginType
 	version    int
+	port       string
 }
 
 func NewMockAvailablePlugin() *MockAvailablePlugin {
@@ -71,6 +73,7 @@ func NewMockAvailablePlugin() *MockAvailablePlugin {
 		strategy:   lruRouting,
 		pluginType: plugin.CollectorPluginType,
 		version:    version,
+		port:       port,
 	}
 	return mock
 }
@@ -186,3 +189,7 @@ func (m MockAvailablePlugin) Name() string {
 func (m MockAvailablePlugin) Version() int {
 	return m.version
 }
+
+func (m MockAvailablePlugin) Port() string {
+	return m.port
+}

core/plugin.go

@@ -70,6 +70,7 @@ type AvailablePlugin interface {
 	HitCount() int
 	LastHit() time.Time
 	ID() uint32
+	Port() string
 }
 
 // the public interface for a plugin

mgmt/rest/fixtures/mock_metric_manager.go

@@ -51,6 +51,7 @@ type MockLoadedPlugin struct {
 }
 
 func (m MockLoadedPlugin) Name() string       { return m.MyName }
+func (m MockLoadedPlugin) Port() string       { return "" }
 func (m MockLoadedPlugin) TypeName() string   { return m.MyType }
 func (m MockLoadedPlugin) Version() int       { return m.MyVersion }
 func (m MockLoadedPlugin) Plugin() string     { return "" }

mgmt/rest/flags.go

@@ -56,7 +56,11 @@ var (
 		Name:  "rest-auth",
 		Usage: "Enables snap's REST API authentication",
 	}
+	flPProf = cli.BoolFlag{
+		Name:  "pprof",
+		Usage: "Enables profiling tools",
+	}
 
 	// Flags consumed by snapd
-	Flags = []cli.Flag{flAPIDisabled, flAPIAddr, flAPIPort, flRestHTTPS, flRestCert, flRestKey, flRestAuth}
+	Flags = []cli.Flag{flAPIDisabled, flAPIAddr, flAPIPort, flRestHTTPS, flRestCert, flRestKey, flRestAuth, flPProf}
 )

mgmt/rest/plugin.go

@@ -295,6 +295,7 @@ func getPlugins(mm managesMetrics, detail bool, h string, plName string, plType
 				LastHitTimestamp: p.LastHit().Unix(),
 				ID:               p.ID(),
 				Href:             pluginURI(h, p),
+				PprofPort:        p.Port(),
 			}
 		}
 	}

mgmt/rest/rbody/plugin.go

@@ -107,4 +107,5 @@ type AvailablePlugin struct {
 	LastHitTimestamp int64  `json:"last_hit_timestamp"`
 	ID               uint32 `json:"id"`
 	Href             string `json:"href"`
+	PprofPort        string `json:"pprof_port"`
 }

mgmt/rest/server.go

@@ -27,6 +27,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/http/pprof"
 	"strings"
 	"sync"
 	"time"
@@ -60,6 +61,7 @@ const (
 	defaultAuth            bool   = false
 	defaultAuthPassword    string = ""
 	defaultPortSetByConfig bool   = false
+	defaultPprof           bool   = false
 )
 
 var (
@@ -83,6 +85,7 @@ type Config struct {
 	RestAuth         bool   `json:"rest_auth"yaml:"rest_auth"`
 	RestAuthPassword string `json:"rest_auth_password"yaml:"rest_auth_password"`
 	portSetByConfig  bool   ``
+	Pprof            bool   `json:"pprof"yaml:"pprof"`
 }
 
 const (
@@ -115,6 +118,9 @@ const (
 					},
 					"addr" : {
 						"type": "string"
+					},
+					"pprof": {
+						"type": "boolean"
 					}
 				},
 				"additionalProperties": false
@@ -174,6 +180,7 @@ type Server struct {
 	r          *httprouter.Router
 	snapTLS    *snapTLS
 	auth       bool
+	pprof      bool
 	authpwd    string
 	addrString string
 	addr       net.Addr
@@ -191,10 +198,12 @@ func New(cfg *Config) (*Server, error) {
 	https := cfg.HTTPS
 	cpath := cfg.RestCertificate
 	kpath := cfg.RestKey
+	pprof := cfg.Pprof
 	s := &Server{
 		err:        make(chan error),
 		killChan:   make(chan struct{}),
 		addrString: cfg.Address,
+		pprof:      pprof,
 	}
 	if https {
 		var err error
@@ -229,6 +238,7 @@ func GetDefaultConfig() *Config {
 		RestAuth:         defaultAuth,
 		RestAuthPassword: defaultAuthPassword,
 		portSetByConfig:  defaultPortSetByConfig,
+		Pprof:            defaultPprof,
 	}
 }
 
@@ -285,6 +295,10 @@ func (c *Config) UnmarshalJSON(data []byte) error {
 			if err := json.Unmarshal(v, &(c.RestAuthPassword)); err != nil {
 				return fmt.Errorf("%v (while parsing 'restapi::rest_auth_password')", err)
 			}
+		case "pprof":
+			if err := json.Unmarshal(v, &(c.Pprof)); err != nil {
+				return fmt.Errorf("%v (while parsing 'restapi::pprof')", err)
+			}
 		default:
 			return fmt.Errorf("Unrecognized key '%v' in global config file while parsing 'restapi'", k)
 		}
@@ -456,6 +470,7 @@ func (s *Server) BindConfigManager(c managesConfig) {
 }
 
 func (s *Server) addRoutes() {
+
 	// plugin routes
 	s.r.GET("/v1/plugins", s.getPlugins)
 	s.r.GET("/v1/plugins/:type", s.getPlugins)
@@ -492,6 +507,37 @@ func (s *Server) addRoutes() {
 		s.r.GET("/v1/tribe/members", s.getMembers)
 		s.r.GET("/v1/tribe/member/:name", s.getMember)
 	}
+
+	// profiling tools routes
+	if s.pprof {
+		s.r.GET("/v1/debug/pprof/", s.index)
+		s.r.GET("/v1/debug/pprof/cmdline", s.cmdline)
+		s.r.GET("/v1/debug/pprof/profile", s.profile)
+		s.r.GET("/v1/debug/pprof/symbol", s.symbol)
+		s.r.GET("/v1/debug/pprof/trace", s.trace)
+	}
+}
+
+// profiling tools handlers
+
+func (s *Server) index(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
+	pprof.Index(w, r)
+}
+
+func (s *Server) cmdline(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
+	pprof.Cmdline(w, r)
+}
+
+func (s *Server) profile(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
+	pprof.Profile(w, r)
+}
+
+func (s *Server) symbol(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
+	pprof.Symbol(w, r)
+}
+
+func (s *Server) trace(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
+	pprof.Trace(w, r)
 }
 
 func respond(code int, b rbody.Body, w http.ResponseWriter) {

snapd.go

@@ -795,6 +795,7 @@ func applyCmdLineFlags(cfg *Config, ctx *cli.Context) {
 	cfg.Control.CacheExpiration = jsonutil.Duration{setDurationVal(cfg.Control.CacheExpiration.Duration, ctx, "cache-expiration")}
 	cfg.Control.ListenAddr = setStringVal(cfg.Control.ListenAddr, ctx, "control-listen-addr")
 	cfg.Control.ListenPort = setIntVal(cfg.Control.ListenPort, ctx, "control-listen-port")
+	cfg.Control.Pprof = setBoolVal(cfg.Control.Pprof, ctx, "pprof")
 	// next for the RESTful server related flags
 	cfg.RestAPI.Enable = setBoolVal(cfg.RestAPI.Enable, ctx, "disable-api", invertBoolean)
 	cfg.RestAPI.Port = setIntVal(cfg.RestAPI.Port, ctx, "api-port")
@@ -804,6 +805,7 @@ func applyCmdLineFlags(cfg *Config, ctx *cli.Context) {
 	cfg.RestAPI.RestKey = setStringVal(cfg.RestAPI.RestKey, ctx, "rest-key")
 	cfg.RestAPI.RestAuth = setBoolVal(cfg.RestAPI.RestAuth, ctx, "rest-auth")
 	cfg.RestAPI.RestAuthPassword = setStringVal(cfg.RestAPI.RestAuthPassword, ctx, "rest-auth-pwd")
+	cfg.RestAPI.Pprof = setBoolVal(cfg.RestAPI.Pprof, ctx, "pprof")
 	// next for the scheduler related flags
 	cfg.Scheduler.WorkManagerQueueSize = setUIntVal(cfg.Scheduler.WorkManagerQueueSize, ctx, "work-manager-queue-size")
 	cfg.Scheduler.WorkManagerPoolSize = setUIntVal(cfg.Scheduler.WorkManagerPoolSize, ctx, "work-manager-pool-size")