From 82b0993291756c59f96fe154b59aca6993f77e57 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 24 Mar 2022 14:28:13 +0000 Subject: [PATCH 01/23] Add stacktrace view to monitor page Using the pprof labels we can now add a stacktrace which can show these labels and name go-routines by the labels if there is a description set. Signed-off-by: Andrew Thornton --- go.mod | 1 + go.sum | 1 + options/locale/locale_en-US.ini | 1 + routers/web/admin/admin.go | 46 ++++++++++++++++++++++--- routers/web/web.go | 1 + templates/admin/process.tmpl | 3 ++ templates/admin/stacktrace.tmpl | 61 +++++++++++++++++++++++++++++++++ 7 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 templates/admin/stacktrace.tmpl diff --git a/go.mod b/go.mod index 0edd0dc2dc55b..67b7aecfb34c6 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 github.com/golang-jwt/jwt/v4 v4.3.0 github.com/google/go-github/v39 v39.2.0 + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 github.com/google/uuid v1.3.0 github.com/gorilla/feeds v1.1.1 github.com/gorilla/sessions v1.2.1 diff --git a/go.sum b/go.sum index 19ed2271c8848..c4b4739b985e3 100644 --- a/go.sum +++ b/go.sum @@ -760,6 +760,7 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg= diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index b371c047e3629..3d031c048df93 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2812,6 +2812,7 @@ monitor.next = Next Time monitor.previous = Previous Time monitor.execute_times = Executions monitor.process = Running Processes +monitor.stacktrace = Stacktraces monitor.desc = Description monitor.start = Start Time monitor.execute_time = Execution Time diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 4c700df354400..04cc9b56cbe78 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -7,14 +7,18 @@ package admin import ( "fmt" + "io" "net/http" "net/url" "os" "runtime" + "runtime/pprof" "strconv" "strings" "time" + "github.com/google/pprof/profile" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" @@ -35,10 +39,11 @@ import ( ) const ( - tplDashboard base.TplName = "admin/dashboard" - tplConfig base.TplName = "admin/config" - tplMonitor base.TplName = "admin/monitor" - tplQueue base.TplName = "admin/queue" + tplDashboard base.TplName = "admin/dashboard" + tplConfig base.TplName = "admin/config" + tplMonitor base.TplName = "admin/monitor" + tplStacktrace base.TplName = "admin/stacktrace" + tplQueue base.TplName = "admin/queue" ) var sysStatus struct { @@ -329,9 +334,42 @@ func Monitor(ctx *context.Context) { ctx.Data["Processes"] = process.GetManager().Processes(true) ctx.Data["Entries"] = cron.ListTasks() ctx.Data["Queues"] = queue.GetManager().ManagedQueues() + + reader, writer := io.Pipe() + defer reader.Close() + go func() { + err := pprof.Lookup("goroutine").WriteTo(writer, 0) + writer.CloseWithError(err) + }() + p, err := profile.Parse(reader) + jP, _ := json.MarshalIndent(p, "", " ") + log.Info("%v, %v", string(jP), err) + ctx.HTML(http.StatusOK, tplMonitor) } +// GoroutineStacktrace show admin monitor goroutines page +func GoroutineStacktrace(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("admin.monitor") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminMonitor"] = true + + reader, writer := io.Pipe() + defer reader.Close() + go func() { + err := pprof.Lookup("goroutine").WriteTo(writer, 0) + writer.CloseWithError(err) + }() + p, err := profile.Parse(reader) + if err != nil { + ctx.ServerError("GoroutineStacktrace", err) + return + } + ctx.Data["Profile"] = p + + ctx.HTML(http.StatusOK, tplStacktrace) +} + // MonitorCancel cancels a process func MonitorCancel(ctx *context.Context) { pid := ctx.Params("pid") diff --git a/routers/web/web.go b/routers/web/web.go index b40a43058d423..87ff00e4182f7 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -408,6 +408,7 @@ func RegisterRoutes(m *web.Route) { m.Post("/config/test_mail", admin.SendTestMail) m.Group("/monitor", func() { m.Get("", admin.Monitor) + m.Get("/stacktrace", admin.GoroutineStacktrace) m.Post("/cancel/{pid}", admin.MonitorCancel) m.Group("/queue/{qid}", func() { m.Get("", admin.Queue) diff --git a/templates/admin/process.tmpl b/templates/admin/process.tmpl index 719c10cead3db..c44300dbb7593 100644 --- a/templates/admin/process.tmpl +++ b/templates/admin/process.tmpl @@ -1,5 +1,8 @@

{{.i18n.Tr "admin.monitor.process"}} +
+ {{.i18n.Tr "admin.monitor.stacktrace"}} +

diff --git a/templates/admin/stacktrace.tmpl b/templates/admin/stacktrace.tmpl new file mode 100644 index 0000000000000..4bae99e5bd8a8 --- /dev/null +++ b/templates/admin/stacktrace.tmpl @@ -0,0 +1,61 @@ +{{template "base/head" .}} +
+ {{template "admin/navbar" .}} +
+ {{template "base/alert" .}} +

+ {{.i18n.Tr "admin.monitor.stacktrace"}} +
+ {{.i18n.Tr "admin.monitor"}} +
+

+
+
+ {{range .Profile.Sample}} +
+
+
+
+
{{if index .Label "process-description"}}{{svg "octicon-terminal" 16 }}
{{index (index .Label "process-description") 0}} ({{index (index .Label "pid") 0 }})
{{else}}{{svg "octicon-cpu" 16 }}
{{(index (index .Location (Subtract (len .Location) 1)).Line 0).Function.Name}}
{{end}}{{if gt (index .Value 0) 1}}
* {{index .Value 0}}
{{end}}
+ {{range $key, $value := .Label}} + {{if ne $key "process-description" }} +
{{$key}}
{{index $value 0}}
+ {{end}} + {{end}} + + +
+ {{range .Location}} +
+ {{svg "octicon-dot-fill" 16 }} +
+
{{(index .Line 0).Function.Name}}
+
{{(index .Line 0).Function.Filename}}:{{(index .Line 0).Line}}
+
+
+ {{end}} +
+
+
+
+
+
+
+ {{end}} +
+
+
+
+
+
+ {{svg "octicon-x" 16 "close inside"}} + {{.i18n.Tr "admin.monitor.process.cancel"}} +
+
+

{{$.i18n.Tr "admin.monitor.process.cancel_notices" `` | Safe}}

+

{{$.i18n.Tr "admin.monitor.process.cancel_desc"}}

+
+ {{template "base/delete_modal_actions" .}} +
+ +{{template "base/footer" .}} From a08537ac172f6b9ad8f8a062474c659e37985bb4 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 24 Mar 2022 21:06:11 +0000 Subject: [PATCH 02/23] Add more process descriptors to goroutines Signed-off-by: Andrew Thornton --- modules/context/private.go | 2 +- modules/eventsource/manager_run.go | 4 ++++ modules/graceful/manager_unix.go | 6 +++++- modules/log/event.go | 12 ++++++++++- modules/log/multichannel.go | 2 +- modules/process/manager.go | 33 +++++++++++++++++++++++++----- modules/process/process.go | 7 +++++++ routers/common/middleware.go | 2 +- routers/web/admin/admin.go | 12 +---------- services/webhook/deliver.go | 11 ++++++---- templates/admin/process-row.tmpl | 5 ++++- templates/admin/stacktrace.tmpl | 2 +- 12 files changed, 71 insertions(+), 27 deletions(-) diff --git a/modules/context/private.go b/modules/context/private.go index 6e5ef1bd121c3..9b31234e54185 100644 --- a/modules/context/private.go +++ b/modules/context/private.go @@ -79,6 +79,6 @@ func PrivateContexter() func(http.Handler) http.Handler { // the underlying request has timed out from the ssh/http push func OverrideContext(ctx *PrivateContext) (cancel context.CancelFunc) { // We now need to override the request context as the base for our work because even if the request is cancelled we have to continue this work - ctx.Override, _, cancel = process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("PrivateContext: %s", ctx.Req.RequestURI)) + ctx.Override, _, cancel = process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), fmt.Sprintf("PrivateContext: %s", ctx.Req.RequestURI), process.RequestProcessType) return } diff --git a/modules/eventsource/manager_run.go b/modules/eventsource/manager_run.go index 60598ecb495f5..0e8dbbfc45c29 100644 --- a/modules/eventsource/manager_run.go +++ b/modules/eventsource/manager_run.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" ) @@ -25,6 +26,9 @@ func (m *Manager) Init() { // Run runs the manager within a provided context func (m *Manager) Run(ctx context.Context) { + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "EventSource", process.SystemProcessType) + defer finished() + then := timeutil.TimeStampNow().Add(-2) timer := time.NewTicker(setting.UI.Notification.EventSourceUpdateTime) loop: diff --git a/modules/graceful/manager_unix.go b/modules/graceful/manager_unix.go index 6fbb2bda29ab0..5611006e0bb22 100644 --- a/modules/graceful/manager_unix.go +++ b/modules/graceful/manager_unix.go @@ -18,6 +18,7 @@ import ( "time" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" ) @@ -73,7 +74,7 @@ func (g *Manager) start(ctx context.Context) { // Set the running state & handle signals g.setState(stateRunning) - go g.handleSignals(ctx) + go g.handleSignals(g.managerCtx) // Handle clean up of unused provided listeners and delayed start-up startupDone := make(chan struct{}) @@ -112,6 +113,9 @@ func (g *Manager) start(ctx context.Context) { } func (g *Manager) handleSignals(ctx context.Context) { + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Graceful: HandleSignals", process.SystemProcessType) + defer finished() + signalChannel := make(chan os.Signal, 1) signal.Notify( diff --git a/modules/log/event.go b/modules/log/event.go index b20dac17c73e4..cae8f0936fd66 100644 --- a/modules/log/event.go +++ b/modules/log/event.go @@ -5,9 +5,12 @@ package log import ( + "context" "fmt" "sync" "time" + + "code.gitea.io/gitea/modules/process" ) // Event represents a logging event @@ -34,6 +37,7 @@ type EventLogger interface { // ChannelledLog represents a cached channel to a LoggerProvider type ChannelledLog struct { + parent string name string provider string queue chan *Event @@ -44,9 +48,10 @@ type ChannelledLog struct { } // NewChannelledLog a new logger instance with given logger provider and config. -func NewChannelledLog(name, provider, config string, bufferLength int64) (*ChannelledLog, error) { +func NewChannelledLog(parent, name, provider, config string, bufferLength int64) (*ChannelledLog, error) { if log, ok := providers[provider]; ok { l := &ChannelledLog{ + parent: parent, queue: make(chan *Event, bufferLength), flush: make(chan bool), close: make(chan bool), @@ -66,6 +71,8 @@ func NewChannelledLog(name, provider, config string, bufferLength int64) (*Chann // Start processing the ChannelledLog func (l *ChannelledLog) Start() { + _, _, cancel := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Logger: %s.%s(%s)", l.parent, l.name, l.provider), process.SystemProcessType) + defer cancel() for { select { case event, ok := <-l.queue: @@ -277,6 +284,9 @@ func (m *MultiChannelledLog) Start() { m.rwmutex.Unlock() return } + _, _, cancel := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Logger: %s", m.name), process.SystemProcessType) + defer cancel() + m.started = true m.rwmutex.Unlock() paused := false diff --git a/modules/log/multichannel.go b/modules/log/multichannel.go index c725df4f3ed57..e5c13f1f011c5 100644 --- a/modules/log/multichannel.go +++ b/modules/log/multichannel.go @@ -31,7 +31,7 @@ func newLogger(name string, buffer int64) *MultiChannelledLogger { // SetLogger sets new logger instance with given logger provider and config. func (l *MultiChannelledLogger) SetLogger(name, provider, config string) error { - eventLogger, err := NewChannelledLog(name, provider, config, l.bufferLength) + eventLogger, err := NewChannelledLog(l.name, name, provider, config, l.bufferLength) if err != nil { return fmt.Errorf("Failed to create sublogger (%s): %v", name, err) } diff --git a/modules/process/manager.go b/modules/process/manager.go index 50dbbbe6c807c..4220906b05d62 100644 --- a/modules/process/manager.go +++ b/modules/process/manager.go @@ -69,7 +69,26 @@ func GetManager() *Manager { func (pm *Manager) AddContext(parent context.Context, description string) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) { ctx, cancel = context.WithCancel(parent) - ctx, pid, finished := pm.Add(ctx, description, cancel) + ctx, pid, finished := pm.Add(ctx, description, cancel, NormalProcessType) + + return &Context{ + Context: ctx, + pid: pid, + }, cancel, finished +} + +// AddTypedContext creates a new context and adds it as a process. Once the process is finished, finished must be called +// to remove the process from the process table. It should not be called until the process is finished but must always be called. +// +// cancel should be used to cancel the returned context, however it will not remove the process from the process table. +// finished will cancel the returned context and remove it from the process table. +// +// Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the +// process table. +func (pm *Manager) AddTypedContext(parent context.Context, description, processType string) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) { + ctx, cancel = context.WithCancel(parent) + + ctx, pid, finished := pm.Add(ctx, description, cancel, processType) return &Context{ Context: ctx, @@ -88,7 +107,7 @@ func (pm *Manager) AddContext(parent context.Context, description string) (ctx c func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Duration, description string) (ctx context.Context, cancel context.CancelFunc, finshed FinishedFunc) { ctx, cancel = context.WithTimeout(parent, timeout) - ctx, pid, finshed := pm.Add(ctx, description, cancel) + ctx, pid, finshed := pm.Add(ctx, description, cancel, NormalProcessType) return &Context{ Context: ctx, @@ -97,7 +116,7 @@ func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Durati } // Add create a new process -func (pm *Manager) Add(ctx context.Context, description string, cancel context.CancelFunc) (context.Context, IDType, FinishedFunc) { +func (pm *Manager) Add(ctx context.Context, description string, cancel context.CancelFunc, processType string) (context.Context, IDType, FinishedFunc) { parentPID := GetParentPID(ctx) pm.mutex.Lock() @@ -114,6 +133,7 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C Description: description, Start: start, Cancel: cancel, + Type: processType, } finished := func() { @@ -128,7 +148,7 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C pm.processes[pid] = process pm.mutex.Unlock() - pprofCtx := pprof.WithLabels(ctx, pprof.Labels("process-description", description, "ppid", string(parentPID), "pid", string(pid))) + pprofCtx := pprof.WithLabels(ctx, pprof.Labels("process-description", description, "ppid", string(parentPID), "pid", string(pid), "process-type", processType)) pprof.SetGoroutineLabels(pprofCtx) return pprofCtx, pid, finished @@ -191,7 +211,10 @@ func (pm *Manager) Processes(onlyRoots bool) []*Process { processes := make([]*Process, 0, len(pm.processes)) if onlyRoots { for _, process := range pm.processes { - if _, has := pm.processes[process.ParentPID]; !has { + if process.Type == SystemProcessType { + continue + } + if parent, has := pm.processes[process.ParentPID]; !has || parent.Type == SystemProcessType { processes = append(processes, process) } } diff --git a/modules/process/process.go b/modules/process/process.go index 662f878d7f3db..e104ace970b02 100644 --- a/modules/process/process.go +++ b/modules/process/process.go @@ -10,6 +10,12 @@ import ( "time" ) +var ( + SystemProcessType = "system" + RequestProcessType = "request" + NormalProcessType = "normal" +) + // Process represents a working process inheriting from Gitea. type Process struct { PID IDType // Process ID, not system one. @@ -17,6 +23,7 @@ type Process struct { Description string Start time.Time Cancel context.CancelFunc + Type string lock sync.Mutex children []*Process diff --git a/routers/common/middleware.go b/routers/common/middleware.go index 591c4cf30e8e9..db70dc46b1a89 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -27,7 +27,7 @@ func Middlewares() []func(http.Handler) http.Handler { // First of all escape the URL RawPath to ensure that all routing is done using a correctly escaped URL req.URL.RawPath = req.URL.EscapedPath() - ctx, _, finished := process.GetManager().AddContext(req.Context(), fmt.Sprintf("%s: %s", req.Method, req.RequestURI)) + ctx, _, finished := process.GetManager().AddTypedContext(req.Context(), fmt.Sprintf("%s: %s", req.Method, req.RequestURI), process.RequestProcessType) defer finished() next.ServeHTTP(context.NewResponse(resp), req.WithContext(ctx)) }) diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 04cc9b56cbe78..f490bf327e0ca 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -335,16 +335,6 @@ func Monitor(ctx *context.Context) { ctx.Data["Entries"] = cron.ListTasks() ctx.Data["Queues"] = queue.GetManager().ManagedQueues() - reader, writer := io.Pipe() - defer reader.Close() - go func() { - err := pprof.Lookup("goroutine").WriteTo(writer, 0) - writer.CloseWithError(err) - }() - p, err := profile.Parse(reader) - jP, _ := json.MarshalIndent(p, "", " ") - log.Info("%v, %v", string(jP), err) - ctx.HTML(http.StatusOK, tplMonitor) } @@ -358,7 +348,7 @@ func GoroutineStacktrace(ctx *context.Context) { defer reader.Close() go func() { err := pprof.Lookup("goroutine").WriteTo(writer, 0) - writer.CloseWithError(err) + _ = writer.CloseWithError(err) }() p, err := profile.Parse(reader) if err != nil { diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index 88b709cb41e74..45744792b6f88 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/hostmatcher" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/proxy" "code.gitea.io/gitea/modules/setting" @@ -31,7 +32,7 @@ import ( ) // Deliver deliver hook task -func Deliver(t *webhook_model.HookTask) error { +func Deliver(ctx context.Context, t *webhook_model.HookTask) error { w, err := webhook_model.GetWebhookByID(t.HookID) if err != nil { return err @@ -172,7 +173,7 @@ func Deliver(t *webhook_model.HookTask) error { return fmt.Errorf("webhook task skipped (webhooks disabled): [%d]", t.ID) } - resp, err := webhookHTTPClient.Do(req.WithContext(graceful.GetManager().ShutdownContext())) + resp, err := webhookHTTPClient.Do(req.WithContext(ctx)) if err != nil { t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err) return err @@ -204,6 +205,8 @@ func DeliverHooks(ctx context.Context) { return default: } + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "DeliverHooks", process.SystemProcessType) + defer finished() tasks, err := webhook_model.FindUndeliveredHookTasks() if err != nil { log.Error("DeliverHooks: %v", err) @@ -217,7 +220,7 @@ func DeliverHooks(ctx context.Context) { return default: } - if err = Deliver(t); err != nil { + if err = Deliver(ctx, t); err != nil { log.Error("deliver: %v", err) } } @@ -249,7 +252,7 @@ func DeliverHooks(ctx context.Context) { return default: } - if err = Deliver(t); err != nil { + if err = Deliver(ctx, t); err != nil { log.Error("deliver: %v", err) } } diff --git a/templates/admin/process-row.tmpl b/templates/admin/process-row.tmpl index 146ecc7b29b0f..2e20c5c1f7d8b 100644 --- a/templates/admin/process-row.tmpl +++ b/templates/admin/process-row.tmpl @@ -1,11 +1,14 @@
+
{{if eq .Process.Type "request"}}{{svg "octicon-globe" 16 }}{{else if eq .Process.Type "system"}}{{svg "octicon-cpu" 16 }}{{else}}{{svg "octicon-terminal" 16 }}{{end}}
{{.Process.Description}}
{{TimeSince .Process.Start .root.i18n.Lang}}
- {{svg "octicon-trash" 16 "text-red"}} + {{if ne .Process.Type "system"}} + {{svg "octicon-trash" 16 "text-red"}} + {{end}}
diff --git a/templates/admin/stacktrace.tmpl b/templates/admin/stacktrace.tmpl index 4bae99e5bd8a8..c7ebf78c4427e 100644 --- a/templates/admin/stacktrace.tmpl +++ b/templates/admin/stacktrace.tmpl @@ -16,7 +16,7 @@
-
{{if index .Label "process-description"}}{{svg "octicon-terminal" 16 }}
{{index (index .Label "process-description") 0}} ({{index (index .Label "pid") 0 }})
{{else}}{{svg "octicon-cpu" 16 }}
{{(index (index .Location (Subtract (len .Location) 1)).Line 0).Function.Name}}
{{end}}{{if gt (index .Value 0) 1}}
* {{index .Value 0}}
{{end}}
+
{{if index .Label "process-description"}}{{if eq (index (index .Label "process-type") 0) "request"}}{{svg "octicon-globe" 16 }}{{else if eq (index (index .Label "process-type") 0) "system"}}{{svg "octicon-cpu" 16 }}{{else}}{{svg "octicon-terminal" 16 }}{{end}}
{{index (index .Label "process-description") 0}}
{{else}}{{svg "octicon-code" 16 }}
{{(index (index .Location (Subtract (len .Location) 1)).Line 0).Function.Name}}
{{end}}{{if gt (index .Value 0) 1}}
* {{index .Value 0}}
{{end}}
{{range $key, $value := .Label}} {{if ne $key "process-description" }}
{{$key}}
{{index $value 0}}
From 1804e520167a95ae4ec56e13626636c494ee603a Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 12:46:14 +0000 Subject: [PATCH 03/23] Reformat stacktraces to keep with processes Signed-off-by: Andrew Thornton --- modules/process/manager.go | 24 +++- options/locale/locale_en-US.ini | 1 + routers/web/admin/admin.go | 184 ++++++++++++++++++++++++++-- templates/admin/stacktrace-row.tmpl | 66 ++++++++++ templates/admin/stacktrace.tmpl | 34 +---- 5 files changed, 262 insertions(+), 47 deletions(-) create mode 100644 templates/admin/stacktrace-row.tmpl diff --git a/modules/process/manager.go b/modules/process/manager.go index 4220906b05d62..0e6e1095f5100 100644 --- a/modules/process/manager.go +++ b/modules/process/manager.go @@ -30,6 +30,18 @@ var ( DefaultContext = context.Background() ) +// DescriptionPProfLabel is a label set on goroutines that have a process attached +const DescriptionPProfLabel = "process-description" + +// PIDPProfLabel is a label set on goroutines that have a process attached +const PIDPProfLabel = "pid" + +// PPIDPProfLabel is a label set on goroutines that have a process attached +const PPIDPProfLabel = "ppid" + +// ProcessTypePProfLabel is a label set on goroutines that have a process attached +const ProcessTypePProfLabel = "process-type" + // IDType is a pid type type IDType string @@ -148,7 +160,7 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C pm.processes[pid] = process pm.mutex.Unlock() - pprofCtx := pprof.WithLabels(ctx, pprof.Labels("process-description", description, "ppid", string(parentPID), "pid", string(pid), "process-type", processType)) + pprofCtx := pprof.WithLabels(ctx, pprof.Labels(DescriptionPProfLabel, description, PPIDPProfLabel, string(parentPID), PIDPProfLabel, string(pid), ProcessTypePProfLabel, processType)) pprof.SetGoroutineLabels(pprofCtx) return pprofCtx, pid, finished @@ -206,12 +218,12 @@ func (pm *Manager) Cancel(pid IDType) { } // Processes gets the processes in a thread safe manner -func (pm *Manager) Processes(onlyRoots bool) []*Process { +func (pm *Manager) Processes(onlyRoots, noSystem bool, runInLock func()) []*Process { pm.mutex.Lock() processes := make([]*Process, 0, len(pm.processes)) if onlyRoots { for _, process := range pm.processes { - if process.Type == SystemProcessType { + if noSystem && process.Type == SystemProcessType { continue } if parent, has := pm.processes[process.ParentPID]; !has || parent.Type == SystemProcessType { @@ -220,9 +232,15 @@ func (pm *Manager) Processes(onlyRoots bool) []*Process { } } else { for _, process := range pm.processes { + if noSystem && process.Type == SystemProcessType { + continue + } processes = append(processes, process) } } + if runInLock != nil { + runInLock() + } pm.mutex.Unlock() sort.Slice(processes, func(i, j int) bool { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3d031c048df93..6f412c9773967 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2813,6 +2813,7 @@ monitor.previous = Previous Time monitor.execute_times = Executions monitor.process = Running Processes monitor.stacktrace = Stacktraces +monitor.goroutines=%d Goroutines monitor.desc = Description monitor.start = Start Time monitor.execute_time = Execution Time diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index f490bf327e0ca..61fd785aaf2ef 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -13,12 +13,11 @@ import ( "os" "runtime" "runtime/pprof" + "sort" "strconv" "strings" "time" - "github.com/google/pprof/profile" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" @@ -36,6 +35,7 @@ import ( "code.gitea.io/gitea/services/mailer" "gitea.com/go-chi/session" + "github.com/google/pprof/profile" ) const ( @@ -331,7 +331,7 @@ func Monitor(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.monitor") ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminMonitor"] = true - ctx.Data["Processes"] = process.GetManager().Processes(true) + ctx.Data["Processes"] = process.GetManager().Processes(true, true, nil) ctx.Data["Entries"] = cron.ListTasks() ctx.Data["Queues"] = queue.GetManager().ManagedQueues() @@ -344,18 +344,176 @@ func GoroutineStacktrace(ctx *context.Context) { ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminMonitor"] = true - reader, writer := io.Pipe() - defer reader.Close() - go func() { - err := pprof.Lookup("goroutine").WriteTo(writer, 0) - _ = writer.CloseWithError(err) - }() - p, err := profile.Parse(reader) - if err != nil { - ctx.ServerError("GoroutineStacktrace", err) + var stacks *profile.Profile + processes := process.GetManager().Processes(false, false, func() { + reader, writer := io.Pipe() + defer reader.Close() + go func() { + err := pprof.Lookup("goroutine").WriteTo(writer, 0) + _ = writer.CloseWithError(err) + }() + var err error + stacks, err = profile.Parse(reader) + if err != nil { + ctx.ServerError("GoroutineStacktrace", err) + return + } + }) + if ctx.Written() { return } - ctx.Data["Profile"] = p + + type StackEntry struct { + Function string + File string + Line int + } + + type Label struct { + Name string + Value string + } + + type Stack struct { + Count int64 + Description string + Labels []*Label + Entry []*StackEntry + } + + type ProcessStack struct { + PID process.IDType + ParentPID process.IDType + Description string + Start time.Time + Type string + + Children []*ProcessStack + Stacks []*Stack + } + + // Now earlier we sorted by process time so we know that we should not have children before parents + pidMap := map[process.IDType]*ProcessStack{} + processStacks := make([]*ProcessStack, 0, len(processes)) + for _, process := range processes { + pStack := &ProcessStack{ + PID: process.PID, + ParentPID: process.ParentPID, + Description: process.Description, + Start: process.Start, + Type: process.Type, + } + + pidMap[process.PID] = pStack + if parent, ok := pidMap[process.ParentPID]; ok { + parent.Children = append(parent.Children, pStack) + } else { + processStacks = append(processStacks, pStack) + } + } + + goroutineCount := int64(0) + + // Now walk through the "Sample" slice in the goroutines stack + for _, sample := range stacks.Sample { + stack := &Stack{} + + // Add the labels + for name, value := range sample.Label { + if name == process.DescriptionPProfLabel || name == process.PIDPProfLabel || name == process.PPIDPProfLabel || name == process.ProcessTypePProfLabel { + continue + } + if len(value) != 1 { + // Unexpected... + log.Error("Label: %s in goroutine stack with unexpected number of values: %v", name, value) + continue + } + + stack.Labels = append(stack.Labels, &Label{Name: name, Value: value[0]}) + } + + stack.Count = sample.Value[0] + goroutineCount += stack.Count + + // Now get the processStack for this goroutine sample + var processStack *ProcessStack + if pidvalue, ok := sample.Label[process.PIDPProfLabel]; ok && len(pidvalue) == 1 { + pid := process.IDType(pidvalue[0]) + processStack, ok = pidMap[pid] + if !ok && pid != "" { + ppid := process.IDType("") + if value, ok := sample.Label[process.PPIDPProfLabel]; ok && len(value) == 1 { + ppid = process.IDType(value[0]) + } + description := "(missing process)" + if value, ok := sample.Label[process.DescriptionPProfLabel]; ok && len(value) == 1 { + description = value[0] + " " + description + } + ptype := process.SystemProcessType + if value, ok := sample.Label[process.ProcessTypePProfLabel]; ok && len(value) == 1 { + ptype = value[0] + } + processStack = &ProcessStack{ + PID: pid, + ParentPID: ppid, + Description: description, + Type: ptype, + } + + pidMap[processStack.PID] = processStack + if parent, ok := pidMap[processStack.ParentPID]; ok { + parent.Children = append(parent.Children, processStack) + } + } + } + if processStack == nil { + var ok bool + processStack, ok = pidMap[""] + if !ok { + processStack = &ProcessStack{ + Description: "(unknown)", + Type: "code", + } + pidMap[processStack.PID] = processStack + processStacks = append(processStacks, processStack) + } + } + + // Now walk through the locations... + for _, location := range sample.Location { + for _, line := range location.Line { + entry := &StackEntry{ + Function: line.Function.Name, + File: line.Function.Filename, + Line: int(line.Line), + } + stack.Entry = append(stack.Entry, entry) + } + } + stack.Description = "(others)" + if len(stack.Entry) > 0 { + stack.Description = stack.Entry[len(stack.Entry)-1].Function + } + + processStack.Stacks = append(processStack.Stacks, stack) + } + + // Now finally re-sort the processstacks so the newest processes are at the top + after := func(processStacks []*ProcessStack) func(i, j int) bool { + return func(i, j int) bool { + left, right := processStacks[i], processStacks[j] + return left.Start.After(right.Start) + } + } + sort.Slice(processStacks, after(processStacks)) + for _, processStack := range processStacks { + sort.Slice(processStack.Children, after(processStack.Children)) + } + + ctx.Data["ProcessStacks"] = processStacks + ctx.Data["Profile"] = stacks + + ctx.Data["GoroutineCount"] = goroutineCount ctx.HTML(http.StatusOK, tplStacktrace) } diff --git a/templates/admin/stacktrace-row.tmpl b/templates/admin/stacktrace-row.tmpl new file mode 100644 index 0000000000000..88cda1c8d6f43 --- /dev/null +++ b/templates/admin/stacktrace-row.tmpl @@ -0,0 +1,66 @@ +
+
+
+ {{if eq .Process.Type "request"}} + {{svg "octicon-globe" 16 }} + {{else if eq .Process.Type "system"}} + {{svg "octicon-cpu" 16 }} + {{else if eq .Process.Type "normal"}} + {{svg "octicon-terminal" 16 }} + {{else}} + {{svg "octicon-code" 16 }} + {{end}} +
+
+
{{.Process.Description}}
+
{{if ne .Process.Type "code"}}{{TimeSince .Process.Start .root.i18n.Lang}}{{end}}
+
+
+ {{if or (eq .Process.Type "request") (eq .Process.Type "normal") }} + {{svg "octicon-trash" 16 "text-red"}} + {{end}} +
+
+ {{if .Process.Stacks}} +
+ {{range .Process.Stacks}} +
+
+ +
+
+ {{svg "octicon-code" 16 }}{{.Description}}{{if gt .Count 1}} * {{.Count}}{{end}} +
+
+ {{range .Labels}} +
{{.Name}}
{{.Value}}
+ {{end}} +
+
+ +
+ {{range .Entry}} +
+ {{svg "octicon-dot-fill" 16 }} +
+
{{.Function}}
+
{{.File}}:{{.Line}}
+
+
+ {{end}} +
+
+
+ {{end}} +
+ {{end}} + + {{if .Process.Children}} +
+ {{range .Process.Children}} + {{template "admin/process-row" dict "Process" . "root" $.root}} + {{end}} +
+ {{end}} + +
diff --git a/templates/admin/stacktrace.tmpl b/templates/admin/stacktrace.tmpl index c7ebf78c4427e..68dfbe066d1f9 100644 --- a/templates/admin/stacktrace.tmpl +++ b/templates/admin/stacktrace.tmpl @@ -4,43 +4,15 @@
{{template "base/alert" .}}

- {{.i18n.Tr "admin.monitor.stacktrace"}} + {{.i18n.Tr "admin.monitor.stacktrace"}}: {{.i18n.Tr "admin.monitor.goroutines" .GoroutineCount}}
{{.i18n.Tr "admin.monitor"}}

- {{range .Profile.Sample}} -
-
-
-
-
{{if index .Label "process-description"}}{{if eq (index (index .Label "process-type") 0) "request"}}{{svg "octicon-globe" 16 }}{{else if eq (index (index .Label "process-type") 0) "system"}}{{svg "octicon-cpu" 16 }}{{else}}{{svg "octicon-terminal" 16 }}{{end}}
{{index (index .Label "process-description") 0}}
{{else}}{{svg "octicon-code" 16 }}
{{(index (index .Location (Subtract (len .Location) 1)).Line 0).Function.Name}}
{{end}}{{if gt (index .Value 0) 1}}
* {{index .Value 0}}
{{end}}
- {{range $key, $value := .Label}} - {{if ne $key "process-description" }} -
{{$key}}
{{index $value 0}}
- {{end}} - {{end}} - - -
- {{range .Location}} -
- {{svg "octicon-dot-fill" 16 }} -
-
{{(index .Line 0).Function.Name}}
-
{{(index .Line 0).Function.Filename}}:{{(index .Line 0).Line}}
-
-
- {{end}} -
-
-
-
-
-
-
+ {{range .ProcessStacks}} + {{template "admin/stacktrace-row" dict "Process" . "root" $}} {{end}}
From 0807c24b30b73208ab2492c389d5ea82be52d4a8 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 12:56:55 +0000 Subject: [PATCH 04/23] adjust label for the unbound goroutines Signed-off-by: Andrew Thornton --- routers/web/admin/admin.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 61fd785aaf2ef..648590e058526 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -471,7 +471,7 @@ func GoroutineStacktrace(ctx *context.Context) { processStack, ok = pidMap[""] if !ok { processStack = &ProcessStack{ - Description: "(unknown)", + Description: "(unbound)", Type: "code", } pidMap[processStack.PID] = processStack @@ -490,7 +490,7 @@ func GoroutineStacktrace(ctx *context.Context) { stack.Entry = append(stack.Entry, entry) } } - stack.Description = "(others)" + stack.Description = "(unknown)" if len(stack.Entry) > 0 { stack.Description = stack.Entry[len(stack.Entry)-1].Function } From db2a09d1208bdcced6ab7212c9586238e7c57adf Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 13:40:37 +0000 Subject: [PATCH 05/23] small fixes Signed-off-by: Andrew Thornton --- routers/web/admin/admin.go | 10 ++++++---- templates/admin/stacktrace-row.tmpl | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 648590e058526..d3b2dcfeccd7a 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -449,9 +449,9 @@ func GoroutineStacktrace(ctx *context.Context) { if value, ok := sample.Label[process.DescriptionPProfLabel]; ok && len(value) == 1 { description = value[0] + " " + description } - ptype := process.SystemProcessType + ptype := "code" if value, ok := sample.Label[process.ProcessTypePProfLabel]; ok && len(value) == 1 { - ptype = value[0] + stack.Labels = append(stack.Labels, &Label{Name: process.ProcessTypePProfLabel, Value: value[0]}) } processStack = &ProcessStack{ PID: pid, @@ -461,8 +461,10 @@ func GoroutineStacktrace(ctx *context.Context) { } pidMap[processStack.PID] = processStack - if parent, ok := pidMap[processStack.ParentPID]; ok { - parent.Children = append(parent.Children, processStack) + if processStack.ParentPID != "" { + if parent, ok := pidMap[processStack.ParentPID]; ok { + parent.Children = append(parent.Children, processStack) + } } } } diff --git a/templates/admin/stacktrace-row.tmpl b/templates/admin/stacktrace-row.tmpl index 88cda1c8d6f43..a5bbbb869e904 100644 --- a/templates/admin/stacktrace-row.tmpl +++ b/templates/admin/stacktrace-row.tmpl @@ -58,7 +58,7 @@ {{if .Process.Children}}
{{range .Process.Children}} - {{template "admin/process-row" dict "Process" . "root" $.root}} + {{template "admin/stacktrace-row" dict "Process" . "root" $.root}} {{end}}
{{end}} From 3b759d9a0b550982e824555a8e0e1eae66e4bff4 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 14:42:56 +0000 Subject: [PATCH 06/23] Allow for non-currently running process contexts to be created Signed-off-by: Andrew Thornton --- modules/context/private.go | 2 +- modules/eventsource/manager_run.go | 2 +- modules/graceful/manager_unix.go | 2 +- modules/log/event.go | 4 ++-- modules/process/manager.go | 30 ++++++++++++++++++++---------- modules/queue/workerpool.go | 2 ++ routers/common/middleware.go | 2 +- services/webhook/deliver.go | 2 +- 8 files changed, 29 insertions(+), 17 deletions(-) diff --git a/modules/context/private.go b/modules/context/private.go index 9b31234e54185..b57ba102e69a0 100644 --- a/modules/context/private.go +++ b/modules/context/private.go @@ -79,6 +79,6 @@ func PrivateContexter() func(http.Handler) http.Handler { // the underlying request has timed out from the ssh/http push func OverrideContext(ctx *PrivateContext) (cancel context.CancelFunc) { // We now need to override the request context as the base for our work because even if the request is cancelled we have to continue this work - ctx.Override, _, cancel = process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), fmt.Sprintf("PrivateContext: %s", ctx.Req.RequestURI), process.RequestProcessType) + ctx.Override, _, cancel = process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), fmt.Sprintf("PrivateContext: %s", ctx.Req.RequestURI), process.RequestProcessType, true) return } diff --git a/modules/eventsource/manager_run.go b/modules/eventsource/manager_run.go index 0e8dbbfc45c29..604fc266ce8d7 100644 --- a/modules/eventsource/manager_run.go +++ b/modules/eventsource/manager_run.go @@ -26,7 +26,7 @@ func (m *Manager) Init() { // Run runs the manager within a provided context func (m *Manager) Run(ctx context.Context) { - ctx, _, finished := process.GetManager().AddTypedContext(ctx, "EventSource", process.SystemProcessType) + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "EventSource", process.SystemProcessType, true) defer finished() then := timeutil.TimeStampNow().Add(-2) diff --git a/modules/graceful/manager_unix.go b/modules/graceful/manager_unix.go index 5611006e0bb22..b22b7b5860ae5 100644 --- a/modules/graceful/manager_unix.go +++ b/modules/graceful/manager_unix.go @@ -113,7 +113,7 @@ func (g *Manager) start(ctx context.Context) { } func (g *Manager) handleSignals(ctx context.Context) { - ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Graceful: HandleSignals", process.SystemProcessType) + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Graceful: HandleSignals", process.SystemProcessType, true) defer finished() signalChannel := make(chan os.Signal, 1) diff --git a/modules/log/event.go b/modules/log/event.go index cae8f0936fd66..dafc0e777400a 100644 --- a/modules/log/event.go +++ b/modules/log/event.go @@ -71,7 +71,7 @@ func NewChannelledLog(parent, name, provider, config string, bufferLength int64) // Start processing the ChannelledLog func (l *ChannelledLog) Start() { - _, _, cancel := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Logger: %s.%s(%s)", l.parent, l.name, l.provider), process.SystemProcessType) + _, _, cancel := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Logger: %s.%s(%s)", l.parent, l.name, l.provider), process.SystemProcessType, true) defer cancel() for { select { @@ -284,7 +284,7 @@ func (m *MultiChannelledLog) Start() { m.rwmutex.Unlock() return } - _, _, cancel := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Logger: %s", m.name), process.SystemProcessType) + _, _, cancel := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Logger: %s", m.name), process.SystemProcessType, true) defer cancel() m.started = true diff --git a/modules/process/manager.go b/modules/process/manager.go index 0e6e1095f5100..0417d60d90779 100644 --- a/modules/process/manager.go +++ b/modules/process/manager.go @@ -81,7 +81,7 @@ func GetManager() *Manager { func (pm *Manager) AddContext(parent context.Context, description string) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) { ctx, cancel = context.WithCancel(parent) - ctx, pid, finished := pm.Add(ctx, description, cancel, NormalProcessType) + ctx, pid, finished := pm.Add(ctx, description, cancel, NormalProcessType, true) return &Context{ Context: ctx, @@ -97,10 +97,10 @@ func (pm *Manager) AddContext(parent context.Context, description string) (ctx c // // Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the // process table. -func (pm *Manager) AddTypedContext(parent context.Context, description, processType string) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) { +func (pm *Manager) AddTypedContext(parent context.Context, description, processType string, currentlyRunning bool) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) { ctx, cancel = context.WithCancel(parent) - ctx, pid, finished := pm.Add(ctx, description, cancel, processType) + ctx, pid, finished := pm.Add(ctx, description, cancel, processType, currentlyRunning) return &Context{ Context: ctx, @@ -119,7 +119,7 @@ func (pm *Manager) AddTypedContext(parent context.Context, description, processT func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Duration, description string) (ctx context.Context, cancel context.CancelFunc, finshed FinishedFunc) { ctx, cancel = context.WithTimeout(parent, timeout) - ctx, pid, finshed := pm.Add(ctx, description, cancel, NormalProcessType) + ctx, pid, finshed := pm.Add(ctx, description, cancel, NormalProcessType, true) return &Context{ Context: ctx, @@ -128,7 +128,7 @@ func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Durati } // Add create a new process -func (pm *Manager) Add(ctx context.Context, description string, cancel context.CancelFunc, processType string) (context.Context, IDType, FinishedFunc) { +func (pm *Manager) Add(ctx context.Context, description string, cancel context.CancelFunc, processType string, currentlyRunning bool) (context.Context, IDType, FinishedFunc) { parentPID := GetParentPID(ctx) pm.mutex.Lock() @@ -148,10 +148,18 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C Type: processType, } - finished := func() { - cancel() - pm.remove(process) - pprof.SetGoroutineLabels(ctx) + var finished FinishedFunc + if currentlyRunning { + finished = func() { + cancel() + pm.remove(process) + pprof.SetGoroutineLabels(ctx) + } + } else { + finished = func() { + cancel() + pm.remove(process) + } } if parent != nil { @@ -161,7 +169,9 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C pm.mutex.Unlock() pprofCtx := pprof.WithLabels(ctx, pprof.Labels(DescriptionPProfLabel, description, PPIDPProfLabel, string(parentPID), PIDPProfLabel, string(pid), ProcessTypePProfLabel, processType)) - pprof.SetGoroutineLabels(pprofCtx) + if currentlyRunning { + pprof.SetGoroutineLabels(pprofCtx) + } return pprofCtx, pid, finished } diff --git a/modules/queue/workerpool.go b/modules/queue/workerpool.go index 5f6ec1871019e..e82edbab3425b 100644 --- a/modules/queue/workerpool.go +++ b/modules/queue/workerpool.go @@ -6,6 +6,7 @@ package queue import ( "context" + "runtime/pprof" "sync" "sync/atomic" "time" @@ -299,6 +300,7 @@ func (p *WorkerPool) addWorkers(ctx context.Context, cancel context.CancelFunc, p.numberOfWorkers++ p.lock.Unlock() go func() { + pprof.SetGoroutineLabels(ctx) p.doWork(ctx) p.lock.Lock() diff --git a/routers/common/middleware.go b/routers/common/middleware.go index db70dc46b1a89..6ea1e1dfbe5ea 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -27,7 +27,7 @@ func Middlewares() []func(http.Handler) http.Handler { // First of all escape the URL RawPath to ensure that all routing is done using a correctly escaped URL req.URL.RawPath = req.URL.EscapedPath() - ctx, _, finished := process.GetManager().AddTypedContext(req.Context(), fmt.Sprintf("%s: %s", req.Method, req.RequestURI), process.RequestProcessType) + ctx, _, finished := process.GetManager().AddTypedContext(req.Context(), fmt.Sprintf("%s: %s", req.Method, req.RequestURI), process.RequestProcessType, true) defer finished() next.ServeHTTP(context.NewResponse(resp), req.WithContext(ctx)) }) diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index 45744792b6f88..f27a8826f43c7 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -205,7 +205,7 @@ func DeliverHooks(ctx context.Context) { return default: } - ctx, _, finished := process.GetManager().AddTypedContext(ctx, "DeliverHooks", process.SystemProcessType) + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "DeliverHooks", process.SystemProcessType, true) defer finished() tasks, err := webhook_model.FindUndeliveredHookTasks() if err != nil { From 2edb985e2fbe8f19c0b47df9574e32e4dcb073e4 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 14:55:28 +0000 Subject: [PATCH 07/23] make loggers subprocesses Signed-off-by: Andrew Thornton --- modules/log/event.go | 23 ++++++++++++++++------- modules/log/multichannel.go | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/modules/log/event.go b/modules/log/event.go index dafc0e777400a..f66ecd179b701 100644 --- a/modules/log/event.go +++ b/modules/log/event.go @@ -7,6 +7,7 @@ package log import ( "context" "fmt" + "runtime/pprof" "sync" "time" @@ -37,7 +38,8 @@ type EventLogger interface { // ChannelledLog represents a cached channel to a LoggerProvider type ChannelledLog struct { - parent string + ctx context.Context + finished context.CancelFunc name string provider string queue chan *Event @@ -48,10 +50,10 @@ type ChannelledLog struct { } // NewChannelledLog a new logger instance with given logger provider and config. -func NewChannelledLog(parent, name, provider, config string, bufferLength int64) (*ChannelledLog, error) { +func NewChannelledLog(parent context.Context, name, provider, config string, bufferLength int64) (*ChannelledLog, error) { if log, ok := providers[provider]; ok { + l := &ChannelledLog{ - parent: parent, queue: make(chan *Event, bufferLength), flush: make(chan bool), close: make(chan bool), @@ -63,6 +65,7 @@ func NewChannelledLog(parent, name, provider, config string, bufferLength int64) } l.name = name l.provider = provider + l.ctx, _, l.finished = process.GetManager().AddTypedContext(parent, fmt.Sprintf("Logger: %s(%s)", l.name, l.provider), process.SystemProcessType, false) go l.Start() return l, nil } @@ -71,8 +74,8 @@ func NewChannelledLog(parent, name, provider, config string, bufferLength int64) // Start processing the ChannelledLog func (l *ChannelledLog) Start() { - _, _, cancel := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Logger: %s.%s(%s)", l.parent, l.name, l.provider), process.SystemProcessType, true) - defer cancel() + pprof.SetGoroutineLabels(l.ctx) + defer l.finished() for { select { case event, ok := <-l.queue: @@ -147,6 +150,8 @@ func (l *ChannelledLog) GetName() string { // MultiChannelledLog represents a cached channel to a LoggerProvider type MultiChannelledLog struct { + ctx context.Context + finished context.CancelFunc name string bufferLength int64 queue chan *Event @@ -163,7 +168,11 @@ type MultiChannelledLog struct { // NewMultiChannelledLog a new logger instance with given logger provider and config. func NewMultiChannelledLog(name string, bufferLength int64) *MultiChannelledLog { + ctx, _, finished := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Logger: %s", name), process.SystemProcessType, false) + m := &MultiChannelledLog{ + ctx: ctx, + finished: finished, name: name, queue: make(chan *Event, bufferLength), flush: make(chan bool), @@ -284,8 +293,8 @@ func (m *MultiChannelledLog) Start() { m.rwmutex.Unlock() return } - _, _, cancel := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Logger: %s", m.name), process.SystemProcessType, true) - defer cancel() + pprof.SetGoroutineLabels(m.ctx) + defer m.finished() m.started = true m.rwmutex.Unlock() diff --git a/modules/log/multichannel.go b/modules/log/multichannel.go index e5c13f1f011c5..6036d527ba437 100644 --- a/modules/log/multichannel.go +++ b/modules/log/multichannel.go @@ -31,7 +31,7 @@ func newLogger(name string, buffer int64) *MultiChannelledLogger { // SetLogger sets new logger instance with given logger provider and config. func (l *MultiChannelledLogger) SetLogger(name, provider, config string) error { - eventLogger, err := NewChannelledLog(l.name, name, provider, config, l.bufferLength) + eventLogger, err := NewChannelledLog(l.ctx, name, provider, config, l.bufferLength) if err != nil { return fmt.Errorf("Failed to create sublogger (%s): %v", name, err) } From b76195655a66525a99f287352d4d3541ac4d651c Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 15:15:24 +0000 Subject: [PATCH 08/23] Add process and labels for issueindexer Signed-off-by: Andrew Thornton --- modules/indexer/issues/indexer.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 3aaa27eed213b..749711989c204 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "os" + "runtime/pprof" "sync" "time" @@ -16,6 +17,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -165,6 +167,7 @@ func InitIssueIndexer(syncReindex bool) { // Create the Indexer go func() { + ctx, _, finished := process.GetManager().AddTypedContext(context.Background(), "IssueIndexer", process.SystemProcessType, true) start := time.Now() log.Info("PID %d: Initializing Issue Indexer: %s", os.Getpid(), setting.Indexer.IssueType) var populate bool @@ -193,11 +196,13 @@ func InitIssueIndexer(syncReindex bool) { if issueIndexer != nil { issueIndexer.Close() } + finished() log.Info("PID: %d Issue Indexer closed", os.Getpid()) }) log.Debug("Created Bleve Indexer") case "elasticsearch": graceful.GetManager().RunWithShutdownFns(func(_, atTerminate func(func())) { + pprof.SetGoroutineLabels(ctx) issueIndexer, err := NewElasticSearchIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueIndexerName) if err != nil { log.Fatal("Unable to initialize Elastic Search Issue Indexer at connection: %s Error: %v", setting.Indexer.IssueConnStr, err) @@ -208,10 +213,12 @@ func InitIssueIndexer(syncReindex bool) { } populate = !exist holder.set(issueIndexer) + atTerminate(finished) }) case "db": issueIndexer := &DBIndexer{} holder.set(issueIndexer) + graceful.GetManager().RunAtTerminate(finished) default: holder.cancel() log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType) @@ -272,6 +279,8 @@ func InitIssueIndexer(syncReindex bool) { // populateIssueIndexer populate the issue indexer with issue data func populateIssueIndexer(ctx context.Context) { + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "PopulateIssueIndexer", process.SystemProcessType, true) + defer finished() for page := 1; ; page++ { select { case <-ctx.Done(): From 8775a0314438995a94502c72d65dd3d355a425fb Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 15:42:21 +0000 Subject: [PATCH 09/23] Make queues appear as processes and label their goroutines too Signed-off-by: Andrew Thornton --- modules/queue/queue_bytefifo.go | 4 +++- modules/queue/queue_channel.go | 4 +++- modules/queue/queue_channel_test.go | 2 +- modules/queue/queue_disk_channel.go | 6 ++++-- modules/queue/unique_queue_channel.go | 3 +++ modules/queue/unique_queue_channel_test.go | 2 +- modules/queue/unique_queue_disk_channel.go | 7 +++++-- modules/queue/workerpool.go | 7 ++++++- services/pull/check_test.go | 2 +- 9 files changed, 27 insertions(+), 10 deletions(-) diff --git a/modules/queue/queue_bytefifo.go b/modules/queue/queue_bytefifo.go index ead3828f332b9..99c6428abce75 100644 --- a/modules/queue/queue_bytefifo.go +++ b/modules/queue/queue_bytefifo.go @@ -7,6 +7,7 @@ package queue import ( "context" "fmt" + "runtime/pprof" "sync" "sync/atomic" "time" @@ -20,7 +21,6 @@ import ( type ByteFIFOQueueConfiguration struct { WorkerPoolConfiguration Workers int - Name string WaitOnEmpty bool } @@ -153,6 +153,7 @@ func (q *ByteFIFOQueue) Flush(timeout time.Duration) error { // Run runs the bytefifo queue func (q *ByteFIFOQueue) Run(atShutdown, atTerminate func(func())) { + pprof.SetGoroutineLabels(q.baseCtx) atShutdown(q.Shutdown) atTerminate(q.Terminate) log.Debug("%s: %s Starting", q.typ, q.name) @@ -355,6 +356,7 @@ func (q *ByteFIFOQueue) Terminate() { if err := q.byteFIFO.Close(); err != nil { log.Error("Error whilst closing internal byte fifo in %s: %s: %v", q.typ, q.name, err) } + q.baseCtxFinished() log.Debug("%s: %s Terminated", q.typ, q.name) } diff --git a/modules/queue/queue_channel.go b/modules/queue/queue_channel.go index 5469c03100769..028023d50032c 100644 --- a/modules/queue/queue_channel.go +++ b/modules/queue/queue_channel.go @@ -7,6 +7,7 @@ package queue import ( "context" "fmt" + "runtime/pprof" "sync/atomic" "time" @@ -20,7 +21,6 @@ const ChannelQueueType Type = "channel" type ChannelQueueConfiguration struct { WorkerPoolConfiguration Workers int - Name string } // ChannelQueue implements Queue @@ -84,6 +84,7 @@ func NewChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, erro // Run starts to run the queue func (q *ChannelQueue) Run(atShutdown, atTerminate func(func())) { + pprof.SetGoroutineLabels(q.baseCtx) atShutdown(q.Shutdown) atTerminate(q.Terminate) log.Debug("ChannelQueue: %s Starting", q.name) @@ -169,6 +170,7 @@ func (q *ChannelQueue) Terminate() { default: } q.terminateCtxCancel() + q.baseCtxFinished() log.Debug("ChannelQueue: %s Terminated", q.name) } diff --git a/modules/queue/queue_channel_test.go b/modules/queue/queue_channel_test.go index 26a635b91897e..d30b90886151f 100644 --- a/modules/queue/queue_channel_test.go +++ b/modules/queue/queue_channel_test.go @@ -34,9 +34,9 @@ func TestChannelQueue(t *testing.T) { BlockTimeout: 1 * time.Second, BoostTimeout: 5 * time.Minute, BoostWorkers: 5, + Name: "TestChannelQueue", }, Workers: 0, - Name: "TestChannelQueue", }, &testData{}) assert.NoError(t, err) diff --git a/modules/queue/queue_disk_channel.go b/modules/queue/queue_disk_channel.go index 0494698e0ea22..014d93f5b5d52 100644 --- a/modules/queue/queue_disk_channel.go +++ b/modules/queue/queue_disk_channel.go @@ -7,6 +7,7 @@ package queue import ( "context" "fmt" + "runtime/pprof" "sync" "sync/atomic" "time" @@ -72,9 +73,9 @@ func NewPersistableChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) ( BoostTimeout: config.BoostTimeout, BoostWorkers: config.BoostWorkers, MaxWorkers: config.MaxWorkers, + Name: config.Name + "-channel", }, Workers: config.Workers, - Name: config.Name + "-channel", }, exemplar) if err != nil { return nil, err @@ -90,9 +91,9 @@ func NewPersistableChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) ( BoostTimeout: 5 * time.Minute, BoostWorkers: 1, MaxWorkers: 5, + Name: config.Name + "-level", }, Workers: 0, - Name: config.Name + "-level", }, DataDir: config.DataDir, } @@ -154,6 +155,7 @@ func (q *PersistableChannelQueue) PushBack(data Data) error { // Run starts to run the queue func (q *PersistableChannelQueue) Run(atShutdown, atTerminate func(func())) { + pprof.SetGoroutineLabels(q.channelQueue.baseCtx) log.Debug("PersistableChannelQueue: %s Starting", q.delayedStarter.name) _ = q.channelQueue.AddWorkers(q.channelQueue.workers, 0) diff --git a/modules/queue/unique_queue_channel.go b/modules/queue/unique_queue_channel.go index b7282e6c6cad7..6e8d37a20cc47 100644 --- a/modules/queue/unique_queue_channel.go +++ b/modules/queue/unique_queue_channel.go @@ -7,6 +7,7 @@ package queue import ( "context" "fmt" + "runtime/pprof" "sync" "sync/atomic" "time" @@ -97,6 +98,7 @@ func NewChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue // Run starts to run the queue func (q *ChannelUniqueQueue) Run(atShutdown, atTerminate func(func())) { + pprof.SetGoroutineLabels(q.baseCtx) atShutdown(q.Shutdown) atTerminate(q.Terminate) log.Debug("ChannelUniqueQueue: %s Starting", q.name) @@ -226,6 +228,7 @@ func (q *ChannelUniqueQueue) Terminate() { default: } q.terminateCtxCancel() + q.baseCtxFinished() log.Debug("ChannelUniqueQueue: %s Terminated", q.name) } diff --git a/modules/queue/unique_queue_channel_test.go b/modules/queue/unique_queue_channel_test.go index ef6752079e149..6daf3fc96ecbc 100644 --- a/modules/queue/unique_queue_channel_test.go +++ b/modules/queue/unique_queue_channel_test.go @@ -32,9 +32,9 @@ func TestChannelUniqueQueue(t *testing.T) { BlockTimeout: 1 * time.Second, BoostTimeout: 5 * time.Minute, BoostWorkers: 5, + Name: "TestChannelQueue", }, Workers: 0, - Name: "TestChannelQueue", }, &testData{}) assert.NoError(t, err) diff --git a/modules/queue/unique_queue_disk_channel.go b/modules/queue/unique_queue_disk_channel.go index 5ee1c396fc6a7..6ab03094ba785 100644 --- a/modules/queue/unique_queue_disk_channel.go +++ b/modules/queue/unique_queue_disk_channel.go @@ -6,6 +6,7 @@ package queue import ( "context" + "runtime/pprof" "sync" "time" @@ -72,9 +73,9 @@ func NewPersistableChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interfac BoostTimeout: config.BoostTimeout, BoostWorkers: config.BoostWorkers, MaxWorkers: config.MaxWorkers, + Name: config.Name + "-channel", }, Workers: config.Workers, - Name: config.Name + "-channel", }, exemplar) if err != nil { return nil, err @@ -90,9 +91,9 @@ func NewPersistableChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interfac BoostTimeout: 5 * time.Minute, BoostWorkers: 1, MaxWorkers: 5, + Name: config.Name + "-level", }, Workers: 0, - Name: config.Name + "-level", }, DataDir: config.DataDir, } @@ -183,6 +184,7 @@ func (q *PersistableChannelUniqueQueue) Has(data Data) (bool, error) { // Run starts to run the queue func (q *PersistableChannelUniqueQueue) Run(atShutdown, atTerminate func(func())) { + pprof.SetGoroutineLabels(q.channelQueue.baseCtx) log.Debug("PersistableChannelUniqueQueue: %s Starting", q.delayedStarter.name) q.lock.Lock() @@ -301,6 +303,7 @@ func (q *PersistableChannelUniqueQueue) Terminate() { if q.internal != nil { q.internal.(*LevelUniqueQueue).Terminate() } + q.channelQueue.baseCtxFinished() log.Debug("PersistableChannelUniqueQueue: %s Terminated", q.delayedStarter.name) } diff --git a/modules/queue/workerpool.go b/modules/queue/workerpool.go index e82edbab3425b..a03916fbc7e46 100644 --- a/modules/queue/workerpool.go +++ b/modules/queue/workerpool.go @@ -6,12 +6,14 @@ package queue import ( "context" + "fmt" "runtime/pprof" "sync" "sync/atomic" "time" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/util" ) @@ -23,6 +25,7 @@ type WorkerPool struct { lock sync.Mutex baseCtx context.Context baseCtxCancel context.CancelFunc + baseCtxFinished process.FinishedFunc paused chan struct{} resumed chan struct{} cond *sync.Cond @@ -45,6 +48,7 @@ var ( // WorkerPoolConfiguration is the basic configuration for a WorkerPool type WorkerPoolConfiguration struct { + Name string QueueLength int BatchLength int BlockTimeout time.Duration @@ -55,12 +59,13 @@ type WorkerPoolConfiguration struct { // NewWorkerPool creates a new worker pool func NewWorkerPool(handle HandlerFunc, config WorkerPoolConfiguration) *WorkerPool { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel, finished := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Queue: %s", config.Name), process.SystemProcessType, false) dataChan := make(chan Data, config.QueueLength) pool := &WorkerPool{ baseCtx: ctx, baseCtxCancel: cancel, + baseCtxFinished: finished, batchLength: config.BatchLength, dataChan: dataChan, resumed: closedChan, diff --git a/services/pull/check_test.go b/services/pull/check_test.go index 4cdd17cc7b5c2..65bcb9c0e44db 100644 --- a/services/pull/check_test.go +++ b/services/pull/check_test.go @@ -32,9 +32,9 @@ func TestPullRequest_AddToTaskQueue(t *testing.T) { WorkerPoolConfiguration: queue.WorkerPoolConfiguration{ QueueLength: 10, BatchLength: 1, + Name: "temporary-queue", }, Workers: 1, - Name: "temporary-queue", }, "") assert.NoError(t, err) From ff2f5a0525f69cd548e127a804f48bc744b8a9f9 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 15:43:53 +0000 Subject: [PATCH 10/23] Add some common prefixes Signed-off-by: Andrew Thornton --- modules/eventsource/manager_run.go | 2 +- modules/indexer/issues/indexer.go | 4 ++-- services/webhook/deliver.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/eventsource/manager_run.go b/modules/eventsource/manager_run.go index 604fc266ce8d7..9af5c9e78ac13 100644 --- a/modules/eventsource/manager_run.go +++ b/modules/eventsource/manager_run.go @@ -26,7 +26,7 @@ func (m *Manager) Init() { // Run runs the manager within a provided context func (m *Manager) Run(ctx context.Context) { - ctx, _, finished := process.GetManager().AddTypedContext(ctx, "EventSource", process.SystemProcessType, true) + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Service: EventSource", process.SystemProcessType, true) defer finished() then := timeutil.TimeStampNow().Add(-2) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 749711989c204..0c8ef02d1f58b 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -167,7 +167,7 @@ func InitIssueIndexer(syncReindex bool) { // Create the Indexer go func() { - ctx, _, finished := process.GetManager().AddTypedContext(context.Background(), "IssueIndexer", process.SystemProcessType, true) + ctx, _, finished := process.GetManager().AddTypedContext(context.Background(), "Service: IssueIndexer", process.SystemProcessType, true) start := time.Now() log.Info("PID %d: Initializing Issue Indexer: %s", os.Getpid(), setting.Indexer.IssueType) var populate bool @@ -279,7 +279,7 @@ func InitIssueIndexer(syncReindex bool) { // populateIssueIndexer populate the issue indexer with issue data func populateIssueIndexer(ctx context.Context) { - ctx, _, finished := process.GetManager().AddTypedContext(ctx, "PopulateIssueIndexer", process.SystemProcessType, true) + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Service: PopulateIssueIndexer", process.SystemProcessType, true) defer finished() for page := 1; ; page++ { select { diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index f27a8826f43c7..fab13b53dfacd 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -205,7 +205,7 @@ func DeliverHooks(ctx context.Context) { return default: } - ctx, _, finished := process.GetManager().AddTypedContext(ctx, "DeliverHooks", process.SystemProcessType, true) + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Service: DeliverHooks", process.SystemProcessType, true) defer finished() tasks, err := webhook_model.FindUndeliveredHookTasks() if err != nil { From 9fd39d36da8dc8ee88b5619476eaeb19d618df06 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 16:35:46 +0000 Subject: [PATCH 11/23] Add web processes Signed-off-by: Andrew Thornton --- cmd/web.go | 8 ++++++++ cmd/web_acme.go | 4 ++++ modules/queue/workerpool.go | 1 + modules/ssh/ssh.go | 8 +++++++- modules/web/routing/logger_manager.go | 7 +++++-- 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/cmd/web.go b/cmd/web.go index 710c12775fd02..8c7c026172745 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/routers" "code.gitea.io/gitea/routers/install" @@ -59,6 +60,9 @@ and it takes care of all the other things for you`, } func runHTTPRedirector() { + _, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), "Web: HTTP Redirector", process.SystemProcessType, true) + defer finished() + source := fmt.Sprintf("%s:%s", setting.HTTPAddr, setting.PortToRedirect) dest := strings.TrimSuffix(setting.AppURL, "/") log.Info("Redirecting: %s to %s", source, dest) @@ -141,8 +145,10 @@ func runWeb(ctx *cli.Context) error { if setting.EnablePprof { go func() { + _, _, finished := process.GetManager().AddTypedContext(context.Background(), "Web: PProf Server", process.SystemProcessType, true) log.Info("Starting pprof server on localhost:6060") log.Info("%v", http.ListenAndServe("localhost:6060", nil)) + finished() }() } @@ -204,6 +210,8 @@ func listen(m http.Handler, handleRedirector bool) error { if setting.Protocol != setting.HTTPUnix && setting.Protocol != setting.FCGIUnix { listenAddr = net.JoinHostPort(listenAddr, setting.HTTPPort) } + _, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), "Web: Gitea Server", process.SystemProcessType, true) + defer finished() log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL) // This can be useful for users, many users do wrong to their config and get strange behaviors behind a reverse-proxy. // A user may fix the configuration mistake when he sees this log. diff --git a/cmd/web_acme.go b/cmd/web_acme.go index 459d4f0a76974..7dbeb14a0e64f 100644 --- a/cmd/web_acme.go +++ b/cmd/web_acme.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "github.com/caddyserver/certmagic" @@ -107,6 +108,9 @@ func runACME(listenAddr string, m http.Handler) error { if enableHTTPChallenge { go func() { + _, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), "Web: ACME HTTP challenge server", process.SystemProcessType, true) + defer finished() + log.Info("Running Let's Encrypt handler on %s", setting.HTTPAddr+":"+setting.PortToRedirect) // all traffic coming into HTTP will be redirect to HTTPS automatically (LE HTTP-01 validation happens here) err := runHTTP("tcp", setting.HTTPAddr+":"+setting.PortToRedirect, "Let's Encrypt HTTP Challenge", myACME.HTTPChallengeHandler(http.HandlerFunc(runLetsEncryptFallbackHandler))) diff --git a/modules/queue/workerpool.go b/modules/queue/workerpool.go index a03916fbc7e46..2d8504598a1af 100644 --- a/modules/queue/workerpool.go +++ b/modules/queue/workerpool.go @@ -483,6 +483,7 @@ func (p *WorkerPool) FlushWithContext(ctx context.Context) error { } func (p *WorkerPool) doWork(ctx context.Context) { + pprof.SetGoroutineLabels(ctx) delay := time.Millisecond * 300 // Create a common timer - we will use this elsewhere diff --git a/modules/ssh/ssh.go b/modules/ssh/ssh.go index 1a92edb795148..70c9731c841ec 100644 --- a/modules/ssh/ssh.go +++ b/modules/ssh/ssh.go @@ -23,7 +23,9 @@ import ( "syscall" asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -317,7 +319,11 @@ func Listen(host string, port int, ciphers, keyExchanges, macs []string) { } } - go listen(&srv) + go func() { + _, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), "Service: SSH", process.SystemProcessType, true) + defer finished() + listen(&srv) + }() } // GenKeyPair make a pair of public and private keys for SSH access. diff --git a/modules/web/routing/logger_manager.go b/modules/web/routing/logger_manager.go index cc434c338da5a..7715b0b5d37f4 100644 --- a/modules/web/routing/logger_manager.go +++ b/modules/web/routing/logger_manager.go @@ -11,6 +11,7 @@ import ( "time" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/process" ) // Event indicates when the printer is triggered @@ -40,7 +41,9 @@ type requestRecordsManager struct { } func (manager *requestRecordsManager) startSlowQueryDetector(threshold time.Duration) { - go graceful.GetManager().RunWithShutdownContext(func(baseCtx context.Context) { + go graceful.GetManager().RunWithShutdownContext(func(ctx context.Context) { + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Service: SlowQueryDetector", process.SystemProcessType, true) + defer finished() // This go-routine checks all active requests every second. // If a request has been running for a long time (eg: /user/events), we also print a log with "still-executing" message // After the "still-executing" log is printed, the record will be removed from the map to prevent from duplicated logs in future @@ -49,7 +52,7 @@ func (manager *requestRecordsManager) startSlowQueryDetector(threshold time.Dura t := time.NewTicker(time.Second) for { select { - case <-baseCtx.Done(): + case <-ctx.Done(): return case <-t.C: now := time.Now() From b388c38896e63740498c79bf29911559ed149209 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 17:06:55 +0000 Subject: [PATCH 12/23] label cron, code and issues indexers Signed-off-by: Andrew Thornton --- modules/indexer/code/indexer.go | 7 ++++++- modules/indexer/issues/indexer.go | 5 ++++- routers/init.go | 2 +- services/cron/cron.go | 12 ++++++++++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index d897fcccd5489..3ead3261e9c02 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -7,6 +7,7 @@ package code import ( "context" "os" + "runtime/pprof" "strconv" "strings" "time" @@ -15,6 +16,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -116,7 +118,7 @@ func Init() { return } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel, finished := process.GetManager().AddTypedContext(context.Background(), "Service: CodeIndexer", process.SystemProcessType, false) graceful.GetManager().RunAtTerminate(func() { select { @@ -128,6 +130,7 @@ func Init() { log.Debug("Closing repository indexer") indexer.Close() log.Info("PID: %d Repository Indexer closed", os.Getpid()) + finished() }) waitChannel := make(chan time.Duration) @@ -172,6 +175,7 @@ func Init() { } go func() { + pprof.SetGoroutineLabels(ctx) start := time.Now() var ( rIndexer Indexer @@ -247,6 +251,7 @@ func Init() { if setting.Indexer.StartupTimeout > 0 { go func() { + pprof.SetGoroutineLabels(ctx) timeout := setting.Indexer.StartupTimeout if graceful.GetManager().IsChild() && setting.GracefulHammerTime > 0 { timeout += setting.GracefulHammerTime diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 0c8ef02d1f58b..1343b0bddd372 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -102,6 +102,8 @@ var ( // InitIssueIndexer initialize issue indexer, syncReindex is true then reindex until // all issue index done. func InitIssueIndexer(syncReindex bool) { + ctx, _, finished := process.GetManager().AddTypedContext(context.Background(), "Service: IssueIndexer", process.SystemProcessType, false) + waitChannel := make(chan time.Duration) // Create the Queue @@ -167,7 +169,7 @@ func InitIssueIndexer(syncReindex bool) { // Create the Indexer go func() { - ctx, _, finished := process.GetManager().AddTypedContext(context.Background(), "Service: IssueIndexer", process.SystemProcessType, true) + pprof.SetGoroutineLabels(ctx) start := time.Now() log.Info("PID %d: Initializing Issue Indexer: %s", os.Getpid(), setting.Indexer.IssueType) var populate bool @@ -258,6 +260,7 @@ func InitIssueIndexer(syncReindex bool) { } } else if setting.Indexer.StartupTimeout > 0 { go func() { + pprof.SetGoroutineLabels(ctx) timeout := setting.Indexer.StartupTimeout if graceful.GetManager().IsChild() && setting.GracefulHammerTime > 0 { timeout += setting.GracefulHammerTime diff --git a/routers/init.go b/routers/init.go index 804dfd65335b6..2ccc187a393fa 100644 --- a/routers/init.go +++ b/routers/init.go @@ -140,7 +140,7 @@ func GlobalInitInstalled(ctx context.Context) { models.NewRepoContext() // Booting long running goroutines. - cron.NewContext() + cron.NewContext(ctx) issue_indexer.InitIssueIndexer(false) code_indexer.Init() mustInit(stats_indexer.Init) diff --git a/services/cron/cron.go b/services/cron/cron.go index 19f703caf1848..acd03533cbba7 100644 --- a/services/cron/cron.go +++ b/services/cron/cron.go @@ -7,15 +7,20 @@ package cron import ( "context" + "runtime/pprof" "time" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/sync" "github.com/gogs/cron" ) -var c = cron.New() +var ( + c = cron.New() + cronContext context.Context +) // Prevent duplicate running tasks. var taskStatusTable = sync.NewStatusTable() @@ -23,7 +28,9 @@ var taskStatusTable = sync.NewStatusTable() // NewContext begins cron tasks // Each cron task is run within the shutdown context as a running server // AtShutdown the cron server is stopped -func NewContext() { +func NewContext(original context.Context) { + defer pprof.SetGoroutineLabels(original) + _, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().ShutdownContext(), "Service: Cron", process.SystemProcessType, true) initBasicTasks() initExtendedTasks() @@ -42,6 +49,7 @@ func NewContext() { lock.Lock() started = false lock.Unlock() + finished() }) } From f39554c53e300d7679ff6eb83f7107cc5c665b74 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 17:33:31 +0000 Subject: [PATCH 13/23] Ensure NoSQLDB goroutines are associated properly Signed-off-by: Andrew Thornton --- modules/nosql/manager.go | 9 ++- modules/nosql/manager_leveldb.go | 33 ++++++---- modules/nosql/manager_redis.go | 104 +++++++++++++++++-------------- 3 files changed, 88 insertions(+), 58 deletions(-) diff --git a/modules/nosql/manager.go b/modules/nosql/manager.go index a89b5bb633907..dab30812ce73f 100644 --- a/modules/nosql/manager.go +++ b/modules/nosql/manager.go @@ -5,10 +5,12 @@ package nosql import ( + "context" "strconv" "sync" "time" + "code.gitea.io/gitea/modules/process" "github.com/go-redis/redis/v8" "github.com/syndtr/goleveldb/leveldb" ) @@ -17,7 +19,9 @@ var manager *Manager // Manager is the nosql connection manager type Manager struct { - mutex sync.Mutex + ctx context.Context + finished context.CancelFunc + mutex sync.Mutex RedisConnections map[string]*redisClientHolder LevelDBConnections map[string]*levelDBHolder @@ -46,7 +50,10 @@ func init() { // GetManager returns a Manager and initializes one as singleton is there's none yet func GetManager() *Manager { if manager == nil { + ctx, _, finished := process.GetManager().AddTypedContext(context.Background(), "Service: NoSQL", process.SystemProcessType, false) manager = &Manager{ + ctx: ctx, + finished: finished, RedisConnections: make(map[string]*redisClientHolder), LevelDBConnections: make(map[string]*levelDBHolder), } diff --git a/modules/nosql/manager_leveldb.go b/modules/nosql/manager_leveldb.go index de4ef14d7dcf4..d2dc302e82ef9 100644 --- a/modules/nosql/manager_leveldb.go +++ b/modules/nosql/manager_leveldb.go @@ -7,6 +7,7 @@ package nosql import ( "fmt" "path" + "runtime/pprof" "strconv" "strings" @@ -163,20 +164,30 @@ func (m *Manager) GetLevelDB(connection string) (*leveldb.DB, error) { } } + // Now because we want associate any goroutines created by this call to the main nosqldb context we need to + // wrap the openFile within a goroutine which we can label nicely. var err error - db.db, err = leveldb.OpenFile(dataDir, opts) - if err != nil { - if !errors.IsCorrupted(err) { - if strings.Contains(err.Error(), "resource temporarily unavailable") { - return nil, fmt.Errorf("unable to lock level db at %s: %w", dataDir, err) - } - - return nil, fmt.Errorf("unable to open level db at %s: %w", dataDir, err) - } - db.db, err = leveldb.RecoverFile(dataDir, opts) + done := make(chan struct{}) + go func() { + pprof.SetGoroutineLabels(m.ctx) + defer close(done) + db.db, err = leveldb.OpenFile(dataDir, opts) if err != nil { - return nil, err + if !errors.IsCorrupted(err) { + if strings.Contains(err.Error(), "resource temporarily unavailable") { + err = fmt.Errorf("unable to lock level db at %s: %w", dataDir, err) + return + } + + err = fmt.Errorf("unable to open level db at %s: %w", dataDir, err) + return + } + db.db, err = leveldb.RecoverFile(dataDir, opts) } + }() + <-done + if err != nil { + return nil, err } for _, name := range db.name { diff --git a/modules/nosql/manager_redis.go b/modules/nosql/manager_redis.go index b4852cecc849e..074c97cbf7373 100644 --- a/modules/nosql/manager_redis.go +++ b/modules/nosql/manager_redis.go @@ -7,6 +7,7 @@ package nosql import ( "crypto/tls" "path" + "runtime/pprof" "strconv" "strings" @@ -141,57 +142,68 @@ func (m *Manager) GetRedisClient(connection string) redis.UniversalClient { } } - switch uri.Scheme { - case "redis+sentinels": - fallthrough - case "rediss+sentinel": - opts.TLSConfig = tlsConfig - fallthrough - case "redis+sentinel": - if uri.Host != "" { - opts.Addrs = append(opts.Addrs, strings.Split(uri.Host, ",")...) - } - if uri.Path != "" { - if db, err := strconv.Atoi(uri.Path[1:]); err == nil { - opts.DB = db + done := make(chan struct{}) + go func() { + defer close(done) + pprof.SetGoroutineLabels(m.ctx) + + switch uri.Scheme { + case "redis+sentinels": + fallthrough + case "rediss+sentinel": + opts.TLSConfig = tlsConfig + fallthrough + case "redis+sentinel": + if uri.Host != "" { + opts.Addrs = append(opts.Addrs, strings.Split(uri.Host, ",")...) + } + if uri.Path != "" { + if db, err := strconv.Atoi(uri.Path[1:]); err == nil { + opts.DB = db + } } - } - client.UniversalClient = redis.NewFailoverClient(opts.Failover()) - case "redis+clusters": - fallthrough - case "rediss+cluster": - opts.TLSConfig = tlsConfig - fallthrough - case "redis+cluster": - if uri.Host != "" { - opts.Addrs = append(opts.Addrs, strings.Split(uri.Host, ",")...) - } - if uri.Path != "" { - if db, err := strconv.Atoi(uri.Path[1:]); err == nil { - opts.DB = db + client.UniversalClient = redis.NewFailoverClient(opts.Failover()).WithContext(m.ctx) + case "redis+clusters": + fallthrough + case "rediss+cluster": + opts.TLSConfig = tlsConfig + fallthrough + case "redis+cluster": + if uri.Host != "" { + opts.Addrs = append(opts.Addrs, strings.Split(uri.Host, ",")...) } - } - client.UniversalClient = redis.NewClusterClient(opts.Cluster()) - case "redis+socket": - simpleOpts := opts.Simple() - simpleOpts.Network = "unix" - simpleOpts.Addr = path.Join(uri.Host, uri.Path) - client.UniversalClient = redis.NewClient(simpleOpts) - case "rediss": - opts.TLSConfig = tlsConfig - fallthrough - case "redis": - if uri.Host != "" { - opts.Addrs = append(opts.Addrs, strings.Split(uri.Host, ",")...) - } - if uri.Path != "" { - if db, err := strconv.Atoi(uri.Path[1:]); err == nil { - opts.DB = db + if uri.Path != "" { + if db, err := strconv.Atoi(uri.Path[1:]); err == nil { + opts.DB = db + } } + client.UniversalClient = redis.NewClusterClient(opts.Cluster()).WithContext(m.ctx) + case "redis+socket": + simpleOpts := opts.Simple() + simpleOpts.Network = "unix" + simpleOpts.Addr = path.Join(uri.Host, uri.Path) + client.UniversalClient = redis.NewClient(simpleOpts).WithContext(m.ctx) + case "rediss": + opts.TLSConfig = tlsConfig + fallthrough + case "redis": + if uri.Host != "" { + opts.Addrs = append(opts.Addrs, strings.Split(uri.Host, ",")...) + } + if uri.Path != "" { + if db, err := strconv.Atoi(uri.Path[1:]); err == nil { + opts.DB = db + } + } + client.UniversalClient = redis.NewClient(opts.Simple()).WithContext(m.ctx) + default: + return } - client.UniversalClient = redis.NewClient(opts.Simple()) - default: + }() + <-done + + if client.UniversalClient == nil { return nil } From df11f4c9da863cd18158cd67aa6e140b90da650f Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 18:11:04 +0000 Subject: [PATCH 14/23] placate lint Signed-off-by: Andrew Thornton --- services/cron/cron.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/cron/cron.go b/services/cron/cron.go index acd03533cbba7..5d120cd905ba8 100644 --- a/services/cron/cron.go +++ b/services/cron/cron.go @@ -17,10 +17,7 @@ import ( "github.com/gogs/cron" ) -var ( - c = cron.New() - cronContext context.Context -) +var c = cron.New() // Prevent duplicate running tasks. var taskStatusTable = sync.NewStatusTable() From 7eebec788ee3025df59333eb67c0dc6595b9bf18 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 25 Mar 2022 18:13:16 +0000 Subject: [PATCH 15/23] placate lint again Signed-off-by: Andrew Thornton --- templates/admin/process-row.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/admin/process-row.tmpl b/templates/admin/process-row.tmpl index 2e20c5c1f7d8b..2191677a5cee7 100644 --- a/templates/admin/process-row.tmpl +++ b/templates/admin/process-row.tmpl @@ -1,6 +1,6 @@
-
{{if eq .Process.Type "request"}}{{svg "octicon-globe" 16 }}{{else if eq .Process.Type "system"}}{{svg "octicon-cpu" 16 }}{{else}}{{svg "octicon-terminal" 16 }}{{end}}
+
{{if eq .Process.Type "request"}}{{svg "octicon-globe" 16 }}{{else if eq .Process.Type "system"}}{{svg "octicon-cpu" 16 }}{{else}}{{svg "octicon-terminal" 16 }}{{end}}
{{.Process.Description}}
{{TimeSince .Process.Start .root.i18n.Lang}}
From 5b35caa18fa827ed873131be470f675f2b41c162 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 26 Mar 2022 17:08:17 +0000 Subject: [PATCH 16/23] Add manager command to list processes Signed-off-by: Andrew Thornton --- cmd/manager.go | 375 ++---------------------- cmd/manager_logging.go | 382 +++++++++++++++++++++++++ modules/private/manager.go | 23 ++ modules/process/error.go | 26 ++ modules/process/manager.go | 159 +--------- modules/process/manager_exec.go | 79 +++++ modules/process/manager_stacktraces.go | 256 +++++++++++++++++ modules/process/manager_test.go | 6 +- modules/process/process.go | 31 +- routers/private/internal.go | 1 + routers/private/manager_process.go | 150 ++++++++++ routers/web/admin/admin.go | 173 +---------- 12 files changed, 987 insertions(+), 674 deletions(-) create mode 100644 cmd/manager_logging.go create mode 100644 modules/process/error.go create mode 100644 modules/process/manager_exec.go create mode 100644 modules/process/manager_stacktraces.go create mode 100644 routers/private/manager_process.go diff --git a/cmd/manager.go b/cmd/manager.go index 50b66cc7f2301..01804945bea69 100644 --- a/cmd/manager.go +++ b/cmd/manager.go @@ -10,7 +10,6 @@ import ( "os" "time" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" "github.com/urfave/cli" @@ -27,6 +26,7 @@ var ( subcmdRestart, subcmdFlushQueues, subcmdLogging, + subCmdProcesses, }, } subcmdShutdown = cli.Command{ @@ -68,326 +68,34 @@ var ( }, }, } - defaultLoggingFlags = []cli.Flag{ - cli.StringFlag{ - Name: "group, g", - Usage: "Group to add logger to - will default to \"default\"", - }, cli.StringFlag{ - Name: "name, n", - Usage: "Name of the new logger - will default to mode", - }, cli.StringFlag{ - Name: "level, l", - Usage: "Logging level for the new logger", - }, cli.StringFlag{ - Name: "stacktrace-level, L", - Usage: "Stacktrace logging level", - }, cli.StringFlag{ - Name: "flags, F", - Usage: "Flags for the logger", - }, cli.StringFlag{ - Name: "expression, e", - Usage: "Matching expression for the logger", - }, cli.StringFlag{ - Name: "prefix, p", - Usage: "Prefix for the logger", - }, cli.BoolFlag{ - Name: "color", - Usage: "Use color in the logs", - }, cli.BoolFlag{ - Name: "debug", - }, - } - subcmdLogging = cli.Command{ - Name: "logging", - Usage: "Adjust logging commands", - Subcommands: []cli.Command{ - { - Name: "pause", - Usage: "Pause logging (Gitea will buffer logs up to a certain point and will drop them after that point)", - Flags: []cli.Flag{ - cli.BoolFlag{ - Name: "debug", - }, - }, - Action: runPauseLogging, - }, { - Name: "resume", - Usage: "Resume logging", - Flags: []cli.Flag{ - cli.BoolFlag{ - Name: "debug", - }, - }, - Action: runResumeLogging, - }, { - Name: "release-and-reopen", - Usage: "Cause Gitea to release and re-open files used for logging", - Flags: []cli.Flag{ - cli.BoolFlag{ - Name: "debug", - }, - }, - Action: runReleaseReopenLogging, - }, { - Name: "remove", - Usage: "Remove a logger", - ArgsUsage: "[name] Name of logger to remove", - Flags: []cli.Flag{ - cli.BoolFlag{ - Name: "debug", - }, cli.StringFlag{ - Name: "group, g", - Usage: "Group to add logger to - will default to \"default\"", - }, - }, - Action: runRemoveLogger, - }, { - Name: "add", - Usage: "Add a logger", - Subcommands: []cli.Command{ - { - Name: "console", - Usage: "Add a console logger", - Flags: append(defaultLoggingFlags, - cli.BoolFlag{ - Name: "stderr", - Usage: "Output console logs to stderr - only relevant for console", - }), - Action: runAddConsoleLogger, - }, { - Name: "file", - Usage: "Add a file logger", - Flags: append(defaultLoggingFlags, []cli.Flag{ - cli.StringFlag{ - Name: "filename, f", - Usage: "Filename for the logger - this must be set.", - }, cli.BoolTFlag{ - Name: "rotate, r", - Usage: "Rotate logs", - }, cli.Int64Flag{ - Name: "max-size, s", - Usage: "Maximum size in bytes before rotation", - }, cli.BoolTFlag{ - Name: "daily, d", - Usage: "Rotate logs daily", - }, cli.IntFlag{ - Name: "max-days, D", - Usage: "Maximum number of daily logs to keep", - }, cli.BoolTFlag{ - Name: "compress, z", - Usage: "Compress rotated logs", - }, cli.IntFlag{ - Name: "compression-level, Z", - Usage: "Compression level to use", - }, - }...), - Action: runAddFileLogger, - }, { - Name: "conn", - Usage: "Add a net conn logger", - Flags: append(defaultLoggingFlags, []cli.Flag{ - cli.BoolFlag{ - Name: "reconnect-on-message, R", - Usage: "Reconnect to host for every message", - }, cli.BoolFlag{ - Name: "reconnect, r", - Usage: "Reconnect to host when connection is dropped", - }, cli.StringFlag{ - Name: "protocol, P", - Usage: "Set protocol to use: tcp, unix, or udp (defaults to tcp)", - }, cli.StringFlag{ - Name: "address, a", - Usage: "Host address and port to connect to (defaults to :7020)", - }, - }...), - Action: runAddConnLogger, - }, { - Name: "smtp", - Usage: "Add an SMTP logger", - Flags: append(defaultLoggingFlags, []cli.Flag{ - cli.StringFlag{ - Name: "username, u", - Usage: "Mail server username", - }, cli.StringFlag{ - Name: "password, P", - Usage: "Mail server password", - }, cli.StringFlag{ - Name: "host, H", - Usage: "Mail server host (defaults to: 127.0.0.1:25)", - }, cli.StringSliceFlag{ - Name: "send-to, s", - Usage: "Email address(es) to send to", - }, cli.StringFlag{ - Name: "subject, S", - Usage: "Subject header of sent emails", - }, - }...), - Action: runAddSMTPLogger, - }, - }, + subCmdProcesses = cli.Command{ + Name: "processes", + Usage: "Display running processes within the current process", + Action: runProcesses, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "debug", + }, + cli.BoolFlag{ + Name: "no-children", + Usage: "Show processes as flat table rather than as tree", + }, + cli.BoolFlag{ + Name: "requests-only", + Usage: "Only show request processes", + }, + cli.BoolFlag{ + Name: "stacktraces", + Usage: "Show stacktraces", + }, + cli.BoolFlag{ + Name: "json", + Usage: "Output as json", }, }, } ) -func runRemoveLogger(c *cli.Context) error { - setup("manager", c.Bool("debug")) - group := c.String("group") - if len(group) == 0 { - group = log.DEFAULT - } - name := c.Args().First() - ctx, cancel := installSignals() - defer cancel() - - statusCode, msg := private.RemoveLogger(ctx, group, name) - switch statusCode { - case http.StatusInternalServerError: - return fail("InternalServerError", msg) - } - - fmt.Fprintln(os.Stdout, msg) - return nil -} - -func runAddSMTPLogger(c *cli.Context) error { - setup("manager", c.Bool("debug")) - vals := map[string]interface{}{} - mode := "smtp" - if c.IsSet("host") { - vals["host"] = c.String("host") - } else { - vals["host"] = "127.0.0.1:25" - } - - if c.IsSet("username") { - vals["username"] = c.String("username") - } - if c.IsSet("password") { - vals["password"] = c.String("password") - } - - if !c.IsSet("send-to") { - return fmt.Errorf("Some recipients must be provided") - } - vals["sendTos"] = c.StringSlice("send-to") - - if c.IsSet("subject") { - vals["subject"] = c.String("subject") - } else { - vals["subject"] = "Diagnostic message from Gitea" - } - - return commonAddLogger(c, mode, vals) -} - -func runAddConnLogger(c *cli.Context) error { - setup("manager", c.Bool("debug")) - vals := map[string]interface{}{} - mode := "conn" - vals["net"] = "tcp" - if c.IsSet("protocol") { - switch c.String("protocol") { - case "udp": - vals["net"] = "udp" - case "unix": - vals["net"] = "unix" - } - } - if c.IsSet("address") { - vals["address"] = c.String("address") - } else { - vals["address"] = ":7020" - } - if c.IsSet("reconnect") { - vals["reconnect"] = c.Bool("reconnect") - } - if c.IsSet("reconnect-on-message") { - vals["reconnectOnMsg"] = c.Bool("reconnect-on-message") - } - return commonAddLogger(c, mode, vals) -} - -func runAddFileLogger(c *cli.Context) error { - setup("manager", c.Bool("debug")) - vals := map[string]interface{}{} - mode := "file" - if c.IsSet("filename") { - vals["filename"] = c.String("filename") - } else { - return fmt.Errorf("filename must be set when creating a file logger") - } - if c.IsSet("rotate") { - vals["rotate"] = c.Bool("rotate") - } - if c.IsSet("max-size") { - vals["maxsize"] = c.Int64("max-size") - } - if c.IsSet("daily") { - vals["daily"] = c.Bool("daily") - } - if c.IsSet("max-days") { - vals["maxdays"] = c.Int("max-days") - } - if c.IsSet("compress") { - vals["compress"] = c.Bool("compress") - } - if c.IsSet("compression-level") { - vals["compressionLevel"] = c.Int("compression-level") - } - return commonAddLogger(c, mode, vals) -} - -func runAddConsoleLogger(c *cli.Context) error { - setup("manager", c.Bool("debug")) - vals := map[string]interface{}{} - mode := "console" - if c.IsSet("stderr") && c.Bool("stderr") { - vals["stderr"] = c.Bool("stderr") - } - return commonAddLogger(c, mode, vals) -} - -func commonAddLogger(c *cli.Context, mode string, vals map[string]interface{}) error { - if len(c.String("level")) > 0 { - vals["level"] = log.FromString(c.String("level")).String() - } - if len(c.String("stacktrace-level")) > 0 { - vals["stacktraceLevel"] = log.FromString(c.String("stacktrace-level")).String() - } - if len(c.String("expression")) > 0 { - vals["expression"] = c.String("expression") - } - if len(c.String("prefix")) > 0 { - vals["prefix"] = c.String("prefix") - } - if len(c.String("flags")) > 0 { - vals["flags"] = log.FlagsFromString(c.String("flags")) - } - if c.IsSet("color") { - vals["colorize"] = c.Bool("color") - } - group := "default" - if c.IsSet("group") { - group = c.String("group") - } - name := mode - if c.IsSet("name") { - name = c.String("name") - } - ctx, cancel := installSignals() - defer cancel() - - statusCode, msg := private.AddLogger(ctx, group, name, mode, vals) - switch statusCode { - case http.StatusInternalServerError: - return fail("InternalServerError", msg) - } - - fmt.Fprintln(os.Stdout, msg) - return nil -} - func runShutdown(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() @@ -433,47 +141,16 @@ func runFlushQueues(c *cli.Context) error { return nil } -func runPauseLogging(c *cli.Context) error { +func runProcesses(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() setup("manager", c.Bool("debug")) - statusCode, msg := private.PauseLogging(ctx) + statusCode, msg := private.Processes(ctx, os.Stdout, c.Bool("no-children"), c.Bool("requests-only"), c.Bool("stacktraces"), c.Bool("json")) switch statusCode { case http.StatusInternalServerError: return fail("InternalServerError", msg) } - fmt.Fprintln(os.Stdout, msg) - return nil -} - -func runResumeLogging(c *cli.Context) error { - ctx, cancel := installSignals() - defer cancel() - - setup("manager", c.Bool("debug")) - statusCode, msg := private.ResumeLogging(ctx) - switch statusCode { - case http.StatusInternalServerError: - return fail("InternalServerError", msg) - } - - fmt.Fprintln(os.Stdout, msg) - return nil -} - -func runReleaseReopenLogging(c *cli.Context) error { - ctx, cancel := installSignals() - defer cancel() - - setup("manager", c.Bool("debug")) - statusCode, msg := private.ReleaseReopenLogging(ctx) - switch statusCode { - case http.StatusInternalServerError: - return fail("InternalServerError", msg) - } - - fmt.Fprintln(os.Stdout, msg) return nil } diff --git a/cmd/manager_logging.go b/cmd/manager_logging.go new file mode 100644 index 0000000000000..eb311d28926c4 --- /dev/null +++ b/cmd/manager_logging.go @@ -0,0 +1,382 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "fmt" + "net/http" + "os" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/private" + "github.com/urfave/cli" +) + +var ( + defaultLoggingFlags = []cli.Flag{ + cli.StringFlag{ + Name: "group, g", + Usage: "Group to add logger to - will default to \"default\"", + }, cli.StringFlag{ + Name: "name, n", + Usage: "Name of the new logger - will default to mode", + }, cli.StringFlag{ + Name: "level, l", + Usage: "Logging level for the new logger", + }, cli.StringFlag{ + Name: "stacktrace-level, L", + Usage: "Stacktrace logging level", + }, cli.StringFlag{ + Name: "flags, F", + Usage: "Flags for the logger", + }, cli.StringFlag{ + Name: "expression, e", + Usage: "Matching expression for the logger", + }, cli.StringFlag{ + Name: "prefix, p", + Usage: "Prefix for the logger", + }, cli.BoolFlag{ + Name: "color", + Usage: "Use color in the logs", + }, cli.BoolFlag{ + Name: "debug", + }, + } + + subcmdLogging = cli.Command{ + Name: "logging", + Usage: "Adjust logging commands", + Subcommands: []cli.Command{ + { + Name: "pause", + Usage: "Pause logging (Gitea will buffer logs up to a certain point and will drop them after that point)", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "debug", + }, + }, + Action: runPauseLogging, + }, { + Name: "resume", + Usage: "Resume logging", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "debug", + }, + }, + Action: runResumeLogging, + }, { + Name: "release-and-reopen", + Usage: "Cause Gitea to release and re-open files used for logging", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "debug", + }, + }, + Action: runReleaseReopenLogging, + }, { + Name: "remove", + Usage: "Remove a logger", + ArgsUsage: "[name] Name of logger to remove", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "debug", + }, cli.StringFlag{ + Name: "group, g", + Usage: "Group to add logger to - will default to \"default\"", + }, + }, + Action: runRemoveLogger, + }, { + Name: "add", + Usage: "Add a logger", + Subcommands: []cli.Command{ + { + Name: "console", + Usage: "Add a console logger", + Flags: append(defaultLoggingFlags, + cli.BoolFlag{ + Name: "stderr", + Usage: "Output console logs to stderr - only relevant for console", + }), + Action: runAddConsoleLogger, + }, { + Name: "file", + Usage: "Add a file logger", + Flags: append(defaultLoggingFlags, []cli.Flag{ + cli.StringFlag{ + Name: "filename, f", + Usage: "Filename for the logger - this must be set.", + }, cli.BoolTFlag{ + Name: "rotate, r", + Usage: "Rotate logs", + }, cli.Int64Flag{ + Name: "max-size, s", + Usage: "Maximum size in bytes before rotation", + }, cli.BoolTFlag{ + Name: "daily, d", + Usage: "Rotate logs daily", + }, cli.IntFlag{ + Name: "max-days, D", + Usage: "Maximum number of daily logs to keep", + }, cli.BoolTFlag{ + Name: "compress, z", + Usage: "Compress rotated logs", + }, cli.IntFlag{ + Name: "compression-level, Z", + Usage: "Compression level to use", + }, + }...), + Action: runAddFileLogger, + }, { + Name: "conn", + Usage: "Add a net conn logger", + Flags: append(defaultLoggingFlags, []cli.Flag{ + cli.BoolFlag{ + Name: "reconnect-on-message, R", + Usage: "Reconnect to host for every message", + }, cli.BoolFlag{ + Name: "reconnect, r", + Usage: "Reconnect to host when connection is dropped", + }, cli.StringFlag{ + Name: "protocol, P", + Usage: "Set protocol to use: tcp, unix, or udp (defaults to tcp)", + }, cli.StringFlag{ + Name: "address, a", + Usage: "Host address and port to connect to (defaults to :7020)", + }, + }...), + Action: runAddConnLogger, + }, { + Name: "smtp", + Usage: "Add an SMTP logger", + Flags: append(defaultLoggingFlags, []cli.Flag{ + cli.StringFlag{ + Name: "username, u", + Usage: "Mail server username", + }, cli.StringFlag{ + Name: "password, P", + Usage: "Mail server password", + }, cli.StringFlag{ + Name: "host, H", + Usage: "Mail server host (defaults to: 127.0.0.1:25)", + }, cli.StringSliceFlag{ + Name: "send-to, s", + Usage: "Email address(es) to send to", + }, cli.StringFlag{ + Name: "subject, S", + Usage: "Subject header of sent emails", + }, + }...), + Action: runAddSMTPLogger, + }, + }, + }, + }, + } +) + +func runRemoveLogger(c *cli.Context) error { + setup("manager", c.Bool("debug")) + group := c.String("group") + if len(group) == 0 { + group = log.DEFAULT + } + name := c.Args().First() + ctx, cancel := installSignals() + defer cancel() + + statusCode, msg := private.RemoveLogger(ctx, group, name) + switch statusCode { + case http.StatusInternalServerError: + return fail("InternalServerError", msg) + } + + fmt.Fprintln(os.Stdout, msg) + return nil +} + +func runAddSMTPLogger(c *cli.Context) error { + setup("manager", c.Bool("debug")) + vals := map[string]interface{}{} + mode := "smtp" + if c.IsSet("host") { + vals["host"] = c.String("host") + } else { + vals["host"] = "127.0.0.1:25" + } + + if c.IsSet("username") { + vals["username"] = c.String("username") + } + if c.IsSet("password") { + vals["password"] = c.String("password") + } + + if !c.IsSet("send-to") { + return fmt.Errorf("Some recipients must be provided") + } + vals["sendTos"] = c.StringSlice("send-to") + + if c.IsSet("subject") { + vals["subject"] = c.String("subject") + } else { + vals["subject"] = "Diagnostic message from Gitea" + } + + return commonAddLogger(c, mode, vals) +} + +func runAddConnLogger(c *cli.Context) error { + setup("manager", c.Bool("debug")) + vals := map[string]interface{}{} + mode := "conn" + vals["net"] = "tcp" + if c.IsSet("protocol") { + switch c.String("protocol") { + case "udp": + vals["net"] = "udp" + case "unix": + vals["net"] = "unix" + } + } + if c.IsSet("address") { + vals["address"] = c.String("address") + } else { + vals["address"] = ":7020" + } + if c.IsSet("reconnect") { + vals["reconnect"] = c.Bool("reconnect") + } + if c.IsSet("reconnect-on-message") { + vals["reconnectOnMsg"] = c.Bool("reconnect-on-message") + } + return commonAddLogger(c, mode, vals) +} + +func runAddFileLogger(c *cli.Context) error { + setup("manager", c.Bool("debug")) + vals := map[string]interface{}{} + mode := "file" + if c.IsSet("filename") { + vals["filename"] = c.String("filename") + } else { + return fmt.Errorf("filename must be set when creating a file logger") + } + if c.IsSet("rotate") { + vals["rotate"] = c.Bool("rotate") + } + if c.IsSet("max-size") { + vals["maxsize"] = c.Int64("max-size") + } + if c.IsSet("daily") { + vals["daily"] = c.Bool("daily") + } + if c.IsSet("max-days") { + vals["maxdays"] = c.Int("max-days") + } + if c.IsSet("compress") { + vals["compress"] = c.Bool("compress") + } + if c.IsSet("compression-level") { + vals["compressionLevel"] = c.Int("compression-level") + } + return commonAddLogger(c, mode, vals) +} + +func runAddConsoleLogger(c *cli.Context) error { + setup("manager", c.Bool("debug")) + vals := map[string]interface{}{} + mode := "console" + if c.IsSet("stderr") && c.Bool("stderr") { + vals["stderr"] = c.Bool("stderr") + } + return commonAddLogger(c, mode, vals) +} + +func commonAddLogger(c *cli.Context, mode string, vals map[string]interface{}) error { + if len(c.String("level")) > 0 { + vals["level"] = log.FromString(c.String("level")).String() + } + if len(c.String("stacktrace-level")) > 0 { + vals["stacktraceLevel"] = log.FromString(c.String("stacktrace-level")).String() + } + if len(c.String("expression")) > 0 { + vals["expression"] = c.String("expression") + } + if len(c.String("prefix")) > 0 { + vals["prefix"] = c.String("prefix") + } + if len(c.String("flags")) > 0 { + vals["flags"] = log.FlagsFromString(c.String("flags")) + } + if c.IsSet("color") { + vals["colorize"] = c.Bool("color") + } + group := "default" + if c.IsSet("group") { + group = c.String("group") + } + name := mode + if c.IsSet("name") { + name = c.String("name") + } + ctx, cancel := installSignals() + defer cancel() + + statusCode, msg := private.AddLogger(ctx, group, name, mode, vals) + switch statusCode { + case http.StatusInternalServerError: + return fail("InternalServerError", msg) + } + + fmt.Fprintln(os.Stdout, msg) + return nil +} + +func runPauseLogging(c *cli.Context) error { + ctx, cancel := installSignals() + defer cancel() + + setup("manager", c.Bool("debug")) + statusCode, msg := private.PauseLogging(ctx) + switch statusCode { + case http.StatusInternalServerError: + return fail("InternalServerError", msg) + } + + fmt.Fprintln(os.Stdout, msg) + return nil +} + +func runResumeLogging(c *cli.Context) error { + ctx, cancel := installSignals() + defer cancel() + + setup("manager", c.Bool("debug")) + statusCode, msg := private.ResumeLogging(ctx) + switch statusCode { + case http.StatusInternalServerError: + return fail("InternalServerError", msg) + } + + fmt.Fprintln(os.Stdout, msg) + return nil +} + +func runReleaseReopenLogging(c *cli.Context) error { + ctx, cancel := installSignals() + defer cancel() + + setup("manager", c.Bool("debug")) + statusCode, msg := private.ReleaseReopenLogging(ctx) + switch statusCode { + case http.StatusInternalServerError: + return fail("InternalServerError", msg) + } + + fmt.Fprintln(os.Stdout, msg) + return nil +} diff --git a/modules/private/manager.go b/modules/private/manager.go index 2543e141ea41f..e1d43170bf5a5 100644 --- a/modules/private/manager.go +++ b/modules/private/manager.go @@ -7,6 +7,7 @@ package private import ( "context" "fmt" + "io" "net/http" "net/url" "time" @@ -189,3 +190,25 @@ func RemoveLogger(ctx context.Context, group, name string) (int, string) { return http.StatusOK, "Removed" } + +// Processes return the current processes from this gitea instance +func Processes(ctx context.Context, out io.Writer, flat, onlyRequests, stacktraces, json bool) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/processes?flat=%t&requests-only=%t&stacktraces=%t&json=%t", flat, onlyRequests, stacktraces, json) + + req := newInternalRequest(ctx, reqURL, "GET") + resp, err := req.Response() + if err != nil { + return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return resp.StatusCode, decodeJSONError(resp).Err + } + + _, err = io.Copy(out, resp.Body) + if err != nil { + return http.StatusInternalServerError, err.Error() + } + return http.StatusOK, "" +} diff --git a/modules/process/error.go b/modules/process/error.go new file mode 100644 index 0000000000000..7a72bda40e3b5 --- /dev/null +++ b/modules/process/error.go @@ -0,0 +1,26 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package process + +import "fmt" + +// Error is a wrapped error describing the error results of Process Execution +type Error struct { + PID IDType + Description string + Err error + CtxErr error + Stdout string + Stderr string +} + +func (err *Error) Error() string { + return fmt.Sprintf("exec(%s:%s) failed: %v(%v) stdout: %s stderr: %s", err.PID, err.Description, err.Err, err.CtxErr, err.Stdout, err.Stderr) +} + +// Unwrap implements the unwrappable implicit interface for go1.13 Unwrap() +func (err *Error) Unwrap() error { + return err.Err +} diff --git a/modules/process/manager.go b/modules/process/manager.go index 0417d60d90779..ef0233d20ae0f 100644 --- a/modules/process/manager.go +++ b/modules/process/manager.go @@ -6,13 +6,8 @@ package process import ( - "bytes" "context" - "fmt" - "io" - "os/exec" "runtime/pprof" - "sort" "strconv" "sync" "time" @@ -56,14 +51,14 @@ type Manager struct { next int64 lastTime int64 - processes map[IDType]*Process + processes map[IDType]*process } // GetManager returns a Manager and initializes one as singleton if there's none yet func GetManager() *Manager { managerInit.Do(func() { manager = &Manager{ - processes: make(map[IDType]*Process), + processes: make(map[IDType]*process), next: 1, } }) @@ -81,12 +76,9 @@ func GetManager() *Manager { func (pm *Manager) AddContext(parent context.Context, description string) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) { ctx, cancel = context.WithCancel(parent) - ctx, pid, finished := pm.Add(ctx, description, cancel, NormalProcessType, true) + ctx, _, finished = pm.Add(ctx, description, cancel, NormalProcessType, true) - return &Context{ - Context: ctx, - pid: pid, - }, cancel, finished + return ctx, cancel, finished } // AddTypedContext creates a new context and adds it as a process. Once the process is finished, finished must be called @@ -100,12 +92,9 @@ func (pm *Manager) AddContext(parent context.Context, description string) (ctx c func (pm *Manager) AddTypedContext(parent context.Context, description, processType string, currentlyRunning bool) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) { ctx, cancel = context.WithCancel(parent) - ctx, pid, finished := pm.Add(ctx, description, cancel, processType, currentlyRunning) + ctx, _, finished = pm.Add(ctx, description, cancel, processType, currentlyRunning) - return &Context{ - Context: ctx, - pid: pid, - }, cancel, finished + return ctx, cancel, finished } // AddContextTimeout creates a new context and add it as a process. Once the process is finished, finished must be called @@ -119,12 +108,9 @@ func (pm *Manager) AddTypedContext(parent context.Context, description, processT func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Duration, description string) (ctx context.Context, cancel context.CancelFunc, finshed FinishedFunc) { ctx, cancel = context.WithTimeout(parent, timeout) - ctx, pid, finshed := pm.Add(ctx, description, cancel, NormalProcessType, true) + ctx, _, finshed = pm.Add(ctx, description, cancel, NormalProcessType, true) - return &Context{ - Context: ctx, - pid: pid, - }, cancel, finshed + return ctx, cancel, finshed } // Add create a new process @@ -139,7 +125,7 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C parentPID = "" } - process := &Process{ + process := &process{ PID: pid, ParentPID: parentPID, Description: description, @@ -173,7 +159,10 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C pprof.SetGoroutineLabels(pprofCtx) } - return pprofCtx, pid, finished + return &Context{ + Context: pprofCtx, + pid: pid, + }, pid, finished } // nextPID will return the next available PID. pm.mutex should already be locked. @@ -202,7 +191,7 @@ func (pm *Manager) Remove(pid IDType) { pm.mutex.Unlock() } -func (pm *Manager) remove(process *Process) { +func (pm *Manager) remove(process *process) { pm.mutex.Lock() if p := pm.processes[process.PID]; p == process { delete(pm.processes, process.PID) @@ -226,123 +215,3 @@ func (pm *Manager) Cancel(pid IDType) { process.Cancel() } } - -// Processes gets the processes in a thread safe manner -func (pm *Manager) Processes(onlyRoots, noSystem bool, runInLock func()) []*Process { - pm.mutex.Lock() - processes := make([]*Process, 0, len(pm.processes)) - if onlyRoots { - for _, process := range pm.processes { - if noSystem && process.Type == SystemProcessType { - continue - } - if parent, has := pm.processes[process.ParentPID]; !has || parent.Type == SystemProcessType { - processes = append(processes, process) - } - } - } else { - for _, process := range pm.processes { - if noSystem && process.Type == SystemProcessType { - continue - } - processes = append(processes, process) - } - } - if runInLock != nil { - runInLock() - } - pm.mutex.Unlock() - - sort.Slice(processes, func(i, j int) bool { - left, right := processes[i], processes[j] - - return left.Start.Before(right.Start) - }) - - return processes -} - -// Exec a command and use the default timeout. -func (pm *Manager) Exec(desc, cmdName string, args ...string) (string, string, error) { - return pm.ExecDir(DefaultContext, -1, "", desc, cmdName, args...) -} - -// ExecTimeout a command and use a specific timeout duration. -func (pm *Manager) ExecTimeout(timeout time.Duration, desc, cmdName string, args ...string) (string, string, error) { - return pm.ExecDir(DefaultContext, timeout, "", desc, cmdName, args...) -} - -// ExecDir a command and use the default timeout. -func (pm *Manager) ExecDir(ctx context.Context, timeout time.Duration, dir, desc, cmdName string, args ...string) (string, string, error) { - return pm.ExecDirEnv(ctx, timeout, dir, desc, nil, cmdName, args...) -} - -// ExecDirEnv runs a command in given path and environment variables, and waits for its completion -// up to the given timeout (or DefaultTimeout if -1 is given). -// Returns its complete stdout and stderr -// outputs and an error, if any (including timeout) -func (pm *Manager) ExecDirEnv(ctx context.Context, timeout time.Duration, dir, desc string, env []string, cmdName string, args ...string) (string, string, error) { - return pm.ExecDirEnvStdIn(ctx, timeout, dir, desc, env, nil, cmdName, args...) -} - -// ExecDirEnvStdIn runs a command in given path and environment variables with provided stdIN, and waits for its completion -// up to the given timeout (or DefaultTimeout if -1 is given). -// Returns its complete stdout and stderr -// outputs and an error, if any (including timeout) -func (pm *Manager) ExecDirEnvStdIn(ctx context.Context, timeout time.Duration, dir, desc string, env []string, stdIn io.Reader, cmdName string, args ...string) (string, string, error) { - if timeout == -1 { - timeout = 60 * time.Second - } - - stdOut := new(bytes.Buffer) - stdErr := new(bytes.Buffer) - - ctx, _, finished := pm.AddContextTimeout(ctx, timeout, desc) - defer finished() - - cmd := exec.CommandContext(ctx, cmdName, args...) - cmd.Dir = dir - cmd.Env = env - cmd.Stdout = stdOut - cmd.Stderr = stdErr - if stdIn != nil { - cmd.Stdin = stdIn - } - - if err := cmd.Start(); err != nil { - return "", "", err - } - - err := cmd.Wait() - if err != nil { - err = &Error{ - PID: GetPID(ctx), - Description: desc, - Err: err, - CtxErr: ctx.Err(), - Stdout: stdOut.String(), - Stderr: stdErr.String(), - } - } - - return stdOut.String(), stdErr.String(), err -} - -// Error is a wrapped error describing the error results of Process Execution -type Error struct { - PID IDType - Description string - Err error - CtxErr error - Stdout string - Stderr string -} - -func (err *Error) Error() string { - return fmt.Sprintf("exec(%s:%s) failed: %v(%v) stdout: %s stderr: %s", err.PID, err.Description, err.Err, err.CtxErr, err.Stdout, err.Stderr) -} - -// Unwrap implements the unwrappable implicit interface for go1.13 Unwrap() -func (err *Error) Unwrap() error { - return err.Err -} diff --git a/modules/process/manager_exec.go b/modules/process/manager_exec.go new file mode 100644 index 0000000000000..f1406b8aeb6ea --- /dev/null +++ b/modules/process/manager_exec.go @@ -0,0 +1,79 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package process + +import ( + "bytes" + "context" + "io" + "os/exec" + "time" +) + +// Exec a command and use the default timeout. +func (pm *Manager) Exec(desc, cmdName string, args ...string) (string, string, error) { + return pm.ExecDir(DefaultContext, -1, "", desc, cmdName, args...) +} + +// ExecTimeout a command and use a specific timeout duration. +func (pm *Manager) ExecTimeout(timeout time.Duration, desc, cmdName string, args ...string) (string, string, error) { + return pm.ExecDir(DefaultContext, timeout, "", desc, cmdName, args...) +} + +// ExecDir a command and use the default timeout. +func (pm *Manager) ExecDir(ctx context.Context, timeout time.Duration, dir, desc, cmdName string, args ...string) (string, string, error) { + return pm.ExecDirEnv(ctx, timeout, dir, desc, nil, cmdName, args...) +} + +// ExecDirEnv runs a command in given path and environment variables, and waits for its completion +// up to the given timeout (or DefaultTimeout if -1 is given). +// Returns its complete stdout and stderr +// outputs and an error, if any (including timeout) +func (pm *Manager) ExecDirEnv(ctx context.Context, timeout time.Duration, dir, desc string, env []string, cmdName string, args ...string) (string, string, error) { + return pm.ExecDirEnvStdIn(ctx, timeout, dir, desc, env, nil, cmdName, args...) +} + +// ExecDirEnvStdIn runs a command in given path and environment variables with provided stdIN, and waits for its completion +// up to the given timeout (or DefaultTimeout if -1 is given). +// Returns its complete stdout and stderr +// outputs and an error, if any (including timeout) +func (pm *Manager) ExecDirEnvStdIn(ctx context.Context, timeout time.Duration, dir, desc string, env []string, stdIn io.Reader, cmdName string, args ...string) (string, string, error) { + if timeout == -1 { + timeout = 60 * time.Second + } + + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + ctx, _, finished := pm.AddContextTimeout(ctx, timeout, desc) + defer finished() + + cmd := exec.CommandContext(ctx, cmdName, args...) + cmd.Dir = dir + cmd.Env = env + cmd.Stdout = stdOut + cmd.Stderr = stdErr + if stdIn != nil { + cmd.Stdin = stdIn + } + + if err := cmd.Start(); err != nil { + return "", "", err + } + + err := cmd.Wait() + if err != nil { + err = &Error{ + PID: GetPID(ctx), + Description: desc, + Err: err, + CtxErr: ctx.Err(), + Stdout: stdOut.String(), + Stderr: stdErr.String(), + } + } + + return stdOut.String(), stdErr.String(), err +} diff --git a/modules/process/manager_stacktraces.go b/modules/process/manager_stacktraces.go new file mode 100644 index 0000000000000..137fb3c9fdfa1 --- /dev/null +++ b/modules/process/manager_stacktraces.go @@ -0,0 +1,256 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package process + +import ( + "fmt" + "io" + "runtime/pprof" + "sort" + "time" + + "github.com/google/pprof/profile" +) + +// StackEntry is an entry on a stacktrace +type StackEntry struct { + Function string + File string + Line int +} + +// Label represents a pprof label assigned to goroutine stack +type Label struct { + Name string + Value string +} + +// Stack is a stacktrace relating to a goroutine. (Multiple goroutines may have the same stacktrace) +type Stack struct { + Count int64 // Number of goroutines with this stack trace + Description string + Labels []*Label `json:",omitempty"` + Entry []*StackEntry `json:",omitempty"` +} + +// A Process is a combined representation of a Process and a Stacktrace for the goroutines associated with it +type Process struct { + PID IDType + ParentPID IDType + Description string + Start time.Time + Type string + + Children []*Process `json:",omitempty"` + Stacks []*Stack `json:",omitempty"` +} + +// Processes gets the processes in a thread safe manner +func (pm *Manager) Processes(onlyRoots, noSystem bool, runInLock func()) []*Process { + pm.mutex.Lock() + processes := make([]*Process, 0, len(pm.processes)) + if onlyRoots { + for _, process := range pm.processes { + if noSystem && process.Type == SystemProcessType { + continue + } + if parent, has := pm.processes[process.ParentPID]; !has || + (noSystem && parent.Type == SystemProcessType) { + processes = append(processes, process.ToProcess(true)) + } + } + } else { + for _, process := range pm.processes { + if noSystem && process.Type == SystemProcessType { + continue + } + processes = append(processes, process.ToProcess(false)) + } + } + if runInLock != nil { + runInLock() + } + pm.mutex.Unlock() + + sort.Slice(processes, func(i, j int) bool { + left, right := processes[i], processes[j] + + return left.Start.Before(right.Start) + }) + + return processes +} + +// ProcessStacktraces gets the processes and stacktraces in a thread safe manner +func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int64, error) { + var stacks *profile.Profile + var err error + processes := pm.Processes(false, false, func() { + reader, writer := io.Pipe() + defer reader.Close() + go func() { + err := pprof.Lookup("goroutine").WriteTo(writer, 0) + _ = writer.CloseWithError(err) + }() + stacks, err = profile.Parse(reader) + if err != nil { + return + } + }) + if err != nil { + return nil, 0, err + } + + // We cannot use the process pidmaps here because we have released the mutex ... + pidMap := map[IDType]*Process{} + processStacks := make([]*Process, 0, len(processes)) + for _, process := range processes { + pStack := &Process{ + PID: process.PID, + ParentPID: process.ParentPID, + Description: process.Description, + Start: process.Start, + Type: process.Type, + } + + pidMap[process.PID] = pStack + if flat { + processStacks = append(processStacks, pStack) + } else if parent, ok := pidMap[process.ParentPID]; ok { + parent.Children = append(parent.Children, pStack) + } else { + processStacks = append(processStacks, pStack) + } + } + + goroutineCount := int64(0) + + // Now walk through the "Sample" slice in the goroutines stack + for _, sample := range stacks.Sample { + stack := &Stack{} + + // Add the labels + for name, value := range sample.Label { + if name == DescriptionPProfLabel || name == PIDPProfLabel || (!flat && name == PPIDPProfLabel) || name == ProcessTypePProfLabel { + continue + } + if len(value) != 1 { + // Unexpected... + return nil, 0, fmt.Errorf("label: %s in goroutine stack with unexpected number of values: %v", name, value) + } + + stack.Labels = append(stack.Labels, &Label{Name: name, Value: value[0]}) + } + + stack.Count = sample.Value[0] + goroutineCount += stack.Count + + // Now get the processStack for this goroutine sample + var processStack *Process + if pidvalue, ok := sample.Label[PIDPProfLabel]; ok && len(pidvalue) == 1 { + pid := IDType(pidvalue[0]) + processStack, ok = pidMap[pid] + if !ok && pid != "" { + ppid := IDType("") + if value, ok := sample.Label[PPIDPProfLabel]; ok && len(value) == 1 { + ppid = IDType(value[0]) + } + description := "(dead process)" + if value, ok := sample.Label[DescriptionPProfLabel]; ok && len(value) == 1 { + description = value[0] + " " + description + } + ptype := "code" + if value, ok := sample.Label[ProcessTypePProfLabel]; ok && len(value) == 1 { + stack.Labels = append(stack.Labels, &Label{Name: ProcessTypePProfLabel, Value: value[0]}) + } + processStack = &Process{ + PID: pid, + ParentPID: ppid, + Description: description, + Type: ptype, + } + + pidMap[processStack.PID] = processStack + added := false + if processStack.ParentPID != "" && !flat { + if parent, ok := pidMap[processStack.ParentPID]; ok { + parent.Children = append(parent.Children, processStack) + added = true + } + } + if !added { + processStacks = append(processStacks, processStack) + } + } + } + if processStack == nil { + var ok bool + processStack, ok = pidMap[""] + if !ok { + processStack = &Process{ + Description: "(unassociated)", + Type: "code", + } + pidMap[processStack.PID] = processStack + processStacks = append(processStacks, processStack) + } + } + + // Now walk through the locations... + for _, location := range sample.Location { + for _, line := range location.Line { + entry := &StackEntry{ + Function: line.Function.Name, + File: line.Function.Filename, + Line: int(line.Line), + } + stack.Entry = append(stack.Entry, entry) + } + } + stack.Description = "(unknown)" + if len(stack.Entry) > 0 { + stack.Description = stack.Entry[len(stack.Entry)-1].Function + } + + processStack.Stacks = append(processStack.Stacks, stack) + } + + if onlyRequests { + var requestStacks []*Process + i := len(processStacks) - 1 + for i >= 0 { + processStack := processStacks[i] + if processStack.Type == RequestProcessType { + requestStacks = append(requestStacks, processStack) + i-- + continue + } + if len(processStack.Children) > 0 { + processStacks = processStacks[:i] + processStacks = append(processStacks, processStack.Children...) + i = len(processStacks) - 1 + continue + } + i-- + } + processStacks = requestStacks + } + + // Now finally re-sort the processstacks so the newest processes are at the top + after := func(processStacks []*Process) func(i, j int) bool { + return func(i, j int) bool { + left, right := processStacks[i], processStacks[j] + return left.Start.After(right.Start) + } + } + sort.Slice(processStacks, after(processStacks)) + if !flat { + for _, processStack := range processStacks { + sort.Slice(processStack.Children, after(processStack.Children)) + } + } + + return processStacks, goroutineCount, err +} diff --git a/modules/process/manager_test.go b/modules/process/manager_test.go index 152c7a92359ca..36394b0c7bddb 100644 --- a/modules/process/manager_test.go +++ b/modules/process/manager_test.go @@ -22,7 +22,7 @@ func TestGetManager(t *testing.T) { } func TestManager_AddContext(t *testing.T) { - pm := Manager{processes: make(map[IDType]*Process), next: 1} + pm := Manager{processes: make(map[IDType]*process), next: 1} ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -41,7 +41,7 @@ func TestManager_AddContext(t *testing.T) { } func TestManager_Cancel(t *testing.T) { - pm := Manager{processes: make(map[IDType]*Process), next: 1} + pm := Manager{processes: make(map[IDType]*process), next: 1} ctx, _, finished := pm.AddContext(context.Background(), "foo") defer finished() @@ -69,7 +69,7 @@ func TestManager_Cancel(t *testing.T) { } func TestManager_Remove(t *testing.T) { - pm := Manager{processes: make(map[IDType]*Process), next: 1} + pm := Manager{processes: make(map[IDType]*process), next: 1} ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/modules/process/process.go b/modules/process/process.go index e104ace970b02..fc869cfa0582f 100644 --- a/modules/process/process.go +++ b/modules/process/process.go @@ -16,8 +16,8 @@ var ( NormalProcessType = "normal" ) -// Process represents a working process inheriting from Gitea. -type Process struct { +// process represents a working process inheriting from Gitea. +type process struct { PID IDType // Process ID, not system one. ParentPID IDType Description string @@ -26,26 +26,26 @@ type Process struct { Type string lock sync.Mutex - children []*Process + children []*process } // Children gets the children of the process // Note: this function will behave nicely even if p is nil -func (p *Process) Children() (children []*Process) { +func (p *process) Children() (children []*process) { if p == nil { return } p.lock.Lock() defer p.lock.Unlock() - children = make([]*Process, len(p.children)) + children = make([]*process, len(p.children)) copy(children, p.children) return children } // AddChild adds a child process // Note: this function will behave nicely even if p is nil -func (p *Process) AddChild(child *Process) { +func (p *process) AddChild(child *process) { if p == nil { return } @@ -57,7 +57,7 @@ func (p *Process) AddChild(child *Process) { // RemoveChild removes a child process // Note: this function will behave nicely even if p is nil -func (p *Process) RemoveChild(process *Process) { +func (p *process) RemoveChild(process *process) { if p == nil { return } @@ -71,3 +71,20 @@ func (p *Process) RemoveChild(process *Process) { } } } + +// ToProcess converts a process to a externally usable Process +func (p *process) ToProcess(children bool) *Process { + process := &Process{ + PID: p.PID, + ParentPID: p.ParentPID, + Description: p.Description, + Start: p.Start, + Type: p.Type, + } + if children { + for _, child := range p.Children() { + process.Children = append(process.Children, child.ToProcess(children)) + } + } + return process +} diff --git a/routers/private/internal.go b/routers/private/internal.go index 263180bd58e24..6ba87d67bf542 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -70,6 +70,7 @@ func Routes() *web.Route { r.Post("/manager/release-and-reopen-logging", ReleaseReopenLogging) r.Post("/manager/add-logger", bind(private.LoggerOptions{}), AddLogger) r.Post("/manager/remove-logger/{group}/{name}", RemoveLogger) + r.Get("/manager/processes", Processes) r.Post("/mail/send", SendEmail) r.Post("/restore_repo", RestoreRepo) diff --git a/routers/private/manager_process.go b/routers/private/manager_process.go new file mode 100644 index 0000000000000..73d088d95e080 --- /dev/null +++ b/routers/private/manager_process.go @@ -0,0 +1,150 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package private + +import ( + "bytes" + "fmt" + "io" + "net/http" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/private" + process_module "code.gitea.io/gitea/modules/process" +) + +// Processes prints out the processes +func Processes(ctx *context.PrivateContext) { + flat := ctx.FormBool("flat") + requestsOnly := ctx.FormBool("requests-only") + stacktraces := ctx.FormBool("stacktraces") + json := ctx.FormBool("json") + + var processes []*process_module.Process + count := int64(0) + var err error + if stacktraces { + processes, count, err = process_module.GetManager().ProcessStacktraces(flat, requestsOnly) + if err != nil { + log.Error("Unable to get stacktrace: %v", err) + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Failed to get stacktraces: %v", err), + }) + return + } + } else { + processes = process_module.GetManager().Processes(!flat, requestsOnly, func() {}) + } + + if json { + ctx.JSON(http.StatusOK, map[string]interface{}{ + "TotalNumberOfGoroutines": count, + "Processes": processes, + }) + return + } + + ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8") + ctx.Resp.WriteHeader(http.StatusOK) + + if err := writeProcesses(ctx.Resp, processes, count, "", flat); err != nil { + log.Error("Unable to write out process stacktrace: %v", err) + if !ctx.Written() { + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Failed to get stacktraces: %v", err), + }) + } + return + } +} + +func writeProcesses(out io.Writer, processes []*process_module.Process, numberOfGoroutines int64, indent string, flat bool) error { + if numberOfGoroutines > 0 { + if _, err := fmt.Fprintf(out, "%sNumber of goroutines: %d\n", indent, numberOfGoroutines); err != nil { + return err + } + } + if _, err := fmt.Fprintf(out, "%sProcess: %d\n", indent, len(processes)); err != nil { + return err + } + if len(processes) > 0 { + if err := writeProcess(out, processes[0], " ", flat); err != nil { + return err + } + } + if len(processes) > 1 { + for _, process := range processes[1:] { + if _, err := fmt.Fprintf(out, "%s | \n", indent); err != nil { + return err + } + if err := writeProcess(out, process, " ", flat); err != nil { + return err + } + } + } + return nil +} + +func writeProcess(out io.Writer, process *process_module.Process, indent string, flat bool) error { + sb := &bytes.Buffer{} + if flat { + if process.ParentPID != "" { + _, _ = fmt.Fprintf(sb, "%s+ PID: %s\t\tType: %s\n", indent, process.PID, process.Type) + } else { + _, _ = fmt.Fprintf(sb, "%s+ PID: %s:%s\tType: %s\n", indent, process.ParentPID, process.PID, process.Type) + } + } else { + _, _ = fmt.Fprintf(sb, "%s+ PID: %s\tType: %s\n", indent, process.PID, process.Type) + } + indent += "| " + + _, _ = fmt.Fprintf(sb, "%sDescription: %s\n", indent, process.Description) + _, _ = fmt.Fprintf(sb, "%sStart: %s\n", indent, process.Start) + + if len(process.Stacks) > 0 { + _, _ = fmt.Fprintf(sb, "%sGoroutines:\n", indent) + for _, stack := range process.Stacks { + indent := indent + " " + _, _ = fmt.Fprintf(sb, "%s+ Description: %s", indent, stack.Description) + if stack.Count > 1 { + _, _ = fmt.Fprintf(sb, "* %d", stack.Count) + } + _, _ = fmt.Fprintf(sb, "\n") + indent += "| " + if len(stack.Labels) > 0 { + _, _ = fmt.Fprintf(sb, "%sLabels: %q:%q", indent, stack.Labels[0].Name, stack.Labels[0].Value) + + if len(stack.Labels) > 1 { + for _, label := range stack.Labels[1:] { + _, _ = fmt.Fprintf(sb, ", %q:%q", label.Name, label.Value) + } + } + _, _ = fmt.Fprintf(sb, "\n") + } + _, _ = fmt.Fprintf(sb, "%sStack:\n", indent) + indent += " " + for _, entry := range stack.Entry { + _, _ = fmt.Fprintf(sb, "%s+ %s\n", indent, entry.Function) + _, _ = fmt.Fprintf(sb, "%s| %s:%d\n", indent, entry.File, entry.Line) + } + } + } + if _, err := out.Write(sb.Bytes()); err != nil { + return err + } + sb.Reset() + if len(process.Children) > 0 { + if _, err := fmt.Fprintf(out, "%sChildren:\n", indent); err != nil { + return err + } + for _, child := range process.Children { + if err := writeProcess(out, child, indent+" ", flat); err != nil { + return err + } + } + } + return nil +} diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index d3b2dcfeccd7a..868bf9bea78c1 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -7,13 +7,10 @@ package admin import ( "fmt" - "io" "net/http" "net/url" "os" "runtime" - "runtime/pprof" - "sort" "strconv" "strings" "time" @@ -35,7 +32,6 @@ import ( "code.gitea.io/gitea/services/mailer" "gitea.com/go-chi/session" - "github.com/google/pprof/profile" ) const ( @@ -344,176 +340,13 @@ func GoroutineStacktrace(ctx *context.Context) { ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminMonitor"] = true - var stacks *profile.Profile - processes := process.GetManager().Processes(false, false, func() { - reader, writer := io.Pipe() - defer reader.Close() - go func() { - err := pprof.Lookup("goroutine").WriteTo(writer, 0) - _ = writer.CloseWithError(err) - }() - var err error - stacks, err = profile.Parse(reader) - if err != nil { - ctx.ServerError("GoroutineStacktrace", err) - return - } - }) - if ctx.Written() { + processStacks, goroutineCount, err := process.GetManager().ProcessStacktraces(false, false) + if err != nil { + ctx.ServerError("GoroutineStacktrace", err) return } - type StackEntry struct { - Function string - File string - Line int - } - - type Label struct { - Name string - Value string - } - - type Stack struct { - Count int64 - Description string - Labels []*Label - Entry []*StackEntry - } - - type ProcessStack struct { - PID process.IDType - ParentPID process.IDType - Description string - Start time.Time - Type string - - Children []*ProcessStack - Stacks []*Stack - } - - // Now earlier we sorted by process time so we know that we should not have children before parents - pidMap := map[process.IDType]*ProcessStack{} - processStacks := make([]*ProcessStack, 0, len(processes)) - for _, process := range processes { - pStack := &ProcessStack{ - PID: process.PID, - ParentPID: process.ParentPID, - Description: process.Description, - Start: process.Start, - Type: process.Type, - } - - pidMap[process.PID] = pStack - if parent, ok := pidMap[process.ParentPID]; ok { - parent.Children = append(parent.Children, pStack) - } else { - processStacks = append(processStacks, pStack) - } - } - - goroutineCount := int64(0) - - // Now walk through the "Sample" slice in the goroutines stack - for _, sample := range stacks.Sample { - stack := &Stack{} - - // Add the labels - for name, value := range sample.Label { - if name == process.DescriptionPProfLabel || name == process.PIDPProfLabel || name == process.PPIDPProfLabel || name == process.ProcessTypePProfLabel { - continue - } - if len(value) != 1 { - // Unexpected... - log.Error("Label: %s in goroutine stack with unexpected number of values: %v", name, value) - continue - } - - stack.Labels = append(stack.Labels, &Label{Name: name, Value: value[0]}) - } - - stack.Count = sample.Value[0] - goroutineCount += stack.Count - - // Now get the processStack for this goroutine sample - var processStack *ProcessStack - if pidvalue, ok := sample.Label[process.PIDPProfLabel]; ok && len(pidvalue) == 1 { - pid := process.IDType(pidvalue[0]) - processStack, ok = pidMap[pid] - if !ok && pid != "" { - ppid := process.IDType("") - if value, ok := sample.Label[process.PPIDPProfLabel]; ok && len(value) == 1 { - ppid = process.IDType(value[0]) - } - description := "(missing process)" - if value, ok := sample.Label[process.DescriptionPProfLabel]; ok && len(value) == 1 { - description = value[0] + " " + description - } - ptype := "code" - if value, ok := sample.Label[process.ProcessTypePProfLabel]; ok && len(value) == 1 { - stack.Labels = append(stack.Labels, &Label{Name: process.ProcessTypePProfLabel, Value: value[0]}) - } - processStack = &ProcessStack{ - PID: pid, - ParentPID: ppid, - Description: description, - Type: ptype, - } - - pidMap[processStack.PID] = processStack - if processStack.ParentPID != "" { - if parent, ok := pidMap[processStack.ParentPID]; ok { - parent.Children = append(parent.Children, processStack) - } - } - } - } - if processStack == nil { - var ok bool - processStack, ok = pidMap[""] - if !ok { - processStack = &ProcessStack{ - Description: "(unbound)", - Type: "code", - } - pidMap[processStack.PID] = processStack - processStacks = append(processStacks, processStack) - } - } - - // Now walk through the locations... - for _, location := range sample.Location { - for _, line := range location.Line { - entry := &StackEntry{ - Function: line.Function.Name, - File: line.Function.Filename, - Line: int(line.Line), - } - stack.Entry = append(stack.Entry, entry) - } - } - stack.Description = "(unknown)" - if len(stack.Entry) > 0 { - stack.Description = stack.Entry[len(stack.Entry)-1].Function - } - - processStack.Stacks = append(processStack.Stacks, stack) - } - - // Now finally re-sort the processstacks so the newest processes are at the top - after := func(processStacks []*ProcessStack) func(i, j int) bool { - return func(i, j int) bool { - left, right := processStacks[i], processStacks[j] - return left.Start.After(right.Start) - } - } - sort.Slice(processStacks, after(processStacks)) - for _, processStack := range processStacks { - sort.Slice(processStack.Children, after(processStack.Children)) - } - ctx.Data["ProcessStacks"] = processStacks - ctx.Data["Profile"] = stacks ctx.Data["GoroutineCount"] = goroutineCount From 08742e7d8cbe5a05b27fa9a5d5560643e5b23746 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 27 Mar 2022 02:26:23 +0100 Subject: [PATCH 17/23] add cancellation and docs Signed-off-by: Andrew Thornton --- cmd/manager.go | 6 +++++- docs/content/doc/usage/command-line.en-us.md | 7 +++++++ modules/private/manager.go | 4 ++-- modules/process/manager.go | 2 +- routers/private/manager_process.go | 9 +++++++++ 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/cmd/manager.go b/cmd/manager.go index 01804945bea69..62d06e80780cc 100644 --- a/cmd/manager.go +++ b/cmd/manager.go @@ -92,6 +92,10 @@ var ( Name: "json", Usage: "Output as json", }, + cli.StringFlag{ + Name: "cancel", + Usage: "Process PID to cancel. (Only available for non-system processes.)", + }, }, } ) @@ -146,7 +150,7 @@ func runProcesses(c *cli.Context) error { defer cancel() setup("manager", c.Bool("debug")) - statusCode, msg := private.Processes(ctx, os.Stdout, c.Bool("no-children"), c.Bool("requests-only"), c.Bool("stacktraces"), c.Bool("json")) + statusCode, msg := private.Processes(ctx, os.Stdout, c.Bool("no-children"), c.Bool("requests-only"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel")) switch statusCode { case http.StatusInternalServerError: return fail("InternalServerError", msg) diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index 80a2c6716de5f..e6713f2c17902 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -503,6 +503,13 @@ Manage running server operations: - `--host value`, `-H value`: Mail server host (defaults to: 127.0.0.1:25) - `--send-to value`, `-s value`: Email address(es) to send to - `--subject value`, `-S value`: Subject header of sent emails + - `processes`: Display Gitea processes and goroutine information + - Options: + - `--no-children`: Show processes as flat table rather than as tree + - `--requests-only`: Only show request processes + - `--stacktraces`: Show stacktraces for goroutines associated with processes + - `--json`: Output as json + - `--cancel PID`: Send cancel to process with PID. (Only for non-system processes.) ### dump-repo diff --git a/modules/private/manager.go b/modules/private/manager.go index e1d43170bf5a5..53df35bce3128 100644 --- a/modules/private/manager.go +++ b/modules/private/manager.go @@ -192,8 +192,8 @@ func RemoveLogger(ctx context.Context, group, name string) (int, string) { } // Processes return the current processes from this gitea instance -func Processes(ctx context.Context, out io.Writer, flat, onlyRequests, stacktraces, json bool) (int, string) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/processes?flat=%t&requests-only=%t&stacktraces=%t&json=%t", flat, onlyRequests, stacktraces, json) +func Processes(ctx context.Context, out io.Writer, flat, onlyRequests, stacktraces, json bool, cancel string) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/processes?flat=%t&requests-only=%t&stacktraces=%t&json=%t&cancel-pid=%s", flat, onlyRequests, stacktraces, json, url.QueryEscape(cancel)) req := newInternalRequest(ctx, reqURL, "GET") resp, err := req.Response() diff --git a/modules/process/manager.go b/modules/process/manager.go index ef0233d20ae0f..5be40bd7ac9c2 100644 --- a/modules/process/manager.go +++ b/modules/process/manager.go @@ -211,7 +211,7 @@ func (pm *Manager) Cancel(pid IDType) { pm.mutex.Lock() process, ok := pm.processes[pid] pm.mutex.Unlock() - if ok { + if ok && process.Type != SystemProcessType { process.Cancel() } } diff --git a/routers/private/manager_process.go b/routers/private/manager_process.go index 73d088d95e080..a0f695e5b3e68 100644 --- a/routers/private/manager_process.go +++ b/routers/private/manager_process.go @@ -9,6 +9,8 @@ import ( "fmt" "io" "net/http" + "runtime" + "time" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" @@ -18,6 +20,13 @@ import ( // Processes prints out the processes func Processes(ctx *context.PrivateContext) { + pid := ctx.FormString("cancel-pid") + if pid != "" { + process_module.GetManager().Cancel(process_module.IDType(pid)) + runtime.Gosched() + time.Sleep(100 * time.Millisecond) + } + flat := ctx.FormBool("flat") requestsOnly := ctx.FormBool("requests-only") stacktraces := ctx.FormBool("stacktraces") From e909ffc4192cb20c1312d091b0d7d0f59a554a7e Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 28 Mar 2022 15:19:31 +0100 Subject: [PATCH 18/23] Apply suggestions from code review Co-authored-by: Gusted --- modules/process/manager_stacktraces.go | 4 ++-- modules/ssh/ssh.go | 2 +- options/locale/locale_en-US.ini | 2 +- routers/private/manager_process.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/process/manager_stacktraces.go b/modules/process/manager_stacktraces.go index 137fb3c9fdfa1..5747abe0c4e33 100644 --- a/modules/process/manager_stacktraces.go +++ b/modules/process/manager_stacktraces.go @@ -74,6 +74,7 @@ func (pm *Manager) Processes(onlyRoots, noSystem bool, runInLock func()) []*Proc } pm.mutex.Unlock() + // Sort by process' start time. Oldest process appears first. sort.Slice(processes, func(i, j int) bool { left, right := processes[i], processes[j] @@ -228,8 +229,7 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 continue } if len(processStack.Children) > 0 { - processStacks = processStacks[:i] - processStacks = append(processStacks, processStack.Children...) + processStacks = append(processStacks[:i], processStack.Children...) i = len(processStacks) - 1 continue } diff --git a/modules/ssh/ssh.go b/modules/ssh/ssh.go index 70c9731c841ec..44ed431c93093 100644 --- a/modules/ssh/ssh.go +++ b/modules/ssh/ssh.go @@ -320,7 +320,7 @@ func Listen(host string, port int, ciphers, keyExchanges, macs []string) { } go func() { - _, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), "Service: SSH", process.SystemProcessType, true) + _, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), "Service: Built-in SSH server", process.SystemProcessType, true) defer finished() listen(&srv) }() diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 6f412c9773967..02b99d8c44145 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2813,7 +2813,7 @@ monitor.previous = Previous Time monitor.execute_times = Executions monitor.process = Running Processes monitor.stacktrace = Stacktraces -monitor.goroutines=%d Goroutines +monitor.goroutines = %d Goroutines monitor.desc = Description monitor.start = Start Time monitor.execute_time = Execution Time diff --git a/routers/private/manager_process.go b/routers/private/manager_process.go index a0f695e5b3e68..fbfee8030d880 100644 --- a/routers/private/manager_process.go +++ b/routers/private/manager_process.go @@ -45,7 +45,7 @@ func Processes(ctx *context.PrivateContext) { return } } else { - processes = process_module.GetManager().Processes(!flat, requestsOnly, func() {}) + processes = process_module.GetManager().Processes(!flat, requestsOnly, nil) } if json { From 212cc338d1a513278471af03fd4e97672fb86171 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 28 Mar 2022 17:15:51 +0100 Subject: [PATCH 19/23] add comments and clean up processes a bit Signed-off-by: Andrew Thornton --- modules/process/manager_stacktraces.go | 188 +++++++++++++++++-------- routers/private/manager_process.go | 2 +- routers/web/admin/admin.go | 2 +- 3 files changed, 128 insertions(+), 64 deletions(-) diff --git a/modules/process/manager_stacktraces.go b/modules/process/manager_stacktraces.go index 5747abe0c4e33..9b31f637932d5 100644 --- a/modules/process/manager_stacktraces.go +++ b/modules/process/manager_stacktraces.go @@ -48,7 +48,7 @@ type Process struct { } // Processes gets the processes in a thread safe manner -func (pm *Manager) Processes(onlyRoots, noSystem bool, runInLock func()) []*Process { +func (pm *Manager) Processes(onlyRoots, noSystem bool) []*Process { pm.mutex.Lock() processes := make([]*Process, 0, len(pm.processes)) if onlyRoots { @@ -69,9 +69,6 @@ func (pm *Manager) Processes(onlyRoots, noSystem bool, runInLock func()) []*Proc processes = append(processes, process.ToProcess(false)) } } - if runInLock != nil { - runInLock() - } pm.mutex.Unlock() // Sort by process' start time. Oldest process appears first. @@ -88,41 +85,66 @@ func (pm *Manager) Processes(onlyRoots, noSystem bool, runInLock func()) []*Proc func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int64, error) { var stacks *profile.Profile var err error - processes := pm.Processes(false, false, func() { - reader, writer := io.Pipe() - defer reader.Close() - go func() { - err := pprof.Lookup("goroutine").WriteTo(writer, 0) - _ = writer.CloseWithError(err) - }() - stacks, err = profile.Parse(reader) - if err != nil { - return + var processes []*Process + + // We cannot use the process pidmaps here because we will release the mutex ... + pidMap := map[IDType]*Process{} + numberOfRoots := 0 // This is simply a guesstimate to create the number of processes we need if we're not doing flat trees + + // Lock the manager + pm.mutex.Lock() + + // Add a defer to unlock in case there is a panic + unlocked := false + defer func() { + if !unlocked { + pm.mutex.Unlock() } - }) + }() + + // Now if we're doing a flat process list we can simply create the processes list here + if flat { + processes = make([]*Process, 0, len(pm.processes)) + } + for _, internalProcess := range pm.processes { + process := internalProcess.ToProcess(false) + + if process.ParentPID == "" { + numberOfRoots++ + } + pidMap[internalProcess.PID] = process + if flat { + processes = append(processes, process) + } + } + + // Now from within the lock we need to get the goroutines. + // Why? If we release the lock then between between filling the above map and getting + // the stacktraces another process could be created which would then look like a dead process below + reader, writer := io.Pipe() + defer reader.Close() + go func() { + err := pprof.Lookup("goroutine").WriteTo(writer, 0) + _ = writer.CloseWithError(err) + }() + stacks, err = profile.Parse(reader) if err != nil { return nil, 0, err } - // We cannot use the process pidmaps here because we have released the mutex ... - pidMap := map[IDType]*Process{} - processStacks := make([]*Process, 0, len(processes)) - for _, process := range processes { - pStack := &Process{ - PID: process.PID, - ParentPID: process.ParentPID, - Description: process.Description, - Start: process.Start, - Type: process.Type, - } + // Unlock the mutex + pm.mutex.Unlock() + unlocked = true - pidMap[process.PID] = pStack - if flat { - processStacks = append(processStacks, pStack) - } else if parent, ok := pidMap[process.ParentPID]; ok { - parent.Children = append(parent.Children, pStack) - } else { - processStacks = append(processStacks, pStack) + // Now if we're not after a flat list of all procesess we need to grab all of the roots and set them up + if !flat { + processes = make([]*Process, 0, numberOfRoots) + for _, process := range pidMap { + if parent, ok := pidMap[process.ParentPID]; ok { + parent.Children = append(parent.Children, process) + } else { + processes = append(processes, process) + } } } @@ -130,13 +152,20 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 // Now walk through the "Sample" slice in the goroutines stack for _, sample := range stacks.Sample { + // In the "goroutine" pprof profile each sample represents one or more goroutines + // with the same labels and stacktraces. + + // We will represent each goroutine by a `Stack` stack := &Stack{} - // Add the labels + // Add the non-process associated labels from the goroutine sample to the Stack for name, value := range sample.Label { if name == DescriptionPProfLabel || name == PIDPProfLabel || (!flat && name == PPIDPProfLabel) || name == ProcessTypePProfLabel { continue } + + // Labels from the "goroutine" pprof profile only have one value. + // This is because the underlying representation is a map[string]string if len(value) != 1 { // Unexpected... return nil, 0, fmt.Errorf("label: %s in goroutine stack with unexpected number of values: %v", name, value) @@ -145,61 +174,83 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 stack.Labels = append(stack.Labels, &Label{Name: name, Value: value[0]}) } + // The number of goroutines that this sample represents is the `stack.Value[0]` stack.Count = sample.Value[0] goroutineCount += stack.Count - // Now get the processStack for this goroutine sample - var processStack *Process + // Now we want to associate this Stack with a Process. + var process *Process + + // Try to get the PID from the goroutine labels if pidvalue, ok := sample.Label[PIDPProfLabel]; ok && len(pidvalue) == 1 { pid := IDType(pidvalue[0]) - processStack, ok = pidMap[pid] + + // Now try to get the process from our map + process, ok = pidMap[pid] if !ok && pid != "" { + // This means that no process has been found in the process map - but there was a process PID + // Therefore this goroutine belongs to a dead process and it has escaped control of the process as it + // should have died with the process context cancellation. + + // We need to create a dead process holder for this process and label it appropriately + + // get the parent PID ppid := IDType("") if value, ok := sample.Label[PPIDPProfLabel]; ok && len(value) == 1 { ppid = IDType(value[0]) } + + // format the description description := "(dead process)" if value, ok := sample.Label[DescriptionPProfLabel]; ok && len(value) == 1 { description = value[0] + " " + description } + + // override the type of the process to "code" but add the old type as a label on the first stack ptype := "code" if value, ok := sample.Label[ProcessTypePProfLabel]; ok && len(value) == 1 { stack.Labels = append(stack.Labels, &Label{Name: ProcessTypePProfLabel, Value: value[0]}) } - processStack = &Process{ + process = &Process{ PID: pid, ParentPID: ppid, Description: description, Type: ptype, } - pidMap[processStack.PID] = processStack + // Now add the dead process back to the map and tree so we don't go back through this again. + pidMap[process.PID] = process added := false - if processStack.ParentPID != "" && !flat { - if parent, ok := pidMap[processStack.ParentPID]; ok { - parent.Children = append(parent.Children, processStack) + if process.ParentPID != "" && !flat { + if parent, ok := pidMap[process.ParentPID]; ok { + parent.Children = append(parent.Children, process) added = true } } if !added { - processStacks = append(processStacks, processStack) + processes = append(processes, process) } } } - if processStack == nil { + + if process == nil { + // This means that the sample we're looking has no PID label var ok bool - processStack, ok = pidMap[""] + process, ok = pidMap[""] if !ok { - processStack = &Process{ + // this is the first time we've come acrross an unassociated goroutine so create a "process" to hold them + process = &Process{ Description: "(unassociated)", Type: "code", } - pidMap[processStack.PID] = processStack - processStacks = append(processStacks, processStack) + pidMap[process.PID] = process + processes = append(processes, process) } } - // Now walk through the locations... + // The sample.Location represents a stack trace for this goroutine, + // however each Location can represent multiple lines (mostly due to inlining) + // so we need to walk the lines too for _, location := range sample.Location { for _, line := range location.Line { entry := &StackEntry{ @@ -210,47 +261,60 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 stack.Entry = append(stack.Entry, entry) } } + + // Now we need a short-descriptive name to call the stack trace if when it is folded and + // assuming the stack trace has some lines we'll choose the bottom of the stack (i.e. the + // initial function that started the stack trace.) The top of the stack is unlikely to + // be very helpful as a lot of the time it will be runtime.select or some other call into + // a std library. stack.Description = "(unknown)" if len(stack.Entry) > 0 { stack.Description = stack.Entry[len(stack.Entry)-1].Function } - processStack.Stacks = append(processStack.Stacks, stack) + process.Stacks = append(process.Stacks, stack) } + // restrict to only show roots which represent requests if onlyRequests { var requestStacks []*Process - i := len(processStacks) - 1 + i := len(processes) - 1 for i >= 0 { - processStack := processStacks[i] + processStack := processes[i] if processStack.Type == RequestProcessType { requestStacks = append(requestStacks, processStack) i-- continue } if len(processStack.Children) > 0 { - processStacks = append(processStacks[:i], processStack.Children...) - i = len(processStacks) - 1 + processes = append(processes[:i], processStack.Children...) + i = len(processes) - 1 continue } i-- } - processStacks = requestStacks + processes = requestStacks } - // Now finally re-sort the processstacks so the newest processes are at the top - after := func(processStacks []*Process) func(i, j int) bool { + // Now finally re-sort the processes. Newest process appears first + after := func(processes []*Process) func(i, j int) bool { return func(i, j int) bool { - left, right := processStacks[i], processStacks[j] + left, right := processes[i], processes[j] return left.Start.After(right.Start) } } - sort.Slice(processStacks, after(processStacks)) + sort.Slice(processes, after(processes)) if !flat { - for _, processStack := range processStacks { - sort.Slice(processStack.Children, after(processStack.Children)) + + var sortChildren func(process *Process) + + sortChildren = func(process *Process) { + sort.Slice(process.Children, after(process.Children)) + for _, child := range process.Children { + sortChildren(child) + } } } - return processStacks, goroutineCount, err + return processes, goroutineCount, err } diff --git a/routers/private/manager_process.go b/routers/private/manager_process.go index fbfee8030d880..a891fae294f20 100644 --- a/routers/private/manager_process.go +++ b/routers/private/manager_process.go @@ -45,7 +45,7 @@ func Processes(ctx *context.PrivateContext) { return } } else { - processes = process_module.GetManager().Processes(!flat, requestsOnly, nil) + processes = process_module.GetManager().Processes(!flat, requestsOnly) } if json { diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 868bf9bea78c1..ef5b55ce014df 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -327,7 +327,7 @@ func Monitor(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.monitor") ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminMonitor"] = true - ctx.Data["Processes"] = process.GetManager().Processes(true, true, nil) + ctx.Data["Processes"] = process.GetManager().Processes(true, true) ctx.Data["Entries"] = cron.ListTasks() ctx.Data["Queues"] = queue.GetManager().ManagedQueues() From 46bcb45816b91c4a8736efb4f28402bce4c47b1e Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 28 Mar 2022 17:35:00 +0100 Subject: [PATCH 20/23] placate lint Signed-off-by: Andrew Thornton --- modules/process/manager_stacktraces.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/process/manager_stacktraces.go b/modules/process/manager_stacktraces.go index 9b31f637932d5..1e67b0e08e2c7 100644 --- a/modules/process/manager_stacktraces.go +++ b/modules/process/manager_stacktraces.go @@ -136,7 +136,7 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 pm.mutex.Unlock() unlocked = true - // Now if we're not after a flat list of all procesess we need to grab all of the roots and set them up + // Now if we're not after a flat list of all processes we need to grab all of the roots and set them up if !flat { processes = make([]*Process, 0, numberOfRoots) for _, process := range pidMap { From b9f8d51913ba4a452c95b6af09056c31969f25f4 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 28 Mar 2022 20:00:58 +0100 Subject: [PATCH 21/23] further adjustments Signed-off-by: Andrew Thornton --- cmd/manager.go | 8 +- modules/private/manager.go | 4 +- modules/process/manager.go | 30 ++--- modules/process/manager_stacktraces.go | 163 +++++++++++++++---------- modules/process/manager_test.go | 8 +- modules/process/process.go | 55 +-------- routers/private/manager_process.go | 22 ++-- routers/web/admin/admin.go | 5 +- templates/admin/stacktrace-row.tmpl | 2 +- 9 files changed, 137 insertions(+), 160 deletions(-) diff --git a/cmd/manager.go b/cmd/manager.go index 62d06e80780cc..03fe23aa9e3f0 100644 --- a/cmd/manager.go +++ b/cmd/manager.go @@ -77,12 +77,12 @@ var ( Name: "debug", }, cli.BoolFlag{ - Name: "no-children", + Name: "flat", Usage: "Show processes as flat table rather than as tree", }, cli.BoolFlag{ - Name: "requests-only", - Usage: "Only show request processes", + Name: "no-system", + Usage: "Do not show system proceses", }, cli.BoolFlag{ Name: "stacktraces", @@ -150,7 +150,7 @@ func runProcesses(c *cli.Context) error { defer cancel() setup("manager", c.Bool("debug")) - statusCode, msg := private.Processes(ctx, os.Stdout, c.Bool("no-children"), c.Bool("requests-only"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel")) + statusCode, msg := private.Processes(ctx, os.Stdout, c.Bool("flat"), c.Bool("no-system"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel")) switch statusCode { case http.StatusInternalServerError: return fail("InternalServerError", msg) diff --git a/modules/private/manager.go b/modules/private/manager.go index 53df35bce3128..8405bf2c83d88 100644 --- a/modules/private/manager.go +++ b/modules/private/manager.go @@ -192,8 +192,8 @@ func RemoveLogger(ctx context.Context, group, name string) (int, string) { } // Processes return the current processes from this gitea instance -func Processes(ctx context.Context, out io.Writer, flat, onlyRequests, stacktraces, json bool, cancel string) (int, string) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/processes?flat=%t&requests-only=%t&stacktraces=%t&json=%t&cancel-pid=%s", flat, onlyRequests, stacktraces, json, url.QueryEscape(cancel)) +func Processes(ctx context.Context, out io.Writer, flat, noSystem, stacktraces, json bool, cancel string) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/processes?flat=%t&no-system=%t&stacktraces=%t&json=%t&cancel-pid=%s", flat, noSystem, stacktraces, json, url.QueryEscape(cancel)) req := newInternalRequest(ctx, reqURL, "GET") resp, err := req.Response() diff --git a/modules/process/manager.go b/modules/process/manager.go index 5be40bd7ac9c2..4884fc2bc0d44 100644 --- a/modules/process/manager.go +++ b/modules/process/manager.go @@ -51,15 +51,15 @@ type Manager struct { next int64 lastTime int64 - processes map[IDType]*process + processMap map[IDType]*process } // GetManager returns a Manager and initializes one as singleton if there's none yet func GetManager() *Manager { managerInit.Do(func() { manager = &Manager{ - processes: make(map[IDType]*process), - next: 1, + processMap: make(map[IDType]*process), + next: 1, } }) return manager @@ -120,7 +120,7 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C pm.mutex.Lock() start, pid := pm.nextPID() - parent := pm.processes[parentPID] + parent := pm.processMap[parentPID] if parent == nil { parentPID = "" } @@ -148,10 +148,7 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C } } - if parent != nil { - parent.AddChild(process) - } - pm.processes[pid] = process + pm.processMap[pid] = process pm.mutex.Unlock() pprofCtx := pprof.WithLabels(ctx, pprof.Labels(DescriptionPProfLabel, description, PPIDPProfLabel, string(parentPID), PIDPProfLabel, string(pid), ProcessTypePProfLabel, processType)) @@ -187,29 +184,22 @@ func (pm *Manager) nextPID() (start time.Time, pid IDType) { // Remove a process from the ProcessManager. func (pm *Manager) Remove(pid IDType) { pm.mutex.Lock() - delete(pm.processes, pid) + delete(pm.processMap, pid) pm.mutex.Unlock() } func (pm *Manager) remove(process *process) { pm.mutex.Lock() - if p := pm.processes[process.PID]; p == process { - delete(pm.processes, process.PID) + defer pm.mutex.Unlock() + if p := pm.processMap[process.PID]; p == process { + delete(pm.processMap, process.PID) } - parent := pm.processes[process.ParentPID] - pm.mutex.Unlock() - - if parent == nil { - return - } - - parent.RemoveChild(process) } // Cancel a process in the ProcessManager. func (pm *Manager) Cancel(pid IDType) { pm.mutex.Lock() - process, ok := pm.processes[pid] + process, ok := pm.processMap[pid] pm.mutex.Unlock() if ok && process.Type != SystemProcessType { process.Cancel() diff --git a/modules/process/manager_stacktraces.go b/modules/process/manager_stacktraces.go index 1e67b0e08e2c7..fbe3374b87bb3 100644 --- a/modules/process/manager_stacktraces.go +++ b/modules/process/manager_stacktraces.go @@ -48,29 +48,61 @@ type Process struct { } // Processes gets the processes in a thread safe manner -func (pm *Manager) Processes(onlyRoots, noSystem bool) []*Process { +func (pm *Manager) Processes(flat, noSystem bool) ([]*Process, int) { pm.mutex.Lock() - processes := make([]*Process, 0, len(pm.processes)) - if onlyRoots { - for _, process := range pm.processes { + processCount := len(pm.processMap) + processes := make([]*Process, 0, len(pm.processMap)) + if flat { + for _, process := range pm.processMap { if noSystem && process.Type == SystemProcessType { continue } - if parent, has := pm.processes[process.ParentPID]; !has || - (noSystem && parent.Type == SystemProcessType) { - processes = append(processes, process.ToProcess(true)) - } + processes = append(processes, process.toProcess()) } } else { - for _, process := range pm.processes { - if noSystem && process.Type == SystemProcessType { + // We need our own processMap + processMap := map[IDType]*Process{} + for _, internalProcess := range pm.processMap { + process, ok := processMap[internalProcess.PID] + if !ok { + process = internalProcess.toProcess() + processMap[process.PID] = process + } + + // Check its parent + if process.ParentPID == "" { + processes = append(processes, process) + continue + } + + internalParentProcess, ok := pm.processMap[internalProcess.ParentPID] + if ok { + parentProcess, ok := processMap[process.ParentPID] + if !ok { + parentProcess = internalParentProcess.toProcess() + processMap[parentProcess.PID] = parentProcess + } + parentProcess.Children = append(parentProcess.Children, process) continue } - processes = append(processes, process.ToProcess(false)) + + processes = append(processes, process) } } pm.mutex.Unlock() + if !flat && noSystem { + for i := 0; i < len(processes); i++ { + process := processes[i] + if process.Type != SystemProcessType { + continue + } + processes[len(processes)-1], processes[i] = processes[i], processes[len(processes)-1] + processes = append(processes[:len(processes)-1], process.Children...) + i-- + } + } + // Sort by process' start time. Oldest process appears first. sort.Slice(processes, func(i, j int) bool { left, right := processes[i], processes[j] @@ -78,21 +110,21 @@ func (pm *Manager) Processes(onlyRoots, noSystem bool) []*Process { return left.Start.Before(right.Start) }) - return processes + return processes, processCount } // ProcessStacktraces gets the processes and stacktraces in a thread safe manner -func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int64, error) { +func (pm *Manager) ProcessStacktraces(flat, noSystem bool) ([]*Process, int, int64, error) { var stacks *profile.Profile var err error - var processes []*Process - // We cannot use the process pidmaps here because we will release the mutex ... - pidMap := map[IDType]*Process{} - numberOfRoots := 0 // This is simply a guesstimate to create the number of processes we need if we're not doing flat trees + // We cannot use the pm.ProcessMap here because we will release the mutex ... + processMap := map[IDType]*Process{} + processCount := 0 // Lock the manager pm.mutex.Lock() + processCount = len(pm.processMap) // Add a defer to unlock in case there is a panic unlocked := false @@ -102,18 +134,41 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 } }() - // Now if we're doing a flat process list we can simply create the processes list here + processes := make([]*Process, 0, len(pm.processMap)) if flat { - processes = make([]*Process, 0, len(pm.processes)) - } - for _, internalProcess := range pm.processes { - process := internalProcess.ToProcess(false) - - if process.ParentPID == "" { - numberOfRoots++ + for _, internalProcess := range pm.processMap { + process := internalProcess.toProcess() + processMap[process.PID] = process + if noSystem && internalProcess.Type == SystemProcessType { + continue + } + processes = append(processes, process) } - pidMap[internalProcess.PID] = process - if flat { + } else { + for _, internalProcess := range pm.processMap { + process, ok := processMap[internalProcess.PID] + if !ok { + process = internalProcess.toProcess() + processMap[process.PID] = process + } + + // Check its parent + if process.ParentPID == "" { + processes = append(processes, process) + continue + } + + internalParentProcess, ok := pm.processMap[internalProcess.ParentPID] + if ok { + parentProcess, ok := processMap[process.ParentPID] + if !ok { + parentProcess = internalParentProcess.toProcess() + processMap[parentProcess.PID] = parentProcess + } + parentProcess.Children = append(parentProcess.Children, process) + continue + } + processes = append(processes, process) } } @@ -129,25 +184,13 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 }() stacks, err = profile.Parse(reader) if err != nil { - return nil, 0, err + return nil, 0, 0, err } // Unlock the mutex pm.mutex.Unlock() unlocked = true - // Now if we're not after a flat list of all processes we need to grab all of the roots and set them up - if !flat { - processes = make([]*Process, 0, numberOfRoots) - for _, process := range pidMap { - if parent, ok := pidMap[process.ParentPID]; ok { - parent.Children = append(parent.Children, process) - } else { - processes = append(processes, process) - } - } - } - goroutineCount := int64(0) // Now walk through the "Sample" slice in the goroutines stack @@ -168,7 +211,7 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 // This is because the underlying representation is a map[string]string if len(value) != 1 { // Unexpected... - return nil, 0, fmt.Errorf("label: %s in goroutine stack with unexpected number of values: %v", name, value) + return nil, 0, 0, fmt.Errorf("label: %s in goroutine stack with unexpected number of values: %v", name, value) } stack.Labels = append(stack.Labels, &Label{Name: name, Value: value[0]}) @@ -186,7 +229,7 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 pid := IDType(pidvalue[0]) // Now try to get the process from our map - process, ok = pidMap[pid] + process, ok = processMap[pid] if !ok && pid != "" { // This means that no process has been found in the process map - but there was a process PID // Therefore this goroutine belongs to a dead process and it has escaped control of the process as it @@ -207,7 +250,7 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 } // override the type of the process to "code" but add the old type as a label on the first stack - ptype := "code" + ptype := NoneProcessType if value, ok := sample.Label[ProcessTypePProfLabel]; ok && len(value) == 1 { stack.Labels = append(stack.Labels, &Label{Name: ProcessTypePProfLabel, Value: value[0]}) } @@ -219,10 +262,10 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 } // Now add the dead process back to the map and tree so we don't go back through this again. - pidMap[process.PID] = process + processMap[process.PID] = process added := false if process.ParentPID != "" && !flat { - if parent, ok := pidMap[process.ParentPID]; ok { + if parent, ok := processMap[process.ParentPID]; ok { parent.Children = append(parent.Children, process) added = true } @@ -236,14 +279,14 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 if process == nil { // This means that the sample we're looking has no PID label var ok bool - process, ok = pidMap[""] + process, ok = processMap[""] if !ok { // this is the first time we've come acrross an unassociated goroutine so create a "process" to hold them process = &Process{ Description: "(unassociated)", - Type: "code", + Type: NoneProcessType, } - pidMap[process.PID] = process + processMap[process.PID] = process processes = append(processes, process) } } @@ -275,25 +318,17 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 process.Stacks = append(process.Stacks, stack) } - // restrict to only show roots which represent requests - if onlyRequests { - var requestStacks []*Process - i := len(processes) - 1 - for i >= 0 { - processStack := processes[i] - if processStack.Type == RequestProcessType { - requestStacks = append(requestStacks, processStack) - i-- - continue - } - if len(processStack.Children) > 0 { - processes = append(processes[:i], processStack.Children...) - i = len(processes) - 1 + // restrict to not show system processes + if noSystem { + for i := 0; i < len(processes); i++ { + process := processes[i] + if process.Type != SystemProcessType && process.Type != NoneProcessType { continue } + processes[len(processes)-1], processes[i] = processes[i], processes[len(processes)-1] + processes = append(processes[:len(processes)-1], process.Children...) i-- } - processes = requestStacks } // Now finally re-sort the processes. Newest process appears first @@ -316,5 +351,5 @@ func (pm *Manager) ProcessStacktraces(flat, onlyRequests bool) ([]*Process, int6 } } - return processes, goroutineCount, err + return processes, processCount, goroutineCount, err } diff --git a/modules/process/manager_test.go b/modules/process/manager_test.go index 36394b0c7bddb..30eabeb37a486 100644 --- a/modules/process/manager_test.go +++ b/modules/process/manager_test.go @@ -22,7 +22,7 @@ func TestGetManager(t *testing.T) { } func TestManager_AddContext(t *testing.T) { - pm := Manager{processes: make(map[IDType]*process), next: 1} + pm := Manager{processMap: make(map[IDType]*process), next: 1} ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -41,7 +41,7 @@ func TestManager_AddContext(t *testing.T) { } func TestManager_Cancel(t *testing.T) { - pm := Manager{processes: make(map[IDType]*process), next: 1} + pm := Manager{processMap: make(map[IDType]*process), next: 1} ctx, _, finished := pm.AddContext(context.Background(), "foo") defer finished() @@ -69,7 +69,7 @@ func TestManager_Cancel(t *testing.T) { } func TestManager_Remove(t *testing.T) { - pm := Manager{processes: make(map[IDType]*process), next: 1} + pm := Manager{processMap: make(map[IDType]*process), next: 1} ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -85,7 +85,7 @@ func TestManager_Remove(t *testing.T) { pm.Remove(GetPID(p2Ctx)) - _, exists := pm.processes[GetPID(p2Ctx)] + _, exists := pm.processMap[GetPID(p2Ctx)] assert.False(t, exists, "PID %d is in the list but shouldn't", GetPID(p2Ctx)) } diff --git a/modules/process/process.go b/modules/process/process.go index fc869cfa0582f..2f7ea18373259 100644 --- a/modules/process/process.go +++ b/modules/process/process.go @@ -6,7 +6,6 @@ package process import ( "context" - "sync" "time" ) @@ -14,6 +13,7 @@ var ( SystemProcessType = "system" RequestProcessType = "request" NormalProcessType = "normal" + NoneProcessType = "none" ) // process represents a working process inheriting from Gitea. @@ -24,56 +24,10 @@ type process struct { Start time.Time Cancel context.CancelFunc Type string - - lock sync.Mutex - children []*process -} - -// Children gets the children of the process -// Note: this function will behave nicely even if p is nil -func (p *process) Children() (children []*process) { - if p == nil { - return - } - - p.lock.Lock() - defer p.lock.Unlock() - children = make([]*process, len(p.children)) - copy(children, p.children) - return children -} - -// AddChild adds a child process -// Note: this function will behave nicely even if p is nil -func (p *process) AddChild(child *process) { - if p == nil { - return - } - - p.lock.Lock() - defer p.lock.Unlock() - p.children = append(p.children, child) -} - -// RemoveChild removes a child process -// Note: this function will behave nicely even if p is nil -func (p *process) RemoveChild(process *process) { - if p == nil { - return - } - - p.lock.Lock() - defer p.lock.Unlock() - for i, child := range p.children { - if child == process { - p.children = append(p.children[:i], p.children[i+1:]...) - return - } - } } // ToProcess converts a process to a externally usable Process -func (p *process) ToProcess(children bool) *Process { +func (p *process) toProcess() *Process { process := &Process{ PID: p.PID, ParentPID: p.ParentPID, @@ -81,10 +35,5 @@ func (p *process) ToProcess(children bool) *Process { Start: p.Start, Type: p.Type, } - if children { - for _, child := range p.Children() { - process.Children = append(process.Children, child.ToProcess(children)) - } - } return process } diff --git a/routers/private/manager_process.go b/routers/private/manager_process.go index a891fae294f20..f8932d61fae73 100644 --- a/routers/private/manager_process.go +++ b/routers/private/manager_process.go @@ -28,15 +28,16 @@ func Processes(ctx *context.PrivateContext) { } flat := ctx.FormBool("flat") - requestsOnly := ctx.FormBool("requests-only") + noSystem := ctx.FormBool("no-system") stacktraces := ctx.FormBool("stacktraces") json := ctx.FormBool("json") var processes []*process_module.Process - count := int64(0) + goroutineCount := int64(0) + processCount := 0 var err error if stacktraces { - processes, count, err = process_module.GetManager().ProcessStacktraces(flat, requestsOnly) + processes, processCount, goroutineCount, err = process_module.GetManager().ProcessStacktraces(flat, noSystem) if err != nil { log.Error("Unable to get stacktrace: %v", err) ctx.JSON(http.StatusInternalServerError, private.Response{ @@ -45,12 +46,13 @@ func Processes(ctx *context.PrivateContext) { return } } else { - processes = process_module.GetManager().Processes(!flat, requestsOnly) + processes, processCount = process_module.GetManager().Processes(flat, noSystem) } if json { ctx.JSON(http.StatusOK, map[string]interface{}{ - "TotalNumberOfGoroutines": count, + "TotalNumberOfGoroutines": goroutineCount, + "TotalNumberOfProcesses": processCount, "Processes": processes, }) return @@ -59,7 +61,7 @@ func Processes(ctx *context.PrivateContext) { ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8") ctx.Resp.WriteHeader(http.StatusOK) - if err := writeProcesses(ctx.Resp, processes, count, "", flat); err != nil { + if err := writeProcesses(ctx.Resp, processes, processCount, goroutineCount, "", flat); err != nil { log.Error("Unable to write out process stacktrace: %v", err) if !ctx.Written() { ctx.JSON(http.StatusInternalServerError, private.Response{ @@ -70,13 +72,13 @@ func Processes(ctx *context.PrivateContext) { } } -func writeProcesses(out io.Writer, processes []*process_module.Process, numberOfGoroutines int64, indent string, flat bool) error { - if numberOfGoroutines > 0 { - if _, err := fmt.Fprintf(out, "%sNumber of goroutines: %d\n", indent, numberOfGoroutines); err != nil { +func writeProcesses(out io.Writer, processes []*process_module.Process, processCount int, goroutineCount int64, indent string, flat bool) error { + if goroutineCount > 0 { + if _, err := fmt.Fprintf(out, "%sTotal Number of Goroutines: %d\n", indent, goroutineCount); err != nil { return err } } - if _, err := fmt.Fprintf(out, "%sProcess: %d\n", indent, len(processes)); err != nil { + if _, err := fmt.Fprintf(out, "%sTotal Number of Processes: %d\n", indent, processCount); err != nil { return err } if len(processes) > 0 { diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index ef5b55ce014df..d4093f2049ac4 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -327,7 +327,7 @@ func Monitor(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.monitor") ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminMonitor"] = true - ctx.Data["Processes"] = process.GetManager().Processes(true, true) + ctx.Data["Processes"], ctx.Data["ProcessCount"] = process.GetManager().Processes(false, true) ctx.Data["Entries"] = cron.ListTasks() ctx.Data["Queues"] = queue.GetManager().ManagedQueues() @@ -340,7 +340,7 @@ func GoroutineStacktrace(ctx *context.Context) { ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminMonitor"] = true - processStacks, goroutineCount, err := process.GetManager().ProcessStacktraces(false, false) + processStacks, processCount, goroutineCount, err := process.GetManager().ProcessStacktraces(false, false) if err != nil { ctx.ServerError("GoroutineStacktrace", err) return @@ -349,6 +349,7 @@ func GoroutineStacktrace(ctx *context.Context) { ctx.Data["ProcessStacks"] = processStacks ctx.Data["GoroutineCount"] = goroutineCount + ctx.Data["ProcessCount"] = processCount ctx.HTML(http.StatusOK, tplStacktrace) } diff --git a/templates/admin/stacktrace-row.tmpl b/templates/admin/stacktrace-row.tmpl index a5bbbb869e904..a21ef72d6327f 100644 --- a/templates/admin/stacktrace-row.tmpl +++ b/templates/admin/stacktrace-row.tmpl @@ -13,7 +13,7 @@
{{.Process.Description}}
-
{{if ne .Process.Type "code"}}{{TimeSince .Process.Start .root.i18n.Lang}}{{end}}
+
{{if ne .Process.Type "none"}}{{TimeSince .Process.Start .root.i18n.Lang}}{{end}}
{{if or (eq .Process.Type "request") (eq .Process.Type "normal") }} From e0567b01313fc1f2a5a309a6ce62021c51107468 Mon Sep 17 00:00:00 2001 From: zeripath Date: Tue, 29 Mar 2022 22:21:09 +0100 Subject: [PATCH 22/23] Apply suggestions from code review --- docs/content/doc/usage/command-line.en-us.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index e6713f2c17902..3b75a5c843d20 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -505,8 +505,8 @@ Manage running server operations: - `--subject value`, `-S value`: Subject header of sent emails - `processes`: Display Gitea processes and goroutine information - Options: - - `--no-children`: Show processes as flat table rather than as tree - - `--requests-only`: Only show request processes + - `--flat`: Show processes as flat table rather than as tree + - `--no-system`: Do not show system processes - `--stacktraces`: Show stacktraces for goroutines associated with processes - `--json`: Output as json - `--cancel PID`: Send cancel to process with PID. (Only for non-system processes.) From 4daf8ca31dd0fd646e8f4ad4a0a6e986b2678773 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 31 Mar 2022 21:22:10 +0800 Subject: [PATCH 23/23] fix merge --- modules/process/manager.go | 5 +++++ modules/process/manager_exec.go | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/process/manager.go b/modules/process/manager.go index 4884fc2bc0d44..5d7aee760f58d 100644 --- a/modules/process/manager.go +++ b/modules/process/manager.go @@ -106,6 +106,11 @@ func (pm *Manager) AddTypedContext(parent context.Context, description, processT // Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the // process table. func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Duration, description string) (ctx context.Context, cancel context.CancelFunc, finshed FinishedFunc) { + if timeout <= 0 { + // it's meaningless to use timeout <= 0, and it must be a bug! so we must panic here to tell developers to make the timeout correct + panic("the timeout must be greater than zero, otherwise the context will be cancelled immediately") + } + ctx, cancel = context.WithTimeout(parent, timeout) ctx, _, finshed = pm.Add(ctx, description, cancel, NormalProcessType, true) diff --git a/modules/process/manager_exec.go b/modules/process/manager_exec.go index f1406b8aeb6ea..61ddae646f0e4 100644 --- a/modules/process/manager_exec.go +++ b/modules/process/manager_exec.go @@ -36,11 +36,11 @@ func (pm *Manager) ExecDirEnv(ctx context.Context, timeout time.Duration, dir, d } // ExecDirEnvStdIn runs a command in given path and environment variables with provided stdIN, and waits for its completion -// up to the given timeout (or DefaultTimeout if -1 is given). +// up to the given timeout (or DefaultTimeout if timeout <= 0 is given). // Returns its complete stdout and stderr // outputs and an error, if any (including timeout) func (pm *Manager) ExecDirEnvStdIn(ctx context.Context, timeout time.Duration, dir, desc string, env []string, stdIn io.Reader, cmdName string, args ...string) (string, string, error) { - if timeout == -1 { + if timeout <= 0 { timeout = 60 * time.Second }