agentlog: add pprof debug http handler #3237
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,20 +5,24 @@ package agentlog | |
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
| "net/http/pprof" | ||
| "os" | ||
| "os/signal" | ||
| "runtime" | ||
| dbg "runtime/debug" | ||
| "sort" | ||
| "strings" | ||
| "sync" | ||
| "sync/atomic" | ||
| "syscall" | ||
| "time" | ||
|
|
||
| "github.com/lf-edge/eve/pkg/pillar/base" | ||
| "github.com/lf-edge/eve/pkg/pillar/types" | ||
| fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file" | ||
|
There was a problem hiding this comment. I don't understand the need for this change; the package is referenced by uuid in any case. There was a problem hiding this comment. It was just done by gopls/goimports. |
||
| uuid "github.com/satori/go.uuid" | ||
| "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
|
|
@@ -149,58 +153,151 @@ func (hook *SkipCallerHook) Levels() []logrus.Level { | |
|
|
||
| // Wait on channel then handle the signals | ||
| func handleSignals(log *base.LogObject, agentName string, agentPid int, sigs chan os.Signal) { | ||
| for { | ||
| select { | ||
| case sig := <-sigs: | ||
| log.Functionf("handleSignals: received %v\n", sig) | ||
| switch sig { | ||
| case syscall.SIGUSR1: | ||
| dumpStacks(log, agentName) | ||
| case syscall.SIGUSR2: | ||
| go listenDebug(log, agentName) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func dumpMemoryInfo(log *base.LogObject, fileName string) { | ||
| log.Warnf("SIGUSR2 triggered memory info:\n") | ||
| sigUsr2File, err := os.OpenFile(fileName, | ||
| os.O_WRONLY|os.O_CREATE|os.O_SYNC|os.O_TRUNC, 0755) | ||
| if err != nil { | ||
| log.Errorf("handleSignals: Error opening file %s with: %s", fileName, err) | ||
| } else { | ||
| // This goes to /persist/agentdebug/<agentname>/sigusr2 file | ||
| _, err := sigUsr2File.WriteString("SIGUSR2 triggered memory info:\n") | ||
| if err != nil { | ||
| log.Errorf("could not write to %s: %+v", fileName, err) | ||
| } | ||
| } | ||
|
|
||
| logMemUsage(log, sigUsr2File) | ||
| logMemAllocationSites(log, sigUsr2File) | ||
| if sigUsr2File != nil { | ||
| sigUsr2File.Close() | ||
| } | ||
| } | ||
|
|
||
| func dumpStacks(log *base.LogObject, fileName string) { | ||
| stacks := getStacks(true) | ||
| stackArray := strings.Split(stacks, "\n\n") | ||
|
|
||
| sigUsr1File, err := os.OpenFile(fileName, | ||
| os.O_WRONLY|os.O_CREATE|os.O_SYNC|os.O_TRUNC, 0755) | ||
| if err == nil { | ||
| for _, stack := range stackArray { | ||
| // This goes to /persist/agentdebug/<agentname>/sigusr1 file | ||
| _, err := sigUsr1File.WriteString(stack + "\n\n") | ||
| if err != nil { | ||
| log.Errorf("could not write to %s: %+v", fileName, err) | ||
| } | ||
| } | ||
| sigUsr1File.Close() | ||
| } else { | ||
| log.Errorf("handleSignals: Error opening file %s with: %s", fileName, err) | ||
| } | ||
|
|
||
| usr1LogObject := base.EnsureLogObject(log, base.SigUSR1StacksType, | ||
| "", uuid.UUID{}, string(base.SigUSR1StacksType)) | ||
| if usr1LogObject != nil { | ||
| log.Warnf("SIGUSR1 triggered with %d stacks", len(stackArray)) | ||
| for _, stack := range stackArray { | ||
| usr1LogObject.Warnf("%v", stack) | ||
| } | ||
| log.Warnf("SIGUSR1: end of stacks") | ||
| } | ||
| } | ||
|
|
||
| func writeOrLog(log *base.LogObject, w io.Writer, msg string) { | ||
| if _, err := w.Write([]byte(msg)); err != nil { | ||
| log.Errorf("Could not write to %+v: %+v", w, err) | ||
| } | ||
| } | ||
|
|
||
| var listenDebugRunning atomic.Bool | ||
|
|
||
| func listenDebug(log *base.LogObject, agentName string) { | ||
| if listenDebugRunning.Swap(true) { | ||
| return | ||
| } | ||
|
|
||
| agentDebugDir := fmt.Sprintf("%s/%s/", types.PersistDebugDir, agentName) | ||
| stacksDumpFileName := agentDebugDir + "/sigusr1" | ||
| memDumpFileName := agentDebugDir + "/sigusr2" | ||
|
|
||
| mux := http.NewServeMux() | ||
|
|
||
| server := &http.Server{ | ||
| Addr: "localhost:6543", | ||
| Handler: mux, | ||
| ReadHeaderTimeout: 10 * time.Second, | ||
| } | ||
|
|
||
| info := ` | ||
| This server exposes the net/http/pprof API.</br> | ||
| For examples on how to use it, see: <a href="https://pkg.go.dev/net/http/pprof">https://pkg.go.dev/net/http/pprof</a></br> | ||
| <a href="debug/pprof/">pprof methods</a></br></br> | ||
| To create a flamegraph, do: go tool pprof -raw -output=cpu.txt 'http://localhost:6543/debug/pprof/profile?seconds=5';</br> | ||
| stackcollapse-go.pl cpu.txt | flamegraph.pl --width 4096 > flame.svg</br> | ||
| (both scripts can be found <a href="https://github.com/brendangregg/FlameGraph">here</a>) | ||
| ` | ||
|
|
||
| mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
|
There was a problem hiding this comment. zedcloud side has plan to move away from mux router due to: https://github.com/gorilla#gorilla-toolkit, not sure about the EVE status on this. There was a problem hiding this comment. I am not sure about the status in EVE; I guess we need to discuss this soon. There was a problem hiding this comment. @naiming-zededa @christoph-zededa this is not gorillamux; just the simple http.NewServeMux() which we also use in zedrouter/server.go |
||
| w.Header().Set("Content-Type", "text/html") | ||
| writeOrLog(log, w, info) | ||
| })) | ||
| mux.Handle("/index.html", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| w.Header().Set("Content-Type", "text/html") | ||
| writeOrLog(log, w, info) | ||
| })) | ||
|
|
||
| mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) | ||
| mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) | ||
| mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) | ||
| mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) | ||
| mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) | ||
| mux.Handle("/stop", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| if r.Method == http.MethodPost { | ||
| server.Close() | ||
| listenDebugRunning.Swap(false) | ||
| } else { | ||
| http.Error(w, "Did you want to use POST method?", http.StatusMethodNotAllowed) | ||
| return | ||
| } | ||
| })) | ||
| mux.Handle("/dump/stacks", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| if r.Method == http.MethodPost { | ||
| dumpStacks(log, stacksDumpFileName) | ||
| response := fmt.Sprintf("Stacks can be found in logread or %s\n", stacksDumpFileName) | ||
| writeOrLog(log, w, response) | ||
| } else { | ||
| http.Error(w, "Did you want to use POST method?", http.StatusMethodNotAllowed) | ||
| return | ||
| } | ||
| })) | ||
| mux.Handle("/dump/memory", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| if r.Method == http.MethodPost { | ||
| dumpMemoryInfo(log, memDumpFileName) | ||
| response := fmt.Sprintf("Stacks can be found in logread or %s\n", memDumpFileName) | ||
| writeOrLog(log, w, response) | ||
| } else { | ||
| http.Error(w, "Did you want to use POST method?", http.StatusMethodNotAllowed) | ||
| return | ||
| } | ||
| })) | ||
|
|
||
| if err := server.ListenAndServe(); err != nil { | ||
| log.Errorf("Listening failed: %+v", err) | ||
| } | ||
| } | ||
|
|
||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
do you expect the client side to query the pprof stats quite soon? otherwise the tcp will timeout if no activity. Or in rare case the cmd run has error, we need to exit here. Would be more flexible to offer more options. For example, with 'pprof/on' to trigger it to start and send back status of success or not and print out the port being used, 'pprof/off' to shut it down. either use general tcp/localhost:port, or do a 'pprof/run' to do all 3.
also, add the 'help' line in the basics.go for the new command.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
"do a 'pprof/run' to do all 3" <- that was my initial plan, we can just drop
5ea17eb
and it works that way.