From 728552de740260724f61ee8c187dcf2a4da19446 Mon Sep 17 00:00:00 2001 From: Zachary Wasserman Date: Thu, 19 Oct 2017 10:41:03 -0400 Subject: [PATCH 1/6] Add authenticated debug server --- Gopkg.lock | 8 +++++++- Gopkg.toml | 4 ++++ cmd/launcher/launcher.go | 6 ++++++ debug/debug.go | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 debug/debug.go diff --git a/Gopkg.lock b/Gopkg.lock index b0b6ec9ee..716e76bb3 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -53,6 +53,12 @@ packages = [".","client","client/changelist","cryptoservice","storage","trustmanager","trustmanager/yubikey","trustpinning","tuf","tuf/data","tuf/signed","tuf/utils","tuf/validation"] revision = "fab4c6708bc200c4cfe56c74f902d6e8bb26c803" +[[projects]] + branch = "master" + name = "github.com/e-dard/netbug" + packages = ["."] + revision = "e64d308a0b205c901264e88a10e70d64acb1810d" + [[projects]] name = "github.com/go-kit/kit" packages = ["endpoint","log","log/level","transport/grpc"] @@ -194,6 +200,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "1c1aaef7211b2a2a41be582cbe36547df6ef364b5f2e13aeaeb54bd1ff9c8ed8" + inputs-digest = "40165f65135cf5c4a677395ad6c4675aa3d322a8b618dca1e5e49ae89b29d253" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 66fddc2c7..b268f2410 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -45,3 +45,7 @@ [[constraint]] name = "google.golang.org/grpc" + +[[constraint]] + branch = "master" + name = "github.com/e-dard/netbug" diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index 8bd624755..e654f4f87 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -20,6 +20,7 @@ import ( "github.com/kolide/kit/fs" "github.com/kolide/kit/version" "github.com/kolide/launcher/autoupdate" + "github.com/kolide/launcher/debug" "github.com/kolide/launcher/osquery" "github.com/kolide/launcher/service" "github.com/kolide/osquery-go/plugin/config" @@ -92,6 +93,11 @@ func main() { logFatal(logger, "err", errors.Wrap(err, "detecting platform")) } + debugTokenPath := filepath.Join(rootDirectory, "debug_token") + if err := debug.StartDebugServer("localhost:5097", debugTokenPath, logger); err != nil { + logFatal(logger, "err", errors.Wrap(err, "starting debug server")) + } + httpClient := http.DefaultClient if opts.insecureTLS { httpClient = &http.Client{ diff --git a/debug/debug.go b/debug/debug.go new file mode 100644 index 000000000..ff241b27d --- /dev/null +++ b/debug/debug.go @@ -0,0 +1,35 @@ +package debug + +import ( + "io/ioutil" + "net/http" + + "github.com/e-dard/netbug" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/google/uuid" + "github.com/pkg/errors" +) + +// AttachDebugEndpoints will attach the +func StartDebugServer(addr, tokenPath string, logger log.Logger) error { + // Generate new (random) UUID + token, err := uuid.NewRandom() + if err != nil { + return errors.Wrap(err, "generating debug token") + } + + if err := ioutil.WriteFile(tokenPath, []byte(token.String()), 0600); err != nil { + return errors.Wrap(err, "writing debug token") + } + + r := http.NewServeMux() + netbug.RegisterAuthHandler(token.String(), "/debug/", r) + go func() { + if err := http.ListenAndServe(addr, r); err != nil { + level.Info(logger).Log("msg", "starting debug server failed", "err", err) + } + }() + + return nil +} From 8862bd06614fd2314a27c46d64f5220e2b5fc59f Mon Sep 17 00:00:00 2001 From: Zachary Wasserman Date: Fri, 20 Oct 2017 14:58:02 -0400 Subject: [PATCH 2/6] Log message when starting debug server --- debug/debug.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debug/debug.go b/debug/debug.go index ff241b27d..990fe2937 100644 --- a/debug/debug.go +++ b/debug/debug.go @@ -1,6 +1,7 @@ package debug import ( + "fmt" "io/ioutil" "net/http" @@ -31,5 +32,10 @@ func StartDebugServer(addr, tokenPath string, logger log.Logger) error { } }() + level.Info(logger).Log( + "msg", + fmt.Sprintf("debug server available at http://%s/debug/?token=%s", addr, token.String()), + ) + return nil } From 1b6df142a33f6ab938f56354457dcaedfdbcba71 Mon Sep 17 00:00:00 2001 From: Zachary Wasserman Date: Mon, 23 Oct 2017 14:55:10 -0400 Subject: [PATCH 3/6] @marpaia and @groob comments --- cmd/launcher/launcher.go | 4 +- debug/debug.go | 76 ++++++++++++++++++++++++----- debug/debug_test.go | 100 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 16 deletions(-) create mode 100644 debug/debug_test.go diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index e654f4f87..a96ddf7f7 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -94,9 +94,7 @@ func main() { } debugTokenPath := filepath.Join(rootDirectory, "debug_token") - if err := debug.StartDebugServer("localhost:5097", debugTokenPath, logger); err != nil { - logFatal(logger, "err", errors.Wrap(err, "starting debug server")) - } + debug.AttachDebugHandler(debugTokenPath, logger) httpClient := http.DefaultClient if opts.insecureTLS { diff --git a/debug/debug.go b/debug/debug.go index 990fe2937..832c6d20e 100644 --- a/debug/debug.go +++ b/debug/debug.go @@ -1,9 +1,14 @@ package debug import ( + "context" "fmt" "io/ioutil" + "net" "net/http" + "os" + "os/signal" + "syscall" "github.com/e-dard/netbug" "github.com/go-kit/kit/log" @@ -12,30 +17,75 @@ import ( "github.com/pkg/errors" ) -// AttachDebugEndpoints will attach the -func StartDebugServer(addr, tokenPath string, logger log.Logger) error { - // Generate new (random) UUID +const debugSignal = syscall.SIGUSR1 + +// AttachDebugHandler will attach a signal handler that will toggle the debug +// server state when SIGUSR1 is sent to the process. +func AttachDebugHandler(tokenPath string, logger log.Logger) { + sig := make(chan os.Signal, 1) + signal.Notify(sig, debugSignal) + go func() { + for { + // Start server on first signal + <-sig + serv, err := startDebugServer(tokenPath, logger) + if err != nil { + level.Info(logger).Log( + "msg", "starting debug server", + "err", err, + ) + continue + } + + // Stop server on next signal + <-sig + if err := serv.Shutdown(context.Background()); err != nil { + level.Info(logger).Log( + "msg", "error shutting down debug server", + "err", err, + ) + } else { + level.Info(logger).Log( + "msg", "shutdown debug server", + ) + } + } + }() +} + +func startDebugServer(tokenPath string, logger log.Logger) (*http.Server, error) { + // Generate new (random) token to use for debug server auth token, err := uuid.NewRandom() if err != nil { - return errors.Wrap(err, "generating debug token") - } - - if err := ioutil.WriteFile(tokenPath, []byte(token.String()), 0600); err != nil { - return errors.Wrap(err, "writing debug token") + return nil, errors.Wrap(err, "generating debug token") } + // Start the debug server r := http.NewServeMux() netbug.RegisterAuthHandler(token.String(), "/debug/", r) + serv := http.Server{ + Handler: r, + } + listener, err := net.Listen("tcp", "localhost:0") + if err != nil { + return nil, errors.Wrap(err, "opening socket") + } go func() { - if err := http.ListenAndServe(addr, r); err != nil { - level.Info(logger).Log("msg", "starting debug server failed", "err", err) + if err := serv.Serve(listener); err != nil && err != http.ErrServerClosed { + level.Info(logger).Log("msg", "debug server failed", "err", err) } }() + addr := fmt.Sprintf("http://%s/debug/?token=%s", listener.Addr().String(), token.String()) + // Write the token to a file for easy access by users + if err := ioutil.WriteFile(tokenPath, []byte(addr), 0600); err != nil { + return nil, errors.Wrap(err, "writing debug token") + } + level.Info(logger).Log( - "msg", - fmt.Sprintf("debug server available at http://%s/debug/?token=%s", addr, token.String()), + "msg", "debug server started", + "addr", addr, ) - return nil + return &serv, nil } diff --git a/debug/debug_test.go b/debug/debug_test.go new file mode 100644 index 000000000..5708ff0f0 --- /dev/null +++ b/debug/debug_test.go @@ -0,0 +1,100 @@ +package debug + +import ( + "context" + "io/ioutil" + "net/http" + "syscall" + "testing" + "time" + + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func getDebugURL(t *testing.T, tokenPath string) string { + url, err := ioutil.ReadFile(tokenPath) + require.Nil(t, err) + return string(url) +} + +func TestStartDebugServer(t *testing.T) { + t.Parallel() + tokenFile, err := ioutil.TempFile("", "kolide_debug_test") + require.Nil(t, err) + + serv, err := startDebugServer(tokenFile.Name(), log.NewNopLogger()) + require.Nil(t, err) + + url := getDebugURL(t, tokenFile.Name()) + resp, err := http.Get(url) + require.Nil(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + resp.Body.Close() + + err = serv.Shutdown(context.Background()) + require.Nil(t, err) +} + +func TestDebugServerUnauthorized(t *testing.T) { + t.Parallel() + tokenFile, err := ioutil.TempFile("", "kolide_debug_test") + require.Nil(t, err) + + serv, err := startDebugServer(tokenFile.Name(), log.NewNopLogger()) + require.Nil(t, err) + + url := getDebugURL(t, tokenFile.Name()) + resp, err := http.Get(url + "bad_token") + require.Nil(t, err) + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) + resp.Body.Close() + + err = serv.Shutdown(context.Background()) + require.Nil(t, err) +} + +func TestAttachDebugHandler(t *testing.T) { + t.Parallel() + tokenFile, err := ioutil.TempFile("", "kolide_debug_test") + require.Nil(t, err) + + AttachDebugHandler(tokenFile.Name(), log.NewNopLogger()) + + // Start server + syscall.Kill(syscall.Getpid(), debugSignal) + time.Sleep(1 * time.Second) + + url := getDebugURL(t, tokenFile.Name()) + resp, err := http.Get(url) + require.Nil(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + resp.Body.Close() + + // Stop server + syscall.Kill(syscall.Getpid(), debugSignal) + time.Sleep(1 * time.Second) + + resp, err = http.Get(url) + require.NotNil(t, err) + + // Start server + syscall.Kill(syscall.Getpid(), debugSignal) + time.Sleep(1 * time.Second) + + newUrl := getDebugURL(t, tokenFile.Name()) + assert.NotEqual(t, url, newUrl) + + resp, err = http.Get(newUrl) + require.Nil(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + resp.Body.Close() + + // Stop server + syscall.Kill(syscall.Getpid(), debugSignal) + time.Sleep(1 * time.Second) + + resp, err = http.Get(url) + require.NotNil(t, err) +} From f3aa6995ab25a7c18b2135753f95f7ad4028fd6a Mon Sep 17 00:00:00 2001 From: Zachary Wasserman Date: Mon, 23 Oct 2017 16:19:43 -0400 Subject: [PATCH 4/6] cleanup --- cmd/launcher/launcher.go | 5 +++-- debug/debug.go | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index a96ddf7f7..4b836d941 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -93,8 +93,9 @@ func main() { logFatal(logger, "err", errors.Wrap(err, "detecting platform")) } - debugTokenPath := filepath.Join(rootDirectory, "debug_token") - debug.AttachDebugHandler(debugTokenPath, logger) + debugAddrPath := filepath.Join(rootDirectory, "debug_addr") + debug.AttachDebugHandler(debugAddrPath, logger) + defer os.Remove(debugAddrPath) httpClient := http.DefaultClient if opts.insecureTLS { diff --git a/debug/debug.go b/debug/debug.go index 832c6d20e..ca3148112 100644 --- a/debug/debug.go +++ b/debug/debug.go @@ -21,14 +21,14 @@ const debugSignal = syscall.SIGUSR1 // AttachDebugHandler will attach a signal handler that will toggle the debug // server state when SIGUSR1 is sent to the process. -func AttachDebugHandler(tokenPath string, logger log.Logger) { +func AttachDebugHandler(addrPath string, logger log.Logger) { sig := make(chan os.Signal, 1) signal.Notify(sig, debugSignal) go func() { for { // Start server on first signal <-sig - serv, err := startDebugServer(tokenPath, logger) + serv, err := startDebugServer(addrPath, logger) if err != nil { level.Info(logger).Log( "msg", "starting debug server", @@ -53,7 +53,7 @@ func AttachDebugHandler(tokenPath string, logger log.Logger) { }() } -func startDebugServer(tokenPath string, logger log.Logger) (*http.Server, error) { +func startDebugServer(addrPath string, logger log.Logger) (*http.Server, error) { // Generate new (random) token to use for debug server auth token, err := uuid.NewRandom() if err != nil { @@ -70,6 +70,7 @@ func startDebugServer(tokenPath string, logger log.Logger) (*http.Server, error) if err != nil { return nil, errors.Wrap(err, "opening socket") } + go func() { if err := serv.Serve(listener); err != nil && err != http.ErrServerClosed { level.Info(logger).Log("msg", "debug server failed", "err", err) @@ -77,9 +78,9 @@ func startDebugServer(tokenPath string, logger log.Logger) (*http.Server, error) }() addr := fmt.Sprintf("http://%s/debug/?token=%s", listener.Addr().String(), token.String()) - // Write the token to a file for easy access by users - if err := ioutil.WriteFile(tokenPath, []byte(addr), 0600); err != nil { - return nil, errors.Wrap(err, "writing debug token") + // Write the address to a file for easy access by users + if err := ioutil.WriteFile(addrPath, []byte(addr), 0600); err != nil { + return nil, errors.Wrap(err, "writing debug address") } level.Info(logger).Log( From 53cc8b272851c548727b6ebd8a01c537170260cf Mon Sep 17 00:00:00 2001 From: Zachary Wasserman Date: Mon, 23 Oct 2017 18:29:03 -0400 Subject: [PATCH 5/6] Include handler and template --- Gopkg.lock | 8 +---- Gopkg.toml | 4 --- debug/debug.go | 98 ++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 93 insertions(+), 17 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 716e76bb3..b0b6ec9ee 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -53,12 +53,6 @@ packages = [".","client","client/changelist","cryptoservice","storage","trustmanager","trustmanager/yubikey","trustpinning","tuf","tuf/data","tuf/signed","tuf/utils","tuf/validation"] revision = "fab4c6708bc200c4cfe56c74f902d6e8bb26c803" -[[projects]] - branch = "master" - name = "github.com/e-dard/netbug" - packages = ["."] - revision = "e64d308a0b205c901264e88a10e70d64acb1810d" - [[projects]] name = "github.com/go-kit/kit" packages = ["endpoint","log","log/level","transport/grpc"] @@ -200,6 +194,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "40165f65135cf5c4a677395ad6c4675aa3d322a8b618dca1e5e49ae89b29d253" + inputs-digest = "1c1aaef7211b2a2a41be582cbe36547df6ef364b5f2e13aeaeb54bd1ff9c8ed8" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index b268f2410..66fddc2c7 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -45,7 +45,3 @@ [[constraint]] name = "google.golang.org/grpc" - -[[constraint]] - branch = "master" - name = "github.com/e-dard/netbug" diff --git a/debug/debug.go b/debug/debug.go index ca3148112..ec0663da2 100644 --- a/debug/debug.go +++ b/debug/debug.go @@ -3,14 +3,18 @@ package debug import ( "context" "fmt" + "html/template" "io/ioutil" "net" "net/http" + nhpprof "net/http/pprof" + "net/url" "os" "os/signal" + "runtime/pprof" + "strings" "syscall" - "github.com/e-dard/netbug" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/google/uuid" @@ -18,6 +22,7 @@ import ( ) const debugSignal = syscall.SIGUSR1 +const debugPrefix = "/debug/" // AttachDebugHandler will attach a signal handler that will toggle the debug // server state when SIGUSR1 is sent to the process. @@ -44,11 +49,12 @@ func AttachDebugHandler(addrPath string, logger log.Logger) { "msg", "error shutting down debug server", "err", err, ) - } else { - level.Info(logger).Log( - "msg", "shutdown debug server", - ) + continue } + + level.Info(logger).Log( + "msg", "shutdown debug server", + ) } }() } @@ -62,7 +68,7 @@ func startDebugServer(addrPath string, logger log.Logger) (*http.Server, error) // Start the debug server r := http.NewServeMux() - netbug.RegisterAuthHandler(token.String(), "/debug/", r) + registerAuthHandler(token.String(), r, logger) serv := http.Server{ Handler: r, } @@ -90,3 +96,83 @@ func startDebugServer(addrPath string, logger log.Logger) (*http.Server, error) return &serv, nil } + +// The below handler code is adapted from MIT licensed github.com/e-dard/netbug +func handler(token string, logger log.Logger) http.Handler { + info := struct { + Profiles []*pprof.Profile + Token string + }{ + Profiles: pprof.Profiles(), + Token: url.QueryEscape(token), + } + + h := func(w http.ResponseWriter, r *http.Request) { + name := strings.TrimPrefix(r.URL.Path, "/") + switch name { + case "": + // Index page. + if err := indexTmpl.Execute(w, info); err != nil { + level.Info(logger).Log( + "msg", "error rendering debug template", + "err", err, + ) + return + } + case "cmdline": + nhpprof.Cmdline(w, r) + case "profile": + nhpprof.Profile(w, r) + case "trace": + nhpprof.Trace(w, r) + case "symbol": + nhpprof.Symbol(w, r) + default: + // Provides access to all profiles under runtime/pprof + nhpprof.Handler(name).ServeHTTP(w, r) + } + } + return http.HandlerFunc(h) +} + +func authHandler(token string, logger log.Logger) http.Handler { + h := handler(token, logger) + ah := func(w http.ResponseWriter, r *http.Request) { + if r.FormValue("token") == token { + h.ServeHTTP(w, r) + } else { + w.WriteHeader(http.StatusUnauthorized) + fmt.Fprintln(w, "Unauthorized.") + } + } + return http.HandlerFunc(ah) +} + +func registerAuthHandler(token string, mux *http.ServeMux, logger log.Logger) { + mux.Handle(debugPrefix, http.StripPrefix(debugPrefix, authHandler(token, logger))) +} + +var indexTmpl = template.Must(template.New("index").Parse(` + + Debug Information + +
+ + profiles:
+ + {{range .Profiles}} +
{{.Count}}{{.Name}} + {{end}} +
CPU +
5-second trace +
30-second trace +
+
+ debug information:
+ +
cmdline +
symbol +
full goroutine stack dump
+ + +`)) From 991c6cb696eff311b287c7dd4862eef4578ddaf6 Mon Sep 17 00:00:00 2001 From: Zachary Wasserman Date: Mon, 23 Oct 2017 18:56:52 -0400 Subject: [PATCH 6/6] @groob comments --- debug/debug.go | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/debug/debug.go b/debug/debug.go index ec0663da2..1875d0f2a 100644 --- a/debug/debug.go +++ b/debug/debug.go @@ -2,7 +2,6 @@ package debug import ( "context" - "fmt" "html/template" "io/ioutil" "net" @@ -72,6 +71,9 @@ func startDebugServer(addrPath string, logger log.Logger) (*http.Server, error) serv := http.Server{ Handler: r, } + // Allow the OS to pick an open port. Not intended to be a security + // mechanism, only intended to ensure we don't try to bind to an + // already used port. listener, err := net.Listen("tcp", "localhost:0") if err != nil { return nil, errors.Wrap(err, "opening socket") @@ -83,7 +85,13 @@ func startDebugServer(addrPath string, logger log.Logger) (*http.Server, error) } }() - addr := fmt.Sprintf("http://%s/debug/?token=%s", listener.Addr().String(), token.String()) + url := url.URL{ + Scheme: "http", + Host: listener.Addr().String(), + Path: "/debug/", + RawQuery: "token=" + token.String(), + } + addr := url.String() // Write the address to a file for easy access by users if err := ioutil.WriteFile(addrPath, []byte(addr), 0600); err != nil { return nil, errors.Wrap(err, "writing debug address") @@ -98,7 +106,7 @@ func startDebugServer(addrPath string, logger log.Logger) (*http.Server, error) } // The below handler code is adapted from MIT licensed github.com/e-dard/netbug -func handler(token string, logger log.Logger) http.Handler { +func handler(token string, logger log.Logger) http.HandlerFunc { info := struct { Profiles []*pprof.Profile Token string @@ -107,7 +115,7 @@ func handler(token string, logger log.Logger) http.Handler { Token: url.QueryEscape(token), } - h := func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { name := strings.TrimPrefix(r.URL.Path, "/") switch name { case "": @@ -132,24 +140,20 @@ func handler(token string, logger log.Logger) http.Handler { nhpprof.Handler(name).ServeHTTP(w, r) } } - return http.HandlerFunc(h) } -func authHandler(token string, logger log.Logger) http.Handler { - h := handler(token, logger) - ah := func(w http.ResponseWriter, r *http.Request) { +func authHandler(token string, logger log.Logger) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { if r.FormValue("token") == token { - h.ServeHTTP(w, r) + handler(token, logger).ServeHTTP(w, r) } else { - w.WriteHeader(http.StatusUnauthorized) - fmt.Fprintln(w, "Unauthorized.") + http.Error(w, "Request must include valid token.", http.StatusUnauthorized) } } - return http.HandlerFunc(ah) } func registerAuthHandler(token string, mux *http.ServeMux, logger log.Logger) { - mux.Handle(debugPrefix, http.StripPrefix(debugPrefix, authHandler(token, logger))) + mux.Handle(debugPrefix, http.StripPrefix(debugPrefix, http.HandlerFunc(authHandler(token, logger)))) } var indexTmpl = template.Must(template.New("index").Parse(` @@ -158,7 +162,7 @@ var indexTmpl = template.Must(template.New("index").Parse(`
- profiles:
+ Profiles:
{{range .Profiles}}
{{.Count}}{{.Name}} @@ -168,7 +172,7 @@ var indexTmpl = template.Must(template.New("index").Parse(`
30-second trace

- debug information:
+ Debug information:
cmdline
symbol