From 11e76f1740bfe20d2bc8c29ba832f4a13690dd49 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 16 Sep 2019 17:03:20 +0000 Subject: [PATCH 1/4] feature(app): Add a debugging summary function, exposed via http URL is /admin/summary --- app/collector.go | 19 +++++++++++++++++++ app/multitenant/aws_collector.go | 29 +++++++++++++++++++++++++++++ app/router.go | 12 ++++++++++++ prog/app.go | 1 + report/report.go | 20 ++++++++++++++++++++ 5 files changed, 81 insertions(+) diff --git a/app/collector.go b/app/collector.go index 5d9deea9bb..e76df513db 100644 --- a/app/collector.go +++ b/app/collector.go @@ -31,6 +31,7 @@ type Reporter interface { Report(context.Context, time.Time) (report.Report, error) HasReports(context.Context, time.Time) (bool, error) HasHistoricReports() bool + AdminSummary(context.Context, time.Time) (string, error) WaitOn(context.Context, chan struct{}) UnWait(context.Context, chan struct{}) } @@ -172,6 +173,20 @@ func (c *collector) HasHistoricReports() bool { return false } +// AdminSummary returns a string with some internal information about +// the report, which may be useful to troubleshoot. +func (c *collector) AdminSummary(ctx context.Context, timestamp time.Time) (string, error) { + c.mtx.Lock() + defer c.mtx.Unlock() + var b strings.Builder + for i := range c.reports { + fmt.Fprintf(&b, "%v: ", c.timestamps[i].Format(time.StampMilli)) + b.WriteString(c.reports[i].Summary()) + b.WriteByte('\n') + } + return b.String(), nil +} + // remove reports older than the app.window func (c *collector) clean() { var ( @@ -240,6 +255,10 @@ func (c StaticCollector) HasHistoricReports() bool { return false } +func (c StaticCollector) AdminSummary(ctx context.Context, timestamp time.Time) (string, error) { + return "not implemented", nil +} + // Add adds a report to the collector's internal state. It implements Adder. func (c StaticCollector) Add(context.Context, report.Report, []byte) error { return nil } diff --git a/app/multitenant/aws_collector.go b/app/multitenant/aws_collector.go index bdc0f0e6da..7d1c1dd1f4 100644 --- a/app/multitenant/aws_collector.go +++ b/app/multitenant/aws_collector.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "strconv" + "strings" "sync" "time" @@ -444,6 +445,34 @@ func (c *awsCollector) HasHistoricReports() bool { return true } +// AdminSummary returns a string with some internal information about +// the report, which may be useful to troubleshoot. +func (c *awsCollector) AdminSummary(ctx context.Context, timestamp time.Time) (string, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "awsCollector.Report") + defer span.Finish() + userid, err := c.cfg.UserIDer(ctx) + if err != nil { + return "", err + } + end := timestamp + start := end.Add(-c.cfg.Window) + reportKeys, err := c.getReportKeys(ctx, userid, start, end) + if err != nil { + return "", err + } + reports, err := c.reportsForKeysInRange(ctx, reportKeys, start.UnixNano(), end.UnixNano()) + if err != nil { + return "", err + } + var b strings.Builder + for i := range reports { + // TODO: print the key - note reports may be in a different order from reportKeys + b.WriteString(reports[i].Summary()) + b.WriteByte('\n') + } + return b.String(), nil +} + // calculateDynamoKeys generates the row & column keys for Dynamo. func calculateDynamoKeys(userid string, now time.Time) (string, string) { rowKey := fmt.Sprintf("%s-%s", userid, strconv.FormatInt(now.UnixNano()/time.Hour.Nanoseconds(), 10)) diff --git a/app/router.go b/app/router.go index 3bf27060dd..2b66caa6c1 100644 --- a/app/router.go +++ b/app/router.go @@ -159,6 +159,18 @@ func RegisterReportPostHandler(a Adder, router *mux.Router) { })) } +// RegisterAdminRoutes registers routes for admin calls with a http mux. +func RegisterAdminRoutes(router *mux.Router, reporter Reporter) { + get := router.Methods("GET").Subrouter() + get.Handle("/admin/summary", requestContextDecorator(func(ctx context.Context, w http.ResponseWriter, r *http.Request) { + summary, err := reporter.AdminSummary(ctx, time.Now()) + if err != nil { + respondWith(w, http.StatusBadRequest, err) + } + fmt.Fprintln(w, summary) + })) +} + var newVersion = struct { sync.Mutex *xfer.NewVersionInfo diff --git a/prog/app.go b/prog/app.go index 27050e8ec7..31c7847ed7 100644 --- a/prog/app.go +++ b/prog/app.go @@ -67,6 +67,7 @@ func router(collector app.Collector, controlRouter app.ControlRouter, pipeRouter app.RegisterControlRoutes(router, controlRouter) app.RegisterPipeRoutes(router, pipeRouter) app.RegisterTopologyRoutes(router, app.WebReporter{Reporter: collector, MetricsGraphURL: metricsGraphURL}, capabilities) + app.RegisterAdminRoutes(router, collector) uiHandler := http.FileServer(GetFS(externalUI)) router.PathPrefix("/ui").Name("static").Handler( diff --git a/report/report.go b/report/report.go index c7c1f09792..3c2f788d07 100644 --- a/report/report.go +++ b/report/report.go @@ -559,6 +559,26 @@ func (r Report) upgradeDNSRecords() Report { return r } +func (r Report) Summary() string { + ret := "" + if len(r.Host.Nodes) == 1 { + for k, _ := range r.Host.Nodes { + ret = k + ":" + } + } + count := 0 + r.WalkNamedTopologies(func(n string, t *Topology) { + if len(t.Nodes) > 0 { + count++ + if count > 1 { + ret = ret + ", " + } + ret = ret + fmt.Sprintf("%s:%d", n, len(t.Nodes)) + } + }) + return ret +} + // Sampling describes how the packet data sources for this report were // sampled. It can be used to calculate effective sample rates. We can't // just put the rate here, because that can't be accurately merged. Counts From a7d3cbedb56aedf2e856b8c5a0fd5759ca139036 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 18 Sep 2019 14:42:47 +0000 Subject: [PATCH 2/4] lint: make lint happy --- app/collector.go | 1 + report/report.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/collector.go b/app/collector.go index e76df513db..6481b4ef5b 100644 --- a/app/collector.go +++ b/app/collector.go @@ -255,6 +255,7 @@ func (c StaticCollector) HasHistoricReports() bool { return false } +// AdminSummary implements Reporter func (c StaticCollector) AdminSummary(ctx context.Context, timestamp time.Time) (string, error) { return "not implemented", nil } diff --git a/report/report.go b/report/report.go index 3c2f788d07..be4bf7ead6 100644 --- a/report/report.go +++ b/report/report.go @@ -559,10 +559,11 @@ func (r Report) upgradeDNSRecords() Report { return r } +// Summary returns a human-readable string summarising the contents, for diagnostic purposes func (r Report) Summary() string { ret := "" if len(r.Host.Nodes) == 1 { - for k, _ := range r.Host.Nodes { + for k := range r.Host.Nodes { ret = k + ":" } } From 57918b0ac5e4e7aa874c7e45564cfd6336d145bd Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 18 Sep 2019 14:43:49 +0000 Subject: [PATCH 3/4] formatting --- report/report.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/report/report.go b/report/report.go index be4bf7ead6..6fd70f6ffa 100644 --- a/report/report.go +++ b/report/report.go @@ -564,7 +564,7 @@ func (r Report) Summary() string { ret := "" if len(r.Host.Nodes) == 1 { for k := range r.Host.Nodes { - ret = k + ":" + ret = k + ": " } } count := 0 From 20c378dce53d03bc3cc7fc16431dad0b23ecc99a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 18 Sep 2019 14:47:26 +0000 Subject: [PATCH 4/4] docs: Added admin/summary to the faq --- site/faq.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/site/faq.md b/site/faq.md index e138f9213c..53874000e6 100644 --- a/site/faq.md +++ b/site/faq.md @@ -65,6 +65,12 @@ Scope doesn't support LDAP right now. OSS Scope reports aren't persistent and the probe keeps the last 15 seconds of metrics in memory. +## Admin Endpoints + +Scope exposes the following http endpoints that can be used for troubleshooting: + +- `/admin/summary` - lists the reports being used by the app, with counts of each node type (containers, processes, etc.). + ## API Endpoints Scope exposes the following endpoints that can be used by external monitoring services.