From c0a7c01f0e892d8b21a6b5530df20ea56ba5ffb2 Mon Sep 17 00:00:00 2001 From: Sean Lingren Date: Tue, 11 Apr 2023 17:08:30 -0700 Subject: [PATCH 1/2] add metrics, metrics analysis, and concurrency --- .gitignore | 2 + cmd/dashboards.go | 39 ++++++++---- cmd/datadog.go | 155 ++++++++++++++++++++++++++++++++++++++++++++-- cmd/metrics.go | 90 +++++++++++++++++++++++++++ cmd/monitors.go | 39 ++++++++---- cmd/root.go | 6 ++ go.mod | 1 + go.sum | 4 ++ 8 files changed, 308 insertions(+), 28 deletions(-) create mode 100644 cmd/metrics.go diff --git a/.gitignore b/.gitignore index 98b689a..24c2b84 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,10 @@ dist/ datadog-exporter # Export Directories +metrics/ monitors/ dashboards/ +analysis.json # Coverage *.out diff --git a/cmd/dashboards.go b/cmd/dashboards.go index 8b953e9..43602d7 100644 --- a/cmd/dashboards.go +++ b/cmd/dashboards.go @@ -1,10 +1,12 @@ package cmd import ( + "fmt" "path/filepath" "github.com/spf13/cobra" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) // newDashboardsCmd creates the command exporting dashboards. @@ -25,19 +27,32 @@ func (c *cli) newDashboardsCmd() *cobra.Command { c.log.Fatalw("failed to list dashboards", zap.Error(err)) } + g, ctx := errgroup.WithContext(cmd.Context()) + g.SetLimit(concurrency) + for _, id := range dashboards { - log := c.log.With(zap.String("id", id)) - log.Info("exporting dashboard") - - json, err := ddc.dashboardJSON(cmd.Context(), id) - if err != nil { - c.log.Fatalw("failed to get dashboard json", zap.Error(err)) - } - - err = ddc.writeJSONToFile(filepath.Join(args[0], id), json) - if err != nil { - c.log.Fatalw("failed to write dashboard json to file", zap.Error(err)) - } + id := id + g.Go(func() error { + log := c.log.With(zap.String("id", id)) + log.Info("exporting dashboard") + + json, err := ddc.dashboardJSON(ctx, id) + if err != nil { + return fmt.Errorf("failed to get json for dashboard: %s: %w", id, err) + } + + err = ddc.writeJSONToFile(filepath.Join(args[0], id), json) + if err != nil { + return fmt.Errorf("failed to write json for dashboard: %s: %w", id, err) + } + + return nil + }) + } + + err = g.Wait() + if err != nil { + c.log.Fatalw("failed to export dashboards", zap.Error(err)) } c.log.Info("dashboard export completed") diff --git a/cmd/datadog.go b/cmd/datadog.go index fe484d4..34d6568 100644 --- a/cmd/datadog.go +++ b/cmd/datadog.go @@ -7,17 +7,23 @@ import ( "fmt" "io" "os" + "sort" + "strconv" + "strings" "github.com/DataDog/datadog-api-client-go/v2/api/datadog" "github.com/DataDog/datadog-api-client-go/v2/api/datadogV1" + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" ) const fileMode = 0600 // ddc holds the datadog client and other info for all out export operations. type ddc struct { - dashAPI *datadogV1.DashboardsApi - monitorAPI *datadogV1.MonitorsApi + dashAPI *datadogV1.DashboardsApi + monitorAPI *datadogV1.MonitorsApi + metricAPI *datadogV1.MetricsApi + metricV2API *datadogV2.MetricsApi } // newDDC creates a new datadog client. @@ -25,8 +31,10 @@ func newDDC() *ddc { dd := datadog.NewAPIClient(datadog.NewConfiguration()) return &ddc{ - dashAPI: datadogV1.NewDashboardsApi(dd), - monitorAPI: datadogV1.NewMonitorsApi(dd), + dashAPI: datadogV1.NewDashboardsApi(dd), + monitorAPI: datadogV1.NewMonitorsApi(dd), + metricAPI: datadogV1.NewMetricsApi(dd), + metricV2API: datadogV2.NewMetricsApi(dd), } } @@ -64,6 +72,92 @@ func (d *ddc) monitors(ctx context.Context) ([]int64, error) { return mns, nil } +// metrics returns a list of all metrics IDs in the account matching +// the search filter. An empty search will return all metrics. +func (d *ddc) metrics(ctx context.Context, search string) ([]string, error) { + ctx = datadog.NewDefaultContext(ctx) + + list, _, err := d.metricAPI.ListMetrics(ctx, search) + if err != nil { + return nil, fmt.Errorf("failed to list metrics: %w", err) + } + + return list.Results.Metrics, nil +} + +// metricTags return a map of tag keys to a list of tag values for a given metric name. +func (d *ddc) metricTags(ctx context.Context, name string) (map[string][]string, error) { + list, _, err := d.metricV2API.ListTagsByMetricName(ctx, name) + if err != nil { + return nil, fmt.Errorf("failed to get metric tags with name: %s: %w", name, err) + } + + out := map[string][]string{} + for _, tag := range list.Data.Attributes.Tags { + key, value, found := strings.Cut(tag, ":") + + if !found { + out[key] = []string{} + continue + } + + _, ok := out[key] + if !ok { + out[key] = []string{} + } + + out[key] = append(out[key], value) + } + + return out, nil +} + +// metricAnalysis gets all metrics for the given search filter and outputs a list of +// all tags used by those metrics sorted by the cardinality of those tags. +func (d *ddc) metricAnalysis(ctx context.Context, search string) ([][2]string, error) { + ctx = datadog.NewDefaultContext(ctx) + + metrics, err := d.metrics(ctx, search) + if err != nil { + return nil, fmt.Errorf("failed to list metrics: %w", err) + } + + // create a map of tags to their unique values for all metrics + allTags := map[string]map[string]bool{} + + for _, metric := range metrics { + tags, err := d.metricTags(ctx, metric) + if err != nil { + return nil, fmt.Errorf("failed to list tags for metric: %s, %w", metric, err) + } + + for key, values := range tags { + _, ok := allTags[key] + if !ok { + allTags[key] = map[string]bool{} + } + + for _, value := range values { + allTags[key][value] = true + } + } + } + + analysis := [][2]string{} + for key, values := range allTags { + analysis = append(analysis, [2]string{key, strconv.Itoa(len(values))}) + } + + // we know that the string is a valid number so we can ignore conversion errors + sort.Slice(analysis, func(i, j int) bool { + one, _ := strconv.Atoi(analysis[i][1]) //nolint:errcheck + two, _ := strconv.Atoi(analysis[j][1]) //nolint:errcheck + return one > two + }) + + return analysis, nil +} + // dashboardJSON returns the JSON definition for a given dashboard ID. func (d *ddc) dashboardJSON(ctx context.Context, id string) ([]byte, error) { ctx = datadog.NewDefaultContext(ctx) @@ -110,6 +204,59 @@ func (d *ddc) monitorJSON(ctx context.Context, id int64) ([]byte, error) { return dash.Bytes(), nil } +// metricJSON returns the JSON definition for a given metric name. +func (d *ddc) metricJSON(ctx context.Context, name string) ([]byte, error) { + ctx = datadog.NewDefaultContext(ctx) + + _, resp, err := d.metricAPI.GetMetricMetadata(ctx, name) + if err != nil { + return nil, fmt.Errorf("failed to get metric with name: %s: %w", name, err) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body for name: %s: %w", name, err) + } + + respMap := map[string]any{} + + err = json.Unmarshal(body, &respMap) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal json for name: %s: %w", name, err) + } + + tags, err := d.metricTags(ctx, name) + if err != nil { + return nil, fmt.Errorf("failed to get metric tags for metric with name: %s: %w", name, err) + } + + respMap["tags"] = tags + + metric, err := json.MarshalIndent(respMap, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to marshal indent json for name: %s: %w", name, err) + } + + return metric, nil +} + +// metricAnalysisJSON returns the JSON definition for a given metric analysis. +func (d *ddc) metricAnalysisJSON(ctx context.Context, search string) ([]byte, error) { + ctx = datadog.NewDefaultContext(ctx) + + analysis, err := d.metricAnalysis(ctx, search) + if err != nil { + return nil, fmt.Errorf("failed to get metric analysis for search: %s: %w", search, err) + } + + out, err := json.MarshalIndent(map[string]any{"analysis": analysis}, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to marshal metric analysis indent json for search: %s: %w", search, err) + } + + return out, nil +} + // writeJSONToFile writes json bytes to a local file for archiving. func (d *ddc) writeJSONToFile(name string, json []byte) error { err := os.WriteFile(name+".json", json, fileMode) diff --git a/cmd/metrics.go b/cmd/metrics.go new file mode 100644 index 0000000..1a0a973 --- /dev/null +++ b/cmd/metrics.go @@ -0,0 +1,90 @@ +package cmd + +import ( + "fmt" + "path/filepath" + + "github.com/spf13/cobra" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +// newMetricsCmd creates the command exporting metrics. +func (c *cli) newMetricsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "metrics ", + Short: "metrics", + Example: "datadog-exporter metrics ./metrics", + Args: cobra.ExactArgs(1), + + Run: func(cmd *cobra.Command, args []string) { + c.log.Info("metric export started") + ddc := newDDC() + + c.log.Info("listing metrics") + metrics, err := ddc.metrics(cmd.Context(), "") + if err != nil { + c.log.Fatalw("failed to list metrics", zap.Error(err)) + } + + g, ctx := errgroup.WithContext(cmd.Context()) + g.SetLimit(concurrency) + + for _, name := range metrics { + name := name + g.Go(func() error { + log := c.log.With(zap.String("name", name)) + log.Info("exporting metric") + + json, err := ddc.metricJSON(ctx, name) + if err != nil { + return fmt.Errorf("failed to get json for metric: %s: %w", name, err) + } + + err = ddc.writeJSONToFile(filepath.Join(args[0], name), json) + if err != nil { + return fmt.Errorf("failed to write json to file for metric: %s: %w", name, err) + } + + return nil + }) + } + + err = g.Wait() + if err != nil { + c.log.Fatalw("failed to export metrics", zap.Error(err)) + } + + c.log.Info("metric export completed") + }, + } + + return cmd +} + +// newMetricsAnalysisCmd creates the command exporting a metric analysis. +func (c *cli) newMetricsAnalysisCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "metrics-analysis ", + Short: "metrics analysis", + Example: "datadog-exporter metrics-analysis 'system.cpu.' .", + Args: cobra.ExactArgs(2), + + Run: func(cmd *cobra.Command, args []string) { + c.log.Info("metric analysis export started") + ddc := newDDC() + + json, err := ddc.metricAnalysisJSON(cmd.Context(), args[0]) + if err != nil { + c.log.Fatalw("failed to run analysis", zap.Error(err)) + } + + err = ddc.writeJSONToFile(filepath.Join(args[1], "analysis"), json) + if err != nil { + c.log.Fatalw("failed to write metric json to file", zap.Error(err)) + } + }, + } + + return cmd +} diff --git a/cmd/monitors.go b/cmd/monitors.go index 92208ef..046fe5d 100644 --- a/cmd/monitors.go +++ b/cmd/monitors.go @@ -1,11 +1,13 @@ package cmd import ( + "fmt" "path/filepath" "strconv" "github.com/spf13/cobra" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) // newMonitorsCmd creates the command exporting monitors. @@ -26,19 +28,32 @@ func (c *cli) newMonitorsCmd() *cobra.Command { c.log.Fatalw("failed to list monitors", zap.Error(err)) } + g, ctx := errgroup.WithContext(cmd.Context()) + g.SetLimit(concurrency) + for _, id := range monitors { - log := c.log.With(zap.Int64("id", id)) - log.Info("exporting monitor") - - json, err := ddc.monitorJSON(cmd.Context(), id) - if err != nil { - c.log.Fatalw("failed to get monitor json", zap.Error(err)) - } - - err = ddc.writeJSONToFile(filepath.Join(args[0], strconv.FormatInt(id, 10)), json) - if err != nil { - c.log.Fatalw("failed to write monitor json to file", zap.Error(err)) - } + id := id + g.Go(func() error { + log := c.log.With(zap.Int64("id", id)) + log.Info("exporting monitor") + + json, err := ddc.monitorJSON(ctx, id) + if err != nil { + return fmt.Errorf("failed to get json for monitor: %d: %w", id, err) + } + + err = ddc.writeJSONToFile(filepath.Join(args[0], strconv.FormatInt(id, 10)), json) + if err != nil { + return fmt.Errorf("failed to write json for monitor: %d: %w", id, err) + } + + return nil + }) + } + + err = g.Wait() + if err != nil { + c.log.Fatalw("failed to export monitors", zap.Error(err)) } c.log.Info("monitor export completed") diff --git a/cmd/root.go b/cmd/root.go index 11b3b83..3d1ce00 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,6 +4,10 @@ import ( "github.com/spf13/cobra" ) +// this limit can probably be raised significantly once we have retries +// on 429s from datadog. until then large exports will get rate limited. +const concurrency = 2 + // newRootCmd creates our base cobra command to add all subcommands to. func (c *cli) newRootCmd() *cobra.Command { cmd := &cobra.Command{ @@ -18,6 +22,8 @@ func (c *cli) newRootCmd() *cobra.Command { c.newVersionCmd(), c.newDashboardsCmd(), c.newMonitorsCmd(), + c.newMetricsCmd(), + c.newMetricsAnalysisCmd(), ) return cmd diff --git a/go.mod b/go.mod index 0374765..2b8ee20 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.9.0 github.com/spf13/cobra v1.6.1 go.uber.org/zap v1.24.0 + golang.org/x/sync v0.1.0 ) require ( diff --git a/go.sum b/go.sum index 5b30dfa..495136e 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,10 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= From 34329d3d9243a2ed853b62b497519f754dd6b943 Mon Sep 17 00:00:00 2001 From: Sean Lingren Date: Tue, 11 Apr 2023 17:19:24 -0700 Subject: [PATCH 2/2] go mod tidy --- go.sum | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/go.sum b/go.sum index 0dd650e..36390bf 100644 --- a/go.sum +++ b/go.sum @@ -30,11 +30,10 @@ go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=