add metrics, metrics analysis, and concurrency

.gitignore

@@ -4,8 +4,10 @@ dist/
 datadog-exporter
 
 # Export Directories
+metrics/
 monitors/
 dashboards/
+analysis.json
 
 # Coverage
 *.out

cmd/dashboards.go

@@ -1,10 +1,12 @@
 package cmd
 
 import (
+	"fmt"
 	"path/filepath"
 
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
 )
 
 // newDashboardsCmd creates the command exporting dashboards.
@@ -25,19 +27,32 @@ func (c *cli) newDashboardsCmd() *cobra.Command {
 				c.log.Fatalw("failed to list dashboards", zap.Error(err))
 			}
 
+			g, ctx := errgroup.WithContext(cmd.Context())
+			g.SetLimit(concurrency)
+
 			for _, id := range dashboards {
-				log := c.log.With(zap.String("id", id))
-				log.Info("exporting dashboard")
-
-				json, err := ddc.dashboardJSON(cmd.Context(), id)
-				if err != nil {
-					c.log.Fatalw("failed to get dashboard json", zap.Error(err))
-				}
-
-				err = ddc.writeJSONToFile(filepath.Join(args[0], id), json)
-				if err != nil {
-					c.log.Fatalw("failed to write dashboard json to file", zap.Error(err))
-				}
+				id := id
+				g.Go(func() error {
+					log := c.log.With(zap.String("id", id))
+					log.Info("exporting dashboard")
+
+					json, err := ddc.dashboardJSON(ctx, id)
+					if err != nil {
+						return fmt.Errorf("failed to get json for dashboard: %s: %w", id, err)
+					}
+
+					err = ddc.writeJSONToFile(filepath.Join(args[0], id), json)
+					if err != nil {
+						return fmt.Errorf("failed to write json for dashboard: %s: %w", id, err)
+					}
+
+					return nil
+				})
+			}
+
+			err = g.Wait()
+			if err != nil {
+				c.log.Fatalw("failed to export dashboards", zap.Error(err))
 			}
 
 			c.log.Info("dashboard export completed")

cmd/datadog.go

@@ -7,26 +7,34 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"sort"
+	"strconv"
+	"strings"
 
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadog"
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
+	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV2"
 )
 
 const fileMode = 0600
 
 // ddc holds the datadog client and other info for all out export operations.
 type ddc struct {
-	dashAPI    *datadogV1.DashboardsApi
-	monitorAPI *datadogV1.MonitorsApi
+	dashAPI     *datadogV1.DashboardsApi
+	monitorAPI  *datadogV1.MonitorsApi
+	metricAPI   *datadogV1.MetricsApi
+	metricV2API *datadogV2.MetricsApi
 }
 
 // newDDC creates a new datadog client.
 func newDDC() *ddc {
 	dd := datadog.NewAPIClient(datadog.NewConfiguration())
 
 	return &ddc{
-		dashAPI:    datadogV1.NewDashboardsApi(dd),
-		monitorAPI: datadogV1.NewMonitorsApi(dd),
+		dashAPI:     datadogV1.NewDashboardsApi(dd),
+		monitorAPI:  datadogV1.NewMonitorsApi(dd),
+		metricAPI:   datadogV1.NewMetricsApi(dd),
+		metricV2API: datadogV2.NewMetricsApi(dd),
 	}
 }
 
@@ -64,6 +72,92 @@ func (d *ddc) monitors(ctx context.Context) ([]int64, error) {
 	return mns, nil
 }
 
+// metrics returns a list of all metrics IDs in the account matching
+// the search filter. An empty search will return all metrics.
+func (d *ddc) metrics(ctx context.Context, search string) ([]string, error) {
+	ctx = datadog.NewDefaultContext(ctx)
+
+	list, _, err := d.metricAPI.ListMetrics(ctx, search)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list metrics: %w", err)
+	}
+
+	return list.Results.Metrics, nil
+}
+
+// metricTags return a map of tag keys to a list of tag values for a given metric name.
+func (d *ddc) metricTags(ctx context.Context, name string) (map[string][]string, error) {
+	list, _, err := d.metricV2API.ListTagsByMetricName(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get metric tags with name: %s: %w", name, err)
+	}
+
+	out := map[string][]string{}
+	for _, tag := range list.Data.Attributes.Tags {
+		key, value, found := strings.Cut(tag, ":")
+
+		if !found {
+			out[key] = []string{}
+			continue
+		}
+
+		_, ok := out[key]
+		if !ok {
+			out[key] = []string{}
+		}
+
+		out[key] = append(out[key], value)
+	}
+
+	return out, nil
+}
+
+// metricAnalysis gets all metrics for the given search filter and outputs a list of
+// all tags used by those metrics sorted by the cardinality of those tags.
+func (d *ddc) metricAnalysis(ctx context.Context, search string) ([][2]string, error) {
+	ctx = datadog.NewDefaultContext(ctx)
+
+	metrics, err := d.metrics(ctx, search)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list metrics: %w", err)
+	}
+
+	// create a map of tags to their unique values for all metrics
+	allTags := map[string]map[string]bool{}
+
+	for _, metric := range metrics {
+		tags, err := d.metricTags(ctx, metric)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list tags for metric: %s, %w", metric, err)
+		}
+
+		for key, values := range tags {
+			_, ok := allTags[key]
+			if !ok {
+				allTags[key] = map[string]bool{}
+			}
+
+			for _, value := range values {
+				allTags[key][value] = true
+			}
+		}
+	}
+
+	analysis := [][2]string{}
+	for key, values := range allTags {
+		analysis = append(analysis, [2]string{key, strconv.Itoa(len(values))})
+	}
+
+	// we know that the string is a valid number so we can ignore conversion errors
+	sort.Slice(analysis, func(i, j int) bool {
+		one, _ := strconv.Atoi(analysis[i][1]) //nolint:errcheck
+		two, _ := strconv.Atoi(analysis[j][1]) //nolint:errcheck
+		return one > two
+	})
+
+	return analysis, nil
+}
+
 // dashboardJSON returns the JSON definition for a given dashboard ID.
 func (d *ddc) dashboardJSON(ctx context.Context, id string) ([]byte, error) {
 	ctx = datadog.NewDefaultContext(ctx)
@@ -110,6 +204,59 @@ func (d *ddc) monitorJSON(ctx context.Context, id int64) ([]byte, error) {
 	return dash.Bytes(), nil
 }
 
+// metricJSON returns the JSON definition for a given metric name.
+func (d *ddc) metricJSON(ctx context.Context, name string) ([]byte, error) {
+	ctx = datadog.NewDefaultContext(ctx)
+
+	_, resp, err := d.metricAPI.GetMetricMetadata(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get metric with name: %s: %w", name, err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body for name: %s: %w", name, err)
+	}
+
+	respMap := map[string]any{}
+
+	err = json.Unmarshal(body, &respMap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal json for name: %s: %w", name, err)
+	}
+
+	tags, err := d.metricTags(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get metric tags for metric with name: %s: %w", name, err)
+	}
+
+	respMap["tags"] = tags
+
+	metric, err := json.MarshalIndent(respMap, "", "    ")
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal indent json for name: %s: %w", name, err)
+	}
+
+	return metric, nil
+}
+
+// metricAnalysisJSON returns the JSON definition for a given metric analysis.
+func (d *ddc) metricAnalysisJSON(ctx context.Context, search string) ([]byte, error) {
+	ctx = datadog.NewDefaultContext(ctx)
+
+	analysis, err := d.metricAnalysis(ctx, search)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get metric analysis for search: %s: %w", search, err)
+	}
+
+	out, err := json.MarshalIndent(map[string]any{"analysis": analysis}, "", "    ")
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal metric analysis indent json for search: %s: %w", search, err)
+	}
+
+	return out, nil
+}
+
 // writeJSONToFile writes json bytes to a local file for archiving.
 func (d *ddc) writeJSONToFile(name string, json []byte) error {
 	err := os.WriteFile(name+".json", json, fileMode)

cmd/metrics.go

@@ -0,0 +1,90 @@
+package cmd
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
+)
+
+// newMetricsCmd creates the command exporting metrics.
+func (c *cli) newMetricsCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "metrics <path>",
+		Short:   "metrics",
+		Example: "datadog-exporter metrics ./metrics",
+		Args:    cobra.ExactArgs(1),
+
+		Run: func(cmd *cobra.Command, args []string) {
+			c.log.Info("metric export started")
+			ddc := newDDC()
+
+			c.log.Info("listing metrics")
+			metrics, err := ddc.metrics(cmd.Context(), "")
+			if err != nil {
+				c.log.Fatalw("failed to list metrics", zap.Error(err))
+			}
+
+			g, ctx := errgroup.WithContext(cmd.Context())
+			g.SetLimit(concurrency)
+
+			for _, name := range metrics {
+				name := name
+				g.Go(func() error {
+					log := c.log.With(zap.String("name", name))
+					log.Info("exporting metric")
+
+					json, err := ddc.metricJSON(ctx, name)
+					if err != nil {
+						return fmt.Errorf("failed to get json for metric: %s: %w", name, err)
+					}
+
+					err = ddc.writeJSONToFile(filepath.Join(args[0], name), json)
+					if err != nil {
+						return fmt.Errorf("failed to write json to file for metric: %s: %w", name, err)
+					}
+
+					return nil
+				})
+			}
+
+			err = g.Wait()
+			if err != nil {
+				c.log.Fatalw("failed to export metrics", zap.Error(err))
+			}
+
+			c.log.Info("metric export completed")
+		},
+	}
+
+	return cmd
+}
+
+// newMetricsAnalysisCmd creates the command exporting a metric analysis.
+func (c *cli) newMetricsAnalysisCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "metrics-analysis <search> <path>",
+		Short:   "metrics analysis",
+		Example: "datadog-exporter metrics-analysis 'system.cpu.' .",
+		Args:    cobra.ExactArgs(2),
+
+		Run: func(cmd *cobra.Command, args []string) {
+			c.log.Info("metric analysis export started")
+			ddc := newDDC()
+
+			json, err := ddc.metricAnalysisJSON(cmd.Context(), args[0])
+			if err != nil {
+				c.log.Fatalw("failed to run analysis", zap.Error(err))
+			}
+
+			err = ddc.writeJSONToFile(filepath.Join(args[1], "analysis"), json)
+			if err != nil {
+				c.log.Fatalw("failed to write metric json to file", zap.Error(err))
+			}
+		},
+	}
+
+	return cmd
+}