feat: Harvest should export to multiple InfluxDB exporters (#2722)

Fixes: #849

cmd/exporters/influxdb/influxdb.go

@@ -163,7 +163,7 @@ func (e *InfluxDB) Export(data *matrix.Matrix) (exporter.Stats, error) {
 	if metrics, stats, err = e.Render(data); err == nil && len(metrics) != 0 {
 		// fix render time
 		if err = e.Metadata.LazyAddValueInt64("time", "render", time.Since(s).Microseconds()); err != nil {
-			e.Logger.Error().Stack().Err(err).Msg("metadata render time")
+			e.Logger.Error().Err(err).Msg("metadata render time")
 		}
 		// in debug mode, don't actually export but write to log
 		if e.Options.Debug {
@@ -174,15 +174,11 @@ func (e *InfluxDB) Export(data *matrix.Matrix) (exporter.Stats, error) {
 			return stats, nil
 			// otherwise, to the actual export: send to the DB
 		} else if err = e.Emit(metrics); err != nil {
-			e.Logger.Error().Stack().Err(err).
-				Str("object", data.Object).
-				Str("uuid", data.UUID).
-				Msg("Failed to emit metrics")
-			return stats, err
+			return stats, fmt.Errorf("unable to emit object: %s, uuid: %s, err=%w", data.Object, data.UUID, err)
 		}
 	}
 
-	e.Logger.Debug().Msgf("(%s.%s) --> exported %d data points", data.Object, data.UUID, len(metrics))
+	e.Logger.Debug().Str("object", data.Object).Str("uuid", data.UUID).Int("numMetric", len(metrics)).Msg("exported")
 
 	// update metadata
 	if err = e.Metadata.LazySetValueInt64("time", "export", time.Since(s).Microseconds()); err != nil {

cmd/poller/poller.go

@@ -511,10 +511,10 @@ func (p *Poller) Run() {
 			// @TODO if there are no "master" exporters, don't collect metadata
 			for _, ee := range p.exporters {
 				if _, err := ee.Export(p.metadata); err != nil {
-					logger.Error().Stack().Err(err).Msg("export component metadata:")
+					logger.Error().Err(err).Msg("export component metadata:")
 				}
 				if _, err := ee.Export(p.status); err != nil {
-					logger.Error().Stack().Err(err).Msg("export target metadata:")
+					logger.Error().Err(err).Msg("export target metadata:")
 				}
 			}
 

cmd/tools/doctor/testdata/testConfig.yml

@@ -163,6 +163,8 @@ Pollers:
       - prometheus
       - foo1
       - foo2
+      - influxy
+      - influxz
 
   pass-with-escape:
     username: admin

pkg/conf/conf.go

@@ -396,20 +396,26 @@ func (i *IntRange) UnmarshalYAML(node *yaml.Node) error {
 }
 
 // GetUniqueExporters returns the unique set of exporter types from the list of export names.
-// For example, if two prometheus exporters are configured for a poller, the last one is returned
+// For example, if two Prometheus exporters are configured for a poller, the last one is returned.
+// Multiple InfluxDB exporters are allowed.
 func GetUniqueExporters(exporterNames []string) []string {
 	var resultExporters []string
-	definedExporters := Config.Exporters
-	exporterMap := make(map[string]string)
+	exporterMap := make(map[string][]string)
+
 	for _, ec := range exporterNames {
-		e, ok := definedExporters[ec]
+		e, ok := Config.Exporters[ec]
 		if ok {
-			exporterMap[e.Type] = ec
+			exporterMap[e.Type] = append(exporterMap[e.Type], ec)
 		}
 	}
 
-	for _, value := range exporterMap {
-		resultExporters = append(resultExporters, value)
+	for eType, value := range exporterMap {
+		if eType == "Prometheus" {
+			// if there are multiple prometheus exporters, only the last one is used
+			resultExporters = append(resultExporters, value[len(value)-1])
+			continue
+		}
+		resultExporters = append(resultExporters, value...)
 	}
 	return resultExporters
 }

pkg/conf/conf_test.go

@@ -196,7 +196,7 @@ func TestUniqueExportersByType(t *testing.T) {
 	t.Run("Exporters are unique by type", func(t *testing.T) {
 		exporters := GetUniqueExporters(poller2.Exporters)
 		sort.Strings(exporters)
-		want := []string{"foo1", "foo2", "prometheus"}
+		want := []string{"foo1", "foo2", "influxy", "influxz", "prometheus"}
 		if !reflect.DeepEqual(want, exporters) {
 			t.Fatalf(`expected %v but got %v`, want, exporters)
 		}