Skip to content

Commit

Permalink
Set message timestamp to the metric time in kafka output (influxdata#…
Browse files Browse the repository at this point in the history
  • Loading branch information
danielnelson authored and Mathieu Lecarme committed Apr 17, 2020
1 parent e287733 commit d20124f
Showing 1 changed file with 11 additions and 4 deletions.
15 changes: 11 additions & 4 deletions plugins/outputs/kafka/kafka.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ type (
// SASL Password
SASLPassword string `toml:"sasl_password"`

Log telegraf.Logger `toml:"-"`

tlsConfig tls.Config
producer sarama.SyncProducer

Expand Down Expand Up @@ -316,13 +318,14 @@ func (k *Kafka) Write(metrics []telegraf.Metric) error {
for _, metric := range metrics {
buf, err := k.serializer.Serialize(metric)
if err != nil {
log.Printf("D! [outputs.kafka] Could not serialize metric: %v", err)
k.Log.Debugf("Could not serialize metric: %v", err)
continue
}

m := &sarama.ProducerMessage{
Topic: k.GetTopicName(metric),
Value: sarama.ByteEncoder(buf),
Topic: k.GetTopicName(metric),
Value: sarama.ByteEncoder(buf),
Timestamp: metric.Time(),
}

key, err := k.routingKey(metric)
Expand All @@ -342,7 +345,11 @@ func (k *Kafka) Write(metrics []telegraf.Metric) error {
if errs, ok := err.(sarama.ProducerErrors); ok {
for _, prodErr := range errs {
if prodErr.Err == sarama.ErrMessageSizeTooLarge {
log.Printf("E! Error writing to output [kafka]: Message too large, consider increasing `max_message_bytes`; dropping batch")
k.Log.Error("Message too large, consider increasing `max_message_bytes`; dropping batch")
return nil
}
if prodErr.Err == sarama.ErrInvalidTimestamp {
k.Log.Error("The timestamp of the message is out of acceptable range, consider increasing broker `message.timestamp.difference.max.ms`; dropping batch")
return nil
}
return prodErr
Expand Down

0 comments on commit d20124f

Please sign in to comment.