Add docs and changelog for elastic#12867 (elastic#23103)

CHANGELOG.next.asciidoc

@@ -32,6 +32,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
 - API address is a required setting in `add_cloudfoundry_metadata`. {pull}21759[21759]
 - Update to ECS 1.7.0. {pull}22571[22571]
+- Add support for SCRAM-SHA-512 and SCRAM-SHA-256 in Kafka output. {pull}12867[12867]
 
 *Auditbeat*
 

auditbeat/auditbeat.reference.yml

@@ -747,6 +747,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Auditbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

filebeat/filebeat.reference.yml

@@ -1618,6 +1618,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Filebeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

heartbeat/heartbeat.reference.yml

@@ -924,6 +924,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Heartbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

journalbeat/journalbeat.reference.yml

@@ -689,6 +689,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Journalbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

libbeat/_meta/config/output-kafka.reference.yml.tmpl

@@ -33,6 +33,10 @@
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version {{.BeatName | title}} is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

libbeat/outputs/kafka/config.go

@@ -73,8 +73,6 @@ type kafkaConfig struct {
 
 type saslConfig struct {
 	SaslMechanism string `config:"mechanism"`
-	//SaslUsername  string `config:"username"` //maybe use ssl.username ssl.password instead in future?
-	//SaslPassword  string `config:"password"`
 }
 
 type metaConfig struct {
@@ -149,12 +147,16 @@ func (c *saslConfig) configureSarama(config *sarama.Config) error {
 	case saslTypePlaintext:
 		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypePlaintext)
 	case saslTypeSCRAMSHA256:
+		cfgwarn.Beta("SCRAM-SHA-256 authentication for Kafka is beta.")
+
 		config.Net.SASL.Handshake = true
 		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA256)
 		config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
 			return &XDGSCRAMClient{HashGeneratorFcn: SHA256}
 		}
 	case saslTypeSCRAMSHA512:
+		cfgwarn.Beta("SCRAM-SHA-512 authentication for Kafka is beta.")
+
 		config.Net.SASL.Handshake = true
 		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA512)
 		config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
@@ -228,7 +230,8 @@ func newSaramaConfig(log *logp.Logger, config *kafkaConfig) (*sarama.Config, err
 		k.Net.TLS.Config = tls.BuildModuleConfig("")
 	}
 
-	if config.Kerberos.IsEnabled() {
+	switch {
+	case config.Kerberos.IsEnabled():
 		cfgwarn.Beta("Kerberos authentication for Kafka is beta.")
 
 		k.Net.SASL.Enable = true
@@ -242,9 +245,8 @@ func newSaramaConfig(log *logp.Logger, config *kafkaConfig) (*sarama.Config, err
 			Password:           config.Kerberos.Password,
 			Realm:              config.Kerberos.Realm,
 		}
-	}
 
-	if config.Username != "" {
+	case config.Username != "":
 		k.Net.SASL.Enable = true
 		k.Net.SASL.User = config.Username
 		k.Net.SASL.Password = config.Password

libbeat/outputs/kafka/docs/kafka.asciidoc

@@ -5,14 +5,6 @@
 <titleabbrev>Kafka</titleabbrev>
 ++++
 
-[IMPORTANT]
-.Known issue in version 7.8.0
-====
-The Kafka output fails to connect when using multiple TLS brokers. We advise
-not to upgrade to {beatname_uc} 7.8.0 if you're using the Kafka output in this
-configuration.
-====
-
 The Kafka output sends events to Apache Kafka.
 
 To use this output, edit the {beatname_uc} configuration file to disable the {es}
@@ -79,12 +71,29 @@ See <<kafka-compatibility>> for information on supported versions.
 ===== `username`
 
 The username for connecting to Kafka. If username is configured, the password
-must be configured as well. Only SASL/PLAIN is supported.
+must be configured as well.
 
 ===== `password`
 
 The password for connecting to Kafka.
 
+===== `sasl.mechanism`
+
+beta[]
+
+The SASL mechanism to use when connecting to Kafka. It can be one of:
+
+* `PLAIN` for SASL/PLAIN.
+* `SCRAM-SHA-256` for SCRAM-SHA-256.
+* `SCRAM-SHA-512` for SCRAM-SHA-512.
+
+If `sasl.mechanism` is not set, `PLAIN` is used if `username` and `password`
+are provided. Otherwise, SASL authentication is disabled.
+
+To use `GSSAPI` mechanism to authenticate with Kerberos, you must leave this
+field empty, and use the <<kerberos-option-kafka>> options.
+
+
 [[topic-option-kafka]]
 ===== `topic`
 
@@ -308,6 +317,7 @@ Configuration options for SSL parameters like the root CA for Kafka connections.
 https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions#why-cant-sarama-connect-to-my-kafka-cluster-using-ssl[Filebeat's Kafka library].
 See <<configuration-ssl>> for more information.
 
+[[kerberos-option-kafka]]
 ===== `kerberos`
 
 beta[]

metricbeat/metricbeat.reference.yml

@@ -1523,6 +1523,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Metricbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

packetbeat/packetbeat.reference.yml

@@ -1241,6 +1241,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Packetbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

winlogbeat/winlogbeat.reference.yml

@@ -669,6 +669,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Winlogbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

x-pack/auditbeat/auditbeat.reference.yml

@@ -803,6 +803,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Auditbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

x-pack/filebeat/filebeat.reference.yml

@@ -3458,6 +3458,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Filebeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

x-pack/heartbeat/heartbeat.reference.yml

@@ -924,6 +924,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Heartbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

x-pack/metricbeat/metricbeat.reference.yml

@@ -2024,6 +2024,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Metricbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

x-pack/packetbeat/packetbeat.reference.yml

@@ -1241,6 +1241,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Packetbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 

x-pack/winlogbeat/winlogbeat.reference.yml

@@ -712,6 +712,10 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
+  # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+  # Defaults to PLAIN when `username` and `password` are configured.
+  #sasl.mechanism: ''
+
   # Kafka version Winlogbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'