diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 02de2e92601..ad292aa3cee 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -32,6 +32,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
- API address is a required setting in `add_cloudfoundry_metadata`. {pull}21759[21759]
- Update to ECS 1.7.0. {pull}22571[22571]
+- Add support for SCRAM-SHA-512 and SCRAM-SHA-256 in Kafka output. {pull}12867[12867]
*Auditbeat*
diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
index 518873f8d07..cc4d3a65feb 100644
--- a/auditbeat/auditbeat.reference.yml
+++ b/auditbeat/auditbeat.reference.yml
@@ -747,6 +747,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Auditbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
index 45d84ca8b56..8c53656512f 100644
--- a/filebeat/filebeat.reference.yml
+++ b/filebeat/filebeat.reference.yml
@@ -1618,6 +1618,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Filebeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
index 49d8197a44a..37e3e2ed122 100644
--- a/heartbeat/heartbeat.reference.yml
+++ b/heartbeat/heartbeat.reference.yml
@@ -924,6 +924,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Heartbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml
index 6eb595e1654..7e875edcf8e 100644
--- a/journalbeat/journalbeat.reference.yml
+++ b/journalbeat/journalbeat.reference.yml
@@ -689,6 +689,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Journalbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/libbeat/_meta/config/output-kafka.reference.yml.tmpl b/libbeat/_meta/config/output-kafka.reference.yml.tmpl
index 169abe2efaa..c1240f75867 100644
--- a/libbeat/_meta/config/output-kafka.reference.yml.tmpl
+++ b/libbeat/_meta/config/output-kafka.reference.yml.tmpl
@@ -33,6 +33,10 @@
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version {{.BeatName | title}} is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/libbeat/outputs/kafka/config.go b/libbeat/outputs/kafka/config.go
index fd3151b92c7..b3c8e984fe9 100644
--- a/libbeat/outputs/kafka/config.go
+++ b/libbeat/outputs/kafka/config.go
@@ -73,8 +73,6 @@ type kafkaConfig struct {
type saslConfig struct {
SaslMechanism string `config:"mechanism"`
- //SaslUsername string `config:"username"` //maybe use ssl.username ssl.password instead in future?
- //SaslPassword string `config:"password"`
}
type metaConfig struct {
@@ -149,12 +147,16 @@ func (c *saslConfig) configureSarama(config *sarama.Config) error {
case saslTypePlaintext:
config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypePlaintext)
case saslTypeSCRAMSHA256:
+ cfgwarn.Beta("SCRAM-SHA-256 authentication for Kafka is beta.")
+
config.Net.SASL.Handshake = true
config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA256)
config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
return &XDGSCRAMClient{HashGeneratorFcn: SHA256}
}
case saslTypeSCRAMSHA512:
+ cfgwarn.Beta("SCRAM-SHA-512 authentication for Kafka is beta.")
+
config.Net.SASL.Handshake = true
config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA512)
config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
@@ -228,7 +230,8 @@ func newSaramaConfig(log *logp.Logger, config *kafkaConfig) (*sarama.Config, err
k.Net.TLS.Config = tls.BuildModuleConfig("")
}
- if config.Kerberos.IsEnabled() {
+ switch {
+ case config.Kerberos.IsEnabled():
cfgwarn.Beta("Kerberos authentication for Kafka is beta.")
k.Net.SASL.Enable = true
@@ -242,9 +245,8 @@ func newSaramaConfig(log *logp.Logger, config *kafkaConfig) (*sarama.Config, err
Password: config.Kerberos.Password,
Realm: config.Kerberos.Realm,
}
- }
- if config.Username != "" {
+ case config.Username != "":
k.Net.SASL.Enable = true
k.Net.SASL.User = config.Username
k.Net.SASL.Password = config.Password
diff --git a/libbeat/outputs/kafka/docs/kafka.asciidoc b/libbeat/outputs/kafka/docs/kafka.asciidoc
index e1dcb77b6bb..f61d4c5d985 100644
--- a/libbeat/outputs/kafka/docs/kafka.asciidoc
+++ b/libbeat/outputs/kafka/docs/kafka.asciidoc
@@ -5,14 +5,6 @@
Kafka
++++
-[IMPORTANT]
-.Known issue in version 7.8.0
-====
-The Kafka output fails to connect when using multiple TLS brokers. We advise
-not to upgrade to {beatname_uc} 7.8.0 if you're using the Kafka output in this
-configuration.
-====
-
The Kafka output sends events to Apache Kafka.
To use this output, edit the {beatname_uc} configuration file to disable the {es}
@@ -79,12 +71,29 @@ See <> for information on supported versions.
===== `username`
The username for connecting to Kafka. If username is configured, the password
-must be configured as well. Only SASL/PLAIN is supported.
+must be configured as well.
===== `password`
The password for connecting to Kafka.
+===== `sasl.mechanism`
+
+beta[]
+
+The SASL mechanism to use when connecting to Kafka. It can be one of:
+
+* `PLAIN` for SASL/PLAIN.
+* `SCRAM-SHA-256` for SCRAM-SHA-256.
+* `SCRAM-SHA-512` for SCRAM-SHA-512.
+
+If `sasl.mechanism` is not set, `PLAIN` is used if `username` and `password`
+are provided. Otherwise, SASL authentication is disabled.
+
+To use `GSSAPI` mechanism to authenticate with Kerberos, you must leave this
+field empty, and use the <> options.
+
+
[[topic-option-kafka]]
===== `topic`
@@ -308,6 +317,7 @@ Configuration options for SSL parameters like the root CA for Kafka connections.
https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions#why-cant-sarama-connect-to-my-kafka-cluster-using-ssl[Filebeat's Kafka library].
See <> for more information.
+[[kerberos-option-kafka]]
===== `kerberos`
beta[]
diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
index ca18dd7ed47..f7530d7e386 100644
--- a/metricbeat/metricbeat.reference.yml
+++ b/metricbeat/metricbeat.reference.yml
@@ -1523,6 +1523,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Metricbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml
index c811a8f0c04..57142c14263 100644
--- a/packetbeat/packetbeat.reference.yml
+++ b/packetbeat/packetbeat.reference.yml
@@ -1241,6 +1241,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Packetbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml
index 0b08e43361a..316df4ae3dd 100644
--- a/winlogbeat/winlogbeat.reference.yml
+++ b/winlogbeat/winlogbeat.reference.yml
@@ -669,6 +669,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Winlogbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml
index 91044ea06ea..97be60e98fe 100644
--- a/x-pack/auditbeat/auditbeat.reference.yml
+++ b/x-pack/auditbeat/auditbeat.reference.yml
@@ -803,6 +803,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Auditbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
index 9c8aea58124..0adca29eaae 100644
--- a/x-pack/filebeat/filebeat.reference.yml
+++ b/x-pack/filebeat/filebeat.reference.yml
@@ -3458,6 +3458,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Filebeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml
index 49d8197a44a..37e3e2ed122 100644
--- a/x-pack/heartbeat/heartbeat.reference.yml
+++ b/x-pack/heartbeat/heartbeat.reference.yml
@@ -924,6 +924,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Heartbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
index 0a6a954ec6f..66c0e02eaa2 100644
--- a/x-pack/metricbeat/metricbeat.reference.yml
+++ b/x-pack/metricbeat/metricbeat.reference.yml
@@ -2024,6 +2024,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Metricbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml
index c811a8f0c04..57142c14263 100644
--- a/x-pack/packetbeat/packetbeat.reference.yml
+++ b/x-pack/packetbeat/packetbeat.reference.yml
@@ -1241,6 +1241,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Packetbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml
index f6188759f70..03652ce2788 100644
--- a/x-pack/winlogbeat/winlogbeat.reference.yml
+++ b/x-pack/winlogbeat/winlogbeat.reference.yml
@@ -712,6 +712,10 @@ output.elasticsearch:
#username: ''
#password: ''
+ # SASL authentication mechanism used. Can be one of PLAIN, SCRAM-SHA-256 or SCRAM-SHA-512.
+ # Defaults to PLAIN when `username` and `password` are configured.
+ #sasl.mechanism: ''
+
# Kafka version Winlogbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'