From 408b5d59742332adcb6628490c1c410100f6bbfe Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 21 Jan 2021 21:25:03 -0500 Subject: [PATCH 1/5] [libbeat] Expose the new Sarama flag 'DisablePAFXFAST' in the Kafka output (#23629) (cherry picked from commit 621c9e2cf614235a104a170229cb324ebfbea811) --- CHANGELOG.next.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 4 ++++ filebeat/filebeat.reference.yml | 4 ++++ heartbeat/heartbeat.reference.yml | 4 ++++ journalbeat/journalbeat.reference.yml | 4 ++++ libbeat/_meta/config/output-kafka.reference.yml.tmpl | 4 ++++ libbeat/outputs/kafka/config.go | 2 ++ libbeat/outputs/kafka/docs/kafka.asciidoc | 6 ++++++ metricbeat/metricbeat.reference.yml | 4 ++++ packetbeat/packetbeat.reference.yml | 4 ++++ winlogbeat/winlogbeat.reference.yml | 4 ++++ x-pack/auditbeat/auditbeat.reference.yml | 4 ++++ x-pack/filebeat/filebeat.reference.yml | 4 ++++ x-pack/heartbeat/heartbeat.reference.yml | 4 ++++ x-pack/metricbeat/metricbeat.reference.yml | 4 ++++ x-pack/packetbeat/packetbeat.reference.yml | 4 ++++ x-pack/winlogbeat/winlogbeat.reference.yml | 4 ++++ 17 files changed, 65 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index cc26fd9ca60..f2817cd51e0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -409,6 +409,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539] - Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595] - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564] +- Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 29fe7904694..1f1eec14811 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -857,6 +857,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 0371b3419b7..0bc83848cd6 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1736,6 +1736,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index efceef26fac..41e92cb8cda 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1034,6 +1034,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index b1ed0f96ddd..a35580637df 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -799,6 +799,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/libbeat/_meta/config/output-kafka.reference.yml.tmpl b/libbeat/_meta/config/output-kafka.reference.yml.tmpl index c1240f75867..366652fd23e 100644 --- a/libbeat/_meta/config/output-kafka.reference.yml.tmpl +++ b/libbeat/_meta/config/output-kafka.reference.yml.tmpl @@ -131,6 +131,10 @@ # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + {{include "ssl.reference.yml.tmpl" . | indent 2 }} # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. #kerberos.enabled: true diff --git a/libbeat/outputs/kafka/config.go b/libbeat/outputs/kafka/config.go index 1d834ecb894..9b4900e47fd 100644 --- a/libbeat/outputs/kafka/config.go +++ b/libbeat/outputs/kafka/config.go @@ -69,6 +69,7 @@ type kafkaConfig struct { Password string `config:"password"` Codec codec.Config `config:"codec"` Sasl saslConfig `config:"sasl"` + EnableFAST bool `config:"enable_krb5_fast"` } type saslConfig struct { @@ -241,6 +242,7 @@ func newSaramaConfig(log *logp.Logger, config *kafkaConfig) (*sarama.Config, err Username: config.Kerberos.Username, Password: config.Kerberos.Password, Realm: config.Kerberos.Realm, + DisablePAFXFAST: !config.EnableFAST, } case config.Username != "": diff --git a/libbeat/outputs/kafka/docs/kafka.asciidoc b/libbeat/outputs/kafka/docs/kafka.asciidoc index 50994e17cb5..54864b226c4 100644 --- a/libbeat/outputs/kafka/docs/kafka.asciidoc +++ b/libbeat/outputs/kafka/docs/kafka.asciidoc @@ -291,6 +291,12 @@ The ACK reliability level required from broker. 0=no response, 1=wait for local Note: If set to 0, no ACKs are returned by Kafka. Messages might be lost silently on error. +===== `enable_krb5_fast` + +beta[] + +Enable Kerberos FAST authentication. This may conflict with some Active Directory installations. It is separate from the standard Kerberos settings because this flag only applies to the Kafka output. The default is `false`. + ===== `ssl` Configuration options for SSL parameters like the root CA for Kafka connections. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 3f231fe65ff..377ed5ae9de 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1633,6 +1633,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 43790cca594..d9da1a46826 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1351,6 +1351,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index cecb09f8adb..94235948d17 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -779,6 +779,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index b1603a93183..ea1465ecf90 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -913,6 +913,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index f1ffb499fe4..bc7dadcc87c 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3534,6 +3534,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index efceef26fac..41e92cb8cda 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1034,6 +1034,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index bbc1f09d158..9e76e01255a 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -2135,6 +2135,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index b65ba79e677..027fd0a529b 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1351,6 +1351,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 65f242e2adc..92b332092f5 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -822,6 +822,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true From 184052f5b1edc2432ebd162089976318d1a52462 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 25 Jan 2021 15:27:21 -0500 Subject: [PATCH 2/5] ping the CI --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f2817cd51e0..1269b37edb9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -407,7 +407,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324] - Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330] - Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539] -- Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595] +- Update the baseline version of Sarama (the Kafka support library) to 1.27.2. {pull}23595[23595] - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564] - Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629] From 2e72054e7e57f0b6f330e56745c3ecbbcd35be7b Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 25 Jan 2021 15:27:58 -0500 Subject: [PATCH 3/5] ping the CI --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1269b37edb9..f2817cd51e0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -407,7 +407,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324] - Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330] - Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539] -- Update the baseline version of Sarama (the Kafka support library) to 1.27.2. {pull}23595[23595] +- Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595] - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564] - Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629] From 27bad8a4835adcb5c5bec0d86e1a7c038defd434 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 26 Jan 2021 14:52:53 -0500 Subject: [PATCH 4/5] ping the CI --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f2817cd51e0..2918e277fd5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -410,7 +410,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595] - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564] - Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629] - + *Auditbeat* - Reference kubernetes manifests include configuration for auditd and enrichment with kubernetes metadata. {pull}17431[17431] From b06709b692dda555069dd0f506e5d6e0fe09e006 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 26 Jan 2021 14:53:02 -0500 Subject: [PATCH 5/5] ping the CI --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2918e277fd5..f2817cd51e0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -410,7 +410,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595] - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564] - Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629] - + *Auditbeat* - Reference kubernetes manifests include configuration for auditd and enrichment with kubernetes metadata. {pull}17431[17431]