From b1a7e90324610b2311844e03357518d868e1e179 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Tue, 29 Sep 2020 11:29:28 +0200 Subject: [PATCH 1/2] o365input: Restart after fatal error (#21258) Update the o365input to restart the input after a fatal error is encountered, for example an authentication token refresh error or a parsing error. This enables the input to be more resilient against transient errors. Before this patch, the input would index an error document and terminate. Now it will index an error and restart after a fixed timeout of 5 minutes. (cherry picked from commit 8716d98860a03cc77456c7766b8a34f8e3482021) --- CHANGELOG.next.asciidoc | 28 +++++++++++++++ x-pack/filebeat/input/o365audit/input.go | 44 +++++++++++++++++------- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 092d2028656..20ecf09d6cb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -50,6 +50,34 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix `setup.dashboards.index` setting not working. {pull}17749[17749] - Fix Elasticsearch license endpoint URL referenced in error message. {issue}17880[17880] {pull}18030[18030] - Change `decode_json_fields` processor, to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958] +- [Autodiscover] Check if runner is already running before starting again. {pull}18564[18564] +- Fix `keystore add` hanging under Windows. {issue}18649[18649] {pull}18654[18654] +- Fix an issue where error messages are not accurate in mapstriface. {issue}18662[18662] {pull}18663[18663] +- Fix regression in `add_kubernetes_metadata`, so configured `indexers` and `matchers` are used if defaults are not disabled. {issue}18481[18481] {pull}18818[18818] +- Fix potential race condition in fingerprint processor. {pull}18738[18738] +- Add better handling for Kubernetes Update and Delete watcher events. {pull}18882[18882] +- Fix the `translate_sid` processor's handling of unconfigured target fields. {issue}18990[18990] {pull}18991[18991] +- Fixed a service restart failure under Windows. {issue}18914[18914] {pull}18916[18916] +- The `monitoring.elasticsearch.api_key` value is correctly base64-encoded before being sent to the monitoring Elasticsearch cluster. {issue}18939[18939] {pull}18945[18945] +- Fix kafka topic setting not allowing upper case characters. {pull}18854[18854] {issue}18640[18640] +- Fix redis key setting not allowing upper case characters. {pull}18854[18854] {issue}18640[18640] +- Fix config reload metrics (`libbeat.config.module.start/stops/running`). {pull}19168[19168] +- Fix metrics hints builder to avoid wrong container metadata usage when port is not exposed {pull}18979[18979] +- Server-side TLS config now validates certificate and key are both specified {pull}19584[19584] +- Fix terminating pod autodiscover issue. {pull}20084[20084] +- Fix seccomp policy for calls to `chmod` and `chown`. {pull}20054[20054] +- Remove unnecessary restarts of metricsets while using Node autodiscover {pull}19974[19974] +- Output errors when Kibana index pattern setup fails. {pull}20121[20121] +- Fix issue in autodiscover that kept inputs stopped after config updates. {pull}20305[20305] +- Log debug message if the Kibana dashboard can not be imported from the archive because of the invalid archive directory structure {issue}12211[12211], {pull}13387[13387] +- Add service resource in k8s cluster role. {pull}20546[20546] +- [Metricbeat][Kubernetes] Change cluster_ip field from ip to keyword. {pull}20571[20571] +- Rename cloud.provider `az` value to `azure` inside the add_cloud_metadata processor. {pull}20689[20689] +- Add missing country_name geo field in `add_host_metadata` and `add_observer_metadata` processors. {issue}20796[20796] {pull}20811[20811] +- [Autodiscover] Handle input-not-finished errors in config reload. {pull}20915[20915] +- Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898] +- Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197] +- The `o365input` and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21259[21258] *Auditbeat* diff --git a/x-pack/filebeat/input/o365audit/input.go b/x-pack/filebeat/input/o365audit/input.go index 1ced85ce337..1a97768c156 100644 --- a/x-pack/filebeat/input/o365audit/input.go +++ b/x-pack/filebeat/input/o365audit/input.go @@ -26,6 +26,9 @@ import ( const ( pluginName = "o365audit" fieldsPrefix = pluginName + + // How long to retry when a fatal error is encountered in the input. + failureRetryInterval = time.Minute * 5 ) type o365input struct { @@ -107,6 +110,34 @@ func (inp *o365input) Run( src cursor.Source, cursor cursor.Cursor, publisher cursor.Publisher, +) error { + for ctx.Cancelation.Err() == nil { + err := inp.runOnce(ctx, src, cursor, publisher) + if err == nil { + break + } + if ctx.Cancelation.Err() != err && err != context.Canceled { + msg := common.MapStr{} + msg.Put("error.message", err.Error()) + msg.Put("event.kind", "pipeline_error") + event := beat.Event{ + Timestamp: time.Now(), + Fields: msg, + } + publisher.Publish(event, nil) + ctx.Logger.Errorf("Input failed: %v", err) + ctx.Logger.Infof("Restarting in %v", failureRetryInterval) + time.Sleep(failureRetryInterval) + } + } + return nil +} + +func (inp *o365input) runOnce( + ctx v2.Context, + src cursor.Source, + cursor cursor.Cursor, + publisher cursor.Publisher, ) error { stream := src.(*stream) tenantID, contentType := stream.tenantID, stream.contentType @@ -156,18 +187,7 @@ func (inp *o365input) Run( } log.Infow("Start fetching events", "cursor", start) - err = poller.Run(action) - if err != nil && ctx.Cancelation.Err() != err && err != context.Canceled { - msg := common.MapStr{} - msg.Put("error.message", err.Error()) - msg.Put("event.kind", "pipeline_error") - event := beat.Event{ - Timestamp: time.Now(), - Fields: msg, - } - publisher.Publish(event, nil) - } - return err + return poller.Run(action) } func initCheckpoint(log *logp.Logger, c cursor.Cursor, maxRetention time.Duration) checkpoint { From d26a50f54cdcb78b0db471a3ef0dd5d372c5a10d Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Tue, 29 Sep 2020 17:01:21 +0200 Subject: [PATCH 2/2] Fix changelog --- CHANGELOG.next.asciidoc | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 20ecf09d6cb..d85eaf585d4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -50,34 +50,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix `setup.dashboards.index` setting not working. {pull}17749[17749] - Fix Elasticsearch license endpoint URL referenced in error message. {issue}17880[17880] {pull}18030[18030] - Change `decode_json_fields` processor, to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958] -- [Autodiscover] Check if runner is already running before starting again. {pull}18564[18564] -- Fix `keystore add` hanging under Windows. {issue}18649[18649] {pull}18654[18654] -- Fix an issue where error messages are not accurate in mapstriface. {issue}18662[18662] {pull}18663[18663] -- Fix regression in `add_kubernetes_metadata`, so configured `indexers` and `matchers` are used if defaults are not disabled. {issue}18481[18481] {pull}18818[18818] -- Fix potential race condition in fingerprint processor. {pull}18738[18738] -- Add better handling for Kubernetes Update and Delete watcher events. {pull}18882[18882] -- Fix the `translate_sid` processor's handling of unconfigured target fields. {issue}18990[18990] {pull}18991[18991] -- Fixed a service restart failure under Windows. {issue}18914[18914] {pull}18916[18916] -- The `monitoring.elasticsearch.api_key` value is correctly base64-encoded before being sent to the monitoring Elasticsearch cluster. {issue}18939[18939] {pull}18945[18945] -- Fix kafka topic setting not allowing upper case characters. {pull}18854[18854] {issue}18640[18640] -- Fix redis key setting not allowing upper case characters. {pull}18854[18854] {issue}18640[18640] -- Fix config reload metrics (`libbeat.config.module.start/stops/running`). {pull}19168[19168] -- Fix metrics hints builder to avoid wrong container metadata usage when port is not exposed {pull}18979[18979] -- Server-side TLS config now validates certificate and key are both specified {pull}19584[19584] -- Fix terminating pod autodiscover issue. {pull}20084[20084] -- Fix seccomp policy for calls to `chmod` and `chown`. {pull}20054[20054] -- Remove unnecessary restarts of metricsets while using Node autodiscover {pull}19974[19974] -- Output errors when Kibana index pattern setup fails. {pull}20121[20121] -- Fix issue in autodiscover that kept inputs stopped after config updates. {pull}20305[20305] -- Log debug message if the Kibana dashboard can not be imported from the archive because of the invalid archive directory structure {issue}12211[12211], {pull}13387[13387] -- Add service resource in k8s cluster role. {pull}20546[20546] -- [Metricbeat][Kubernetes] Change cluster_ip field from ip to keyword. {pull}20571[20571] -- Rename cloud.provider `az` value to `azure` inside the add_cloud_metadata processor. {pull}20689[20689] -- Add missing country_name geo field in `add_host_metadata` and `add_observer_metadata` processors. {issue}20796[20796] {pull}20811[20811] -- [Autodiscover] Handle input-not-finished errors in config reload. {pull}20915[20915] -- Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898] -- Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197] -- The `o365input` and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21259[21258] +- The `o365input` and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21258[21258] *Auditbeat*