From 83bbd57548e71272597e0bd846eeeee14f817f22 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Tue, 9 Jun 2020 10:22:15 +0200 Subject: [PATCH] Filebeat: Fix o365 module issues (#18948) - Fix scary data-loss warning on startup - Avoid API errors being processed by the JS pipeline - Fix dissect error about overiding client.port - Fix module passing API settings to the input - Document max_period using the right units --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/filebeat.reference.yml | 2 +- x-pack/filebeat/input/o365audit/state.go | 5 +---- x-pack/filebeat/input/o365audit/state_test.go | 8 ++------ x-pack/filebeat/module/o365/_meta/config.yml | 2 +- x-pack/filebeat/module/o365/audit/config/input.yml | 6 ++---- x-pack/filebeat/module/o365/audit/config/pipeline.js | 5 ++++- x-pack/filebeat/modules.d/o365.yml.disabled | 2 +- 8 files changed, 13 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ce3f03918cc..c53a9c9789e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -172,6 +172,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix Kubernetes Watcher goroutine leaks when input config is invalid and `input.reload` is enabled. {issue}18629[18629] {pull}18630[18630] - Okta module now sets the Elasticsearch `_id` field to the Okta UUID value contained in each system log to minimize the possibility of duplicating events. {pull}18953[18953] - Fix improper nesting of session_issuer object in aws cloudtrail fileset. {issue}18894[18894] {pull}18915[18915] +- Fix `o365` module ignoring `var.api` settings. {pull}18948[18948] *Heartbeat* diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index ebe415b688c..02722e6ae70 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -852,7 +852,7 @@ filebeat.modules: # authentication_endpoint: "https://login.microsoftonline.us/" # resource: "https://manage.office365.us" # - # max_retention: 7d + # max_retention: 168h # max_requests_per_minute: 2000 # poll_interval: 3m diff --git a/x-pack/filebeat/input/o365audit/state.go b/x-pack/filebeat/input/o365audit/state.go index ecdb8fc89ff..6992437ccab 100644 --- a/x-pack/filebeat/input/o365audit/state.go +++ b/x-pack/filebeat/input/o365audit/state.go @@ -114,10 +114,7 @@ func (s *stateStorage) Load(key stream) (cursor, error) { } cur, err := s.persister.Load(key) if err != nil { - if err != errStateNotFound { - return cur, err - } - cur = newCursor(key, time.Time{}) + return newCursor(key, time.Time{}), err } return cur, s.saveUnsafe(cur) } diff --git a/x-pack/filebeat/input/o365audit/state_test.go b/x-pack/filebeat/input/o365audit/state_test.go index 71b778d16ec..4e274578e83 100644 --- a/x-pack/filebeat/input/o365audit/state_test.go +++ b/x-pack/filebeat/input/o365audit/state_test.go @@ -21,18 +21,14 @@ func TestNoopState(t *testing.T) { t.Run("new state", func(t *testing.T) { st := newStateStorage(noopPersister{}) cur, err := st.Load(myStream) - if !assert.NoError(t, err) { - t.Fatal(err) - } + assert.Equal(t, errStateNotFound, err) empty := newCursor(myStream, time.Time{}) assert.Equal(t, empty, cur) }) t.Run("update state", func(t *testing.T) { st := newStateStorage(noopPersister{}) cur, err := st.Load(myStream) - if !assert.NoError(t, err) { - t.Fatal(err) - } + assert.Equal(t, errStateNotFound, err) advanced := cur.TryAdvance(content{ Type: tn, ID: "1234", diff --git a/x-pack/filebeat/module/o365/_meta/config.yml b/x-pack/filebeat/module/o365/_meta/config.yml index 8114b404aa4..b1a30d6dbe9 100644 --- a/x-pack/filebeat/module/o365/_meta/config.yml +++ b/x-pack/filebeat/module/o365/_meta/config.yml @@ -40,6 +40,6 @@ # authentication_endpoint: "https://login.microsoftonline.us/" # resource: "https://manage.office365.us" # - # max_retention: 7d + # max_retention: 168h # max_requests_per_minute: 2000 # poll_interval: 3m diff --git a/x-pack/filebeat/module/o365/audit/config/input.yml b/x-pack/filebeat/module/o365/audit/config/input.yml index 061d0f532af..d41a5bb9aab 100644 --- a/x-pack/filebeat/module/o365/audit/config/input.yml +++ b/x-pack/filebeat/module/o365/audit/config/input.yml @@ -19,10 +19,7 @@ content_type: {{ end }} {{ end }} {{ if .api }} -api: -{{ range $k, $v := .api }} - - {{ $k }}: {{ $v -}} -{{ end }} +api: {{.api | tojson }} {{ end }} {{ else if eq .input "file" }} @@ -51,6 +48,7 @@ processors: - 2006-01-02T15:04:05 {{ end }} - script: + when.has_fields: ['o365audit'] lang: javascript id: o365audit_script file: ${path.home}/module/o365/audit/config/pipeline.js diff --git a/x-pack/filebeat/module/o365/audit/config/pipeline.js b/x-pack/filebeat/module/o365/audit/config/pipeline.js index ae8e1a7afe6..8ff5e572fc2 100644 --- a/x-pack/filebeat/module/o365/audit/config/pipeline.js +++ b/x-pack/filebeat/module/o365/audit/config/pipeline.js @@ -729,7 +729,10 @@ function AuditProcessor(tenant_names, debug) { tokenizer: '[%{_ip}]:%{port}', field: 'client.address', target_prefix: 'client', - 'when.contains.client.address': ']:', + 'when.and': [ + {'not.has_fields': ['client._ip', 'client.port']}, + {'contains.client.address': ']:'}, + ], })); builder.Add("extractClientIPv4Port", new processor.Dissect({ tokenizer: '%{_ip}:%{port}', diff --git a/x-pack/filebeat/modules.d/o365.yml.disabled b/x-pack/filebeat/modules.d/o365.yml.disabled index af65cc45d22..a2bdc1ecee3 100644 --- a/x-pack/filebeat/modules.d/o365.yml.disabled +++ b/x-pack/filebeat/modules.d/o365.yml.disabled @@ -43,6 +43,6 @@ # authentication_endpoint: "https://login.microsoftonline.us/" # resource: "https://manage.office365.us" # - # max_retention: 7d + # max_retention: 168h # max_requests_per_minute: 2000 # poll_interval: 3m