Filebeat: Fix o365 module issues (elastic#18948)

- Fix scary data-loss warning on startup - Avoid API errors being processed by the JS pipeline - Fix dissect error about overiding client.port - Fix module passing API settings to the input - Document max_period using the right units
adriansr · Jun 9, 2020 · 83bbd57 · 83bbd57
1 parent 2dad8db
commit 83bbd57
Show file tree

Hide file tree

Showing 8 changed files with 13 additions and 18 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -172,6 +172,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix Kubernetes Watcher goroutine leaks when input config is invalid and `input.reload` is enabled. {issue}18629[18629] {pull}18630[18630]
 - Okta module now sets the Elasticsearch `_id` field to the Okta UUID value contained in each system log to minimize the possibility of duplicating events. {pull}18953[18953]
 - Fix improper nesting of session_issuer object in aws cloudtrail fileset. {issue}18894[18894] {pull}18915[18915]
+- Fix `o365` module ignoring `var.api` settings. {pull}18948[18948]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -852,7 +852,7 @@ filebeat.modules:
     #   authentication_endpoint: "https://login.microsoftonline.us/"
     #   resource: "https://manage.office365.us"
     #
-    #   max_retention: 7d
+    #   max_retention: 168h
     #   max_requests_per_minute: 2000
     #   poll_interval: 3m
 

diff --git a/x-pack/filebeat/input/o365audit/state.go b/x-pack/filebeat/input/o365audit/state.go
@@ -114,10 +114,7 @@ func (s *stateStorage) Load(key stream) (cursor, error) {
 	}
 	cur, err := s.persister.Load(key)
 	if err != nil {
-		if err != errStateNotFound {
-			return cur, err
-		}
-		cur = newCursor(key, time.Time{})
+		return newCursor(key, time.Time{}), err
 	}
 	return cur, s.saveUnsafe(cur)
 }

diff --git a/x-pack/filebeat/input/o365audit/state_test.go b/x-pack/filebeat/input/o365audit/state_test.go
@@ -21,18 +21,14 @@ func TestNoopState(t *testing.T) {
 	t.Run("new state", func(t *testing.T) {
 		st := newStateStorage(noopPersister{})
 		cur, err := st.Load(myStream)
-		if !assert.NoError(t, err) {
-			t.Fatal(err)
-		}
+		assert.Equal(t, errStateNotFound, err)
 		empty := newCursor(myStream, time.Time{})
 		assert.Equal(t, empty, cur)
 	})
 	t.Run("update state", func(t *testing.T) {
 		st := newStateStorage(noopPersister{})
 		cur, err := st.Load(myStream)
-		if !assert.NoError(t, err) {
-			t.Fatal(err)
-		}
+		assert.Equal(t, errStateNotFound, err)
 		advanced := cur.TryAdvance(content{
 			Type:       tn,
 			ID:         "1234",

diff --git a/x-pack/filebeat/module/o365/_meta/config.yml b/x-pack/filebeat/module/o365/_meta/config.yml
@@ -40,6 +40,6 @@
     #   authentication_endpoint: "https://login.microsoftonline.us/"
     #   resource: "https://manage.office365.us"
     #
-    #   max_retention: 7d
+    #   max_retention: 168h
     #   max_requests_per_minute: 2000
     #   poll_interval: 3m
diff --git a/x-pack/filebeat/module/o365/audit/config/input.yml b/x-pack/filebeat/module/o365/audit/config/input.yml
@@ -19,10 +19,7 @@ content_type:
 {{ end }}
 {{ end }}
 {{ if .api }}
-api:
-{{ range $k, $v := .api }}
- - {{ $k }}: {{ $v -}}
-{{ end }}
+api: {{.api | tojson }}
 {{ end }}
 
 {{ else if eq .input "file" }}
@@ -51,6 +48,7 @@ processors:
         - 2006-01-02T15:04:05
 {{ end }}
   - script:
+      when.has_fields: ['o365audit']
       lang: javascript
       id: o365audit_script
       file: ${path.home}/module/o365/audit/config/pipeline.js

diff --git a/x-pack/filebeat/module/o365/audit/config/pipeline.js b/x-pack/filebeat/module/o365/audit/config/pipeline.js
@@ -729,7 +729,10 @@ function AuditProcessor(tenant_names, debug) {
         tokenizer: '[%{_ip}]:%{port}',
         field: 'client.address',
         target_prefix: 'client',
-        'when.contains.client.address': ']:',
+        'when.and': [
+            {'not.has_fields': ['client._ip', 'client.port']},
+            {'contains.client.address': ']:'},
+        ],
     }));
     builder.Add("extractClientIPv4Port", new processor.Dissect({
         tokenizer: '%{_ip}:%{port}',

diff --git a/x-pack/filebeat/modules.d/o365.yml.disabled b/x-pack/filebeat/modules.d/o365.yml.disabled
@@ -43,6 +43,6 @@
     #   authentication_endpoint: "https://login.microsoftonline.us/"
     #   resource: "https://manage.office365.us"
     #
-    #   max_retention: 7d
+    #   max_retention: 168h
     #   max_requests_per_minute: 2000
     #   poll_interval: 3m