Merge branch 'feature-ecs-1.8' into juniper-ecs-1.8

elastic · Feb 11, 2021 · ad66c2c · ad66c2c
2 parents 212d599 + 0b27310
commit ad66c2c
Show file tree

Hide file tree

Showing 64 changed files with 666 additions and 814 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -379,6 +379,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777]
 - Fix goroutines leak with some inputs in autodiscover. {pull}23722[23722]
 - Fix various processing errors in the Suricata module. {pull}23236[23236]
+- aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920]
 
 *Heartbeat*
 
@@ -836,6 +837,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Upgrade CEF module to ECS 1.8.0. {pull}23832[23832]
 - Upgrade fortinet/firewall to ECS 1.8 {issue}23118[23118] {pull}23902[23902]
 - Upgrade Zeek to ECS 1.8.0. {issue}23118[23118] {pull}23847[23847]
+- Updated azure module to ECS 1.8. {issue}23118[23118] {pull}23927[23927]
+- Update aws/s3access to ECS 1.8. {issue}23118[23118] {pull}23920[23920]
+- Upgrade panw module to ecs 1.8 {issue}23118[23118] {pull}23931[23931]
+- Updated aws/cloudtrail fileset to ECS 1.8. {issue}23118[23118] {pull}23911[23911]
 - Upgrade juniper/srx to ecs 1.8.0. {issue}23118[23118] {pull}23936[23936]
 
 *Heartbeat*

diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml
@@ -66,4 +66,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/file.yml b/x-pack/filebeat/module/aws/cloudtrail/config/file.yml
@@ -11,4 +11,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
@@ -27,6 +27,11 @@ processors:
       field: "json.userIdentity.type"
       target_field: "aws.cloudtrail.user_identity.type"
       ignore_failure: true
+  - append:
+      field: related.user
+      value: '{{json.userIdentity.userName}}'
+      allow_duplicates: false
+      if: 'ctx.json?.userIdentity?.userName != null'
   - rename:
       field: "json.userIdentity.userName"
       target_field: "user.name"
@@ -225,28 +230,16 @@ processors:
       field: "json.vpcEndpointId"
       target_field: "aws.cloudtrail.vpc_endpoint_id"
       ignore_failure: true
-  - script:
-      lang: painless
-      ignore_failure: true
-      source: >-
-        void addRelatedUser(def ctx, String userName) {
-          if (ctx.related == null) {
-            Map map = new HashMap();
-            ctx.put("related", map);
-          }
-          if (ctx.related.user == null) {
-            ArrayList al = new ArrayList();
-            ctx.related.put("user", al);
-          }
-          ctx.related.user.add(userName);
-        }
-        if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.userName != null) {
-          addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.userName);
-        }
-        if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null) {
-          addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.newUserName);
-        }
-
+  - append:
+      field: related.user
+      value: '{{aws.cloudtrail.flattened.request_parameters.userName}}'
+      allow_duplicates: false
+      if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.userName != null'
+  - append:
+      field: related.user
+      value: '{{aws.cloudtrail.flattened.request_parameters.newUserName}}'
+      allow_duplicates: false
+      if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null'
   - script:
       lang: painless
       ignore_failure: true
@@ -685,6 +678,32 @@ processors:
       field: "json.insightDetails"
       target_field: "aws.cloudtrail.insight_details"
       ignore_failure: true
+  - set:
+      field: group.id
+      value: '{{aws.cloudtrail.flattened.response_elements.group.groupId}}'
+      ignore_empty_value: true
+      ignore_failure: true
+  - set:
+      field: user.target.id
+      value: '{{aws.cloudtrail.flattened.response_elements.user.userId}}'
+      ignore_empty_value: true
+      ignore_failure: true
+  - set:
+      field: user.changes.name
+      value: '{{aws.cloudtrail.flattened.request_parameters.newUserName}}'
+      ignore_empty_value: true
+      ignore_failure: true
+  - set:
+      field: group.name
+      value: '{{aws.cloudtrail.flattened.request_parameters.groupName}}'
+      ignore_empty_value: true
+      ignore_failure: true
+  - set:
+      field: user.target.name
+      value: '{{aws.cloudtrail.flattened.request_parameters.userName}}'
+      ignore_empty_value: true
+      ignore_failure: true
+
   - remove:
       field:
         - "json"

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/add-user-to-group-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/add-user-to-group-json.log-expected.json
@@ -27,9 +27,11 @@
             "change"
         ],
         "fileset.name": "cloudtrail",
+        "group.name": "admin",
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -40,6 +42,7 @@
         ],
         "user.id": "EX_PRINCIPAL_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "AWSConsole"

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/change-password-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/change-password-json.log-expected.json
@@ -29,6 +29,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
@@ -70,6 +73,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 720,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json
@@ -32,6 +32,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "JohnDoe"
+        ],
         "service.type": "aws",
         "source.address": "192.0.2.110",
         "source.ip": "192.0.2.110",
@@ -82,6 +85,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 658,
+        "related.user": [
+            "JaneDoe"
+        ],
         "service.type": "aws",
         "source.address": "192.0.2.100",
         "source.ip": "192.0.2.100",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-access-key-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-access-key-json.log-expected.json
@@ -38,6 +38,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -48,6 +49,7 @@
         ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "signin.amazonaws.com"

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-group-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-group-json.log-expected.json
@@ -36,8 +36,13 @@
             "creation"
         ],
         "fileset.name": "cloudtrail",
+        "group.id": "EXAMPLE_ID",
+        "group.name": "TEST-GROUP",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
@@ -80,8 +85,12 @@
             "creation"
         ],
         "fileset.name": "cloudtrail",
+        "group.name": "TEST-GROUP",
         "input.type": "log",
         "log.offset": 903,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-key-pair-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-key-pair-json.log-expected.json
@@ -32,6 +32,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "72.21.198.64",
         "source.as.number": 16509,

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-trail-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-trail-json.log-expected.json
@@ -41,6 +41,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-user-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-user-json.log-expected.json
@@ -33,6 +33,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -43,6 +44,8 @@
         ],
         "user.id": "EX_PRINCIPAL_ID",
         "user.name": "Alice",
+        "user.target.id": "EXAMPLEUSERID",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "aws-cli",
         "user_agent.original": "aws-cli/1.3.2 Python/2.7.5 Windows/7",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-virtual-mfa-device-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-virtual-mfa-device-json.log-expected.json
@@ -34,6 +34,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/deactivate-mfa-device-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/deactivate-mfa-device-json.log-expected.json
@@ -44,6 +44,7 @@
         ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
+        "user.target.name": "Alice",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "signin.amazonaws.com"

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-access-key-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-access-key-json.log-expected.json
@@ -34,6 +34,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -44,6 +45,7 @@
         ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "signin.amazonaws.com"

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-group-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-group-json.log-expected.json
@@ -30,8 +30,12 @@
             "deletion"
         ],
         "fileset.name": "cloudtrail",
+        "group.name": "TEST-GROUP",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
@@ -74,8 +78,12 @@
             "deletion"
         ],
         "fileset.name": "cloudtrail",
+        "group.name": "TEST-GROUP",
         "input.type": "log",
         "log.offset": 747,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-ssh-public-key-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-ssh-public-key-json.log-expected.json
@@ -34,6 +34,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -44,6 +45,7 @@
         ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "signin.amazonaws.com"

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-trail-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-trail-json.log-expected.json
@@ -24,6 +24,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-user-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-user-json.log-expected.json
@@ -33,6 +33,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -43,6 +44,7 @@
         ],
         "user.id": "EX_PRINCIPAL_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "signin.amazonaws.com"

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-virtual-mfa-device-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-virtual-mfa-device-json.log-expected.json
@@ -32,6 +32,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

diff --git a/...lebeat/module/aws/cloudtrail/test/describe_configuration_recorders-json.log-expected.json b/...lebeat/module/aws/cloudtrail/test/describe_configuration_recorders-json.log-expected.json
@@ -25,6 +25,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "REDACTED"
+        ],
         "service.type": "aws",
         "source.address": "REDACTED",
         "tags": [

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/enable-mfa-device-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/enable-mfa-device-json.log-expected.json
@@ -33,6 +33,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -43,6 +44,7 @@
         ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "console.amazonaws.com"