Cherry-pick #18915 to 7.x: [Filebeat] Fix improper nesting of session_issuer in aws/cloudtrail

CHANGELOG.next.asciidoc

@@ -182,6 +182,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed ingestion of some Cisco ASA and FTD messages when a hostname was used instead of an IP for NAT fields. {issue}14034[14034] {pull}18376[18376]
 - Fix `o365.audit` failing to ingest events when ip address is surrounded by square brackets. {issue}18587[18587] {pull}18591[18591]
 - Fix Kubernetes Watcher goroutine leaks when input config is invalid and `input.reload` is enabled. {issue}18629[18629] {pull}18630[18630]
+- Fix improper nesting of session_issuer object in aws cloudtrail fileset. {issue}18894[18894] {pull}18915[18915]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -1149,22 +1149,13 @@ type: date
 
 --
 
-*`aws.cloudtrail.user_identity.invoked_by`*::
-+
---
-The name of the AWS service that made the request, such as Amazon EC2 Auto Scaling or AWS Elastic Beanstalk.
-
-type: keyword
-
---
-
 [float]
 === session_issuer
 
 If the request was made with temporary security credentials, an element that provides information about how the credentials were obtained.
 
 
-*`aws.cloudtrail.user_identity.session_issuer.type`*::
+*`aws.cloudtrail.user_identity.session_context.session_issuer.type`*::
 +
 --
 The source of the temporary security credentials, such as Root, IAMUser, or Role.
@@ -1173,7 +1164,7 @@ type: keyword
 
 --
 
-*`aws.cloudtrail.user_identity.session_issuer.principal_id`*::
+*`aws.cloudtrail.user_identity.session_context.session_issuer.principal_id`*::
 +
 --
 The internal ID of the entity that was used to get credentials.
@@ -1182,7 +1173,7 @@ type: keyword
 
 --
 
-*`aws.cloudtrail.user_identity.session_issuer.arn`*::
+*`aws.cloudtrail.user_identity.session_context.session_issuer.arn`*::
 +
 --
 The ARN of the source (account, IAM user, or role) that was used to get temporary security credentials.
@@ -1191,7 +1182,7 @@ type: keyword
 
 --
 
-*`aws.cloudtrail.user_identity.session_issuer.account_id`*::
+*`aws.cloudtrail.user_identity.session_context.session_issuer.account_id`*::
 +
 --
 The account that owns the entity that was used to get credentials.
@@ -1200,6 +1191,15 @@ type: keyword
 
 --
 
+*`aws.cloudtrail.user_identity.invoked_by`*::
++
+--
+The name of the AWS service that made the request, such as Amazon EC2 Auto Scaling or AWS Elastic Beanstalk.
+
+type: keyword
+
+--
+
 *`aws.cloudtrail.error_code`*::
 +
 --

x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml

@@ -46,38 +46,38 @@
               type: date
               description: >-
                 The date and time when the temporary security credentials were issued.
+            - name: session_issuer
+              type: group
+              description: >-
+                If the request was made with temporary security
+                credentials, an element that provides information about
+                how the credentials were obtained.
+              fields:
+                - name: type
+                  type: keyword
+                  description: >-
+                    The source of the temporary security credentials, such
+                    as Root, IAMUser, or Role.
+                - name: principal_id
+                  type: keyword
+                  description: >-
+                    The internal ID of the entity that was used to get
+                    credentials.
+                - name: arn
+                  type: keyword
+                  description: >-
+                    The ARN of the source (account, IAM user, or role)
+                    that was used to get temporary security credentials.
+                - name: account_id
+                  type: keyword
+                  description: >-
+                    The account that owns the entity that was used to get
+                    credentials.
         - name: invoked_by
           type: keyword
           description: >-
             The name of the AWS service that made the request, such as
             Amazon EC2 Auto Scaling or AWS Elastic Beanstalk.
-        - name: session_issuer
-          type: group
-          description: >-
-            If the request was made with temporary security
-            credentials, an element that provides information about
-            how the credentials were obtained.
-          fields:
-            - name: type
-              type: keyword
-              description: >-
-                The source of the temporary security credentials, such
-                as Root, IAMUser, or Role.
-            - name: principal_id
-              type: keyword
-              description: >-
-                The internal ID of the entity that was used to get
-                credentials.
-            - name: arn
-              type: keyword
-              description: >-
-                The ARN of the source (account, IAM user, or role)
-                that was used to get temporary security credentials.
-            - name: account_id
-              type: keyword
-              description: >-
-                The account that owns the entity that was used to get
-                credentials.
     - name: error_code
       type: keyword
       description: >-

x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml

@@ -52,29 +52,29 @@ processors:
       formats:
         - ISO8601
   - rename:
-      field: "json.userIdentity.invokedBy"
-      target_field: "aws.cloudtrail.user_identity.invoked_by"
-      ignore_failure: true
-  - rename:
-      field: "json.userIdentity.sessionIssuer.type"
-      target_field: "aws.cloudtrail.user_identity.session_issuer.type"
+      field: "json.userIdentity.sessionContext.sessionIssuer.type"
+      target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.type"
       ignore_failure: true
 # userIdentity.sessionIssuer.userName is only set with assumed roles.
   - rename:
-      field: "json.userIdentity.sessionIssuer.userName"
+      field: "json.userIdentity.sessionContext.sessionIssuer.userName"
       target_field: "user.name"
       ignore_failure: true
   - rename:
-      field: "json.userIdentity.sessionIssuer.principalId"
-      target_field: "aws.cloudtrail.user_identity.session_issuer.principal_id"
+      field: "json.userIdentity.sessionContext.sessionIssuer.principalId"
+      target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id"
       ignore_failure: true
   - rename:
-      field: "json.userIdentity.sessionIssuer.arn"
-      target_field: "aws.cloudtrail.user_identity.session_issuer.arn"
+      field: "json.userIdentity.sessionContext.sessionIssuer.arn"
+      target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.arn"
       ignore_failure: true
   - rename:
-      field: "json.userIdentity.sessionIssuer.accountId"
-      target_field: "aws.cloudtrail.user_identity.session_issuer.account_id"
+      field: "json.userIdentity.sessionContext.sessionIssuer.accountId"
+      target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.account_id"
+      ignore_failure: true
+  - rename:
+      field: "json.userIdentity.invokedBy"
+      target_field: "aws.cloudtrail.user_identity.invoked_by"
       ignore_failure: true
   - rename:
       field: "json.eventSource"

x-pack/filebeat/module/aws/cloudtrail/test/assume-role-json.log-expected.json

@@ -10,6 +10,10 @@
         "aws.cloudtrail.user_identity.arn": "arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1",
         "aws.cloudtrail.user_identity.session_context.creation_date": "2019-10-02T21:50:54.000Z",
         "aws.cloudtrail.user_identity.session_context.mfa_authenticated": "false",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.account_id": "111111111111",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.arn": "arn:aws:iam::111111111111:role/JohnRole1",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id": "AROAIN5ATK5U7KEXAMPLE",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.type": "Role",
         "aws.cloudtrail.user_identity.type": "AssumedRole",
         "cloud.account.id": "111111111111",
         "cloud.region": "us-east-2",
@@ -35,6 +39,7 @@
         "source.geo.region_name": "Chongqing",
         "source.ip": "123.145.67.89",
         "user.id": "AROAIN5ATK5U7KEXAMPLE:JohnRole1",
+        "user.name": "JohnDoe",
         "user_agent.device.name": "Spider",
         "user_agent.name": "aws-cli",
         "user_agent.original": "aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239",

x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log

@@ -1,3 +1,3 @@
 {"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JohnDoe","accountId":"111122223333","userName":"JohnDoe"},"eventTime":"2014-07-16T15:49:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.110","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","requestParameters":null,"responseElements":{"ConsoleLogin":"Success"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/s3/","MFAUsed":"No"},"eventID":"3fcfb182-98f8-4744-bd45-10aEXAMPLE"}
 {"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JaneDoe","accountId":"111122223333","userName":"JaneDoe"},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"}
-{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName","arn":"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName","accountId":"123456789012","accessKeyId":"AKIAIOSFODNN7EXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"20131102T010628Z"}},"sessionIssuer":{"type":"Role","principalId":"AROAIDPPEZS35WEXAMPLE","arn":"arn:aws:iam::123456789012:role/RoleToBeAssumed","accountId":"123456789012","userName":"RoleToBeAssumed"}},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"}
+{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName","arn":"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName","accountId":"123456789012","accessKeyId":"AKIAIOSFODNN7EXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"20131102T010628Z"},"sessionIssuer":{"type":"Role","principalId":"AROAIDPPEZS35WEXAMPLE","arn":"arn:aws:iam::123456789012:role/RoleToBeAssumed","accountId":"123456789012","userName":"RoleToBeAssumed"}}},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"}

x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json

@@ -88,10 +88,10 @@
         "aws.cloudtrail.user_identity.access_key_id": "AKIAIOSFODNN7EXAMPLE",
         "aws.cloudtrail.user_identity.arn": "arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName",
         "aws.cloudtrail.user_identity.session_context.mfa_authenticated": "false",
-        "aws.cloudtrail.user_identity.session_issuer.account_id": "123456789012",
-        "aws.cloudtrail.user_identity.session_issuer.arn": "arn:aws:iam::123456789012:role/RoleToBeAssumed",
-        "aws.cloudtrail.user_identity.session_issuer.principal_id": "AROAIDPPEZS35WEXAMPLE",
-        "aws.cloudtrail.user_identity.session_issuer.type": "Role",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.account_id": "123456789012",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.arn": "arn:aws:iam::123456789012:role/RoleToBeAssumed",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id": "AROAIDPPEZS35WEXAMPLE",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.type": "Role",
         "aws.cloudtrail.user_identity.type": "AssumedRole",
         "cloud.account.id": "123456789012",
         "cloud.region": "us-east-2",
@@ -101,7 +101,7 @@
         "event.id": "11ea990b-4678-4bcd-8fbe-625EXAMPLE",
         "event.kind": "event",
         "event.module": "aws",
-        "event.original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName\",\"accountId\":\"123456789012\",\"accessKeyId\":\"AKIAIOSFODNN7EXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"20131102T010628Z\"}},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE\",\"arn\":\"arn:aws:iam::123456789012:role/RoleToBeAssumed\",\"accountId\":\"123456789012\",\"userName\":\"RoleToBeAssumed\"}},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}",
+        "event.original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName\",\"accountId\":\"123456789012\",\"accessKeyId\":\"AKIAIOSFODNN7EXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"20131102T010628Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE\",\"arn\":\"arn:aws:iam::123456789012:role/RoleToBeAssumed\",\"accountId\":\"123456789012\",\"userName\":\"RoleToBeAssumed\"}}},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}",
         "event.outcome": "failure",
         "event.provider": "signin.amazonaws.com",
         "event.type": "info",

x-pack/filebeat/module/aws/cloudtrail/test/delete-bucket-json.log-expected.json

@@ -9,6 +9,10 @@
         "aws.cloudtrail.user_identity.arn": "arn:aws:sts::777788889999:assumed-role/AssumeNothing/devdsk",
         "aws.cloudtrail.user_identity.session_context.creation_date": "2016-11-14T17:25:26.000Z",
         "aws.cloudtrail.user_identity.session_context.mfa_authenticated": "false",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.account_id": "777788889999",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.arn": "arn:aws:iam::777788889999:role/AssumeNothing",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id": "AIDAQRSTUVWXYZEXAMPLE",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.type": "Role",
         "aws.cloudtrail.user_identity.type": "AssumedRole",
         "cloud.account.id": "777788889999",
         "cloud.region": "us-east-2",
@@ -28,6 +32,7 @@
         "source.address": "192.0.2.1",
         "source.ip": "192.0.2.1",
         "user.id": "AIDAQRSTUVWXYZEXAMPLE:devdsk",
+        "user.name": "AssumeNothing",
         "user_agent.device.name": "Spider",
         "user_agent.name": "aws-cli",
         "user_agent.original": "[aws-cli/1.11.10 Python/2.7.8 Linux/3.2.45-0.6.wd.865.49.315.metal1.x86_64 botocore/1.4.67]",