[Filebeat] AWS CloudTrail preserve eventCategory (elastic#22805)

* AWS CloudTrail preserve eventCategory - map to aws.cloudtrail.event_category Closes elastic#22776 (cherry picked from commit 971b95b)
leehinman · Jan 7, 2021 · 496feb0 · 496feb0
1 parent 45387b0
commit 496feb0
Show file tree

Hide file tree

Showing 8 changed files with 67 additions and 6 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -346,6 +346,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add experimental dataset sonicwall/firewall for Sonicwall Firewalls logs {pull}19713[19713]
 - Add experimental dataset squid/log for Squid Proxy Server logs {pull}19713[19713]
 - Add experimental dataset zscaler/zia for Zscaler Internet Access logs {pull}19713[19713]
+- Preserve AWS CloudTrail eventCategory in aws.cloudtrail.event_category. {issue}22776[22776] {pull}22805[22805]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -1410,6 +1410,19 @@ type: keyword
 
 --
 
+*`aws.cloudtrail.event_category`*::
++
+--
+Shows the event category that is used in LookupEvents calls.
+
+ - For management events, the value is management.
+ - For data events, the value is data.
+ - For Insights events, the value is insight.
+
+type: keyword
+
+--
+
 [float]
 === console_login
 

diff --git a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml
@@ -176,6 +176,14 @@
       description: >-
         Identifies the VPC endpoint in which requests were made from a
         VPC to another AWS service, such as Amazon S3.
+    - name: event_category
+      type: keyword
+      description: |-
+        Shows the event category that is used in LookupEvents calls.
+
+         - For management events, the value is management.
+         - For data events, the value is data.
+         - For Insights events, the value is insight.
     - name: console_login
       type: group
       description: >-

diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
@@ -91,11 +91,10 @@ processors:
       value: "{{json.eventName}}"
       ignore_failure: true
       ignore_empty_value: true
-  - set:
-      field: "event.action"
-      value: "{{json.eventCategory}}"
+  - rename:
+      field: "json.eventCategory"
+      target_field: "aws.cloudtrail.event_category"
       ignore_failure: true
-      ignore_empty_value: true
   - rename:
       field: "json.awsRegion"
       target_field: "cloud.region"

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/describe_configuration_recorders-json.log b/x-pack/filebeat/module/aws/cloudtrail/test/describe_configuration_recorders-json.log
@@ -0,0 +1 @@
+{"awsRegion":"us-east-1","eventCategory":"Management","eventID":"REDACTED","eventName":"DescribeConfigurationRecorders","eventSource":"config.amazonaws.com","eventTime":"2014-03-25T21:08:19Z","eventType":"AwsApiCall","eventVersion":"1.07","managementEvent":true,"readOnly":true,"recipientAccountId":"REDACTED","requestID":"REDACTED","requestParameters":null,"responseElements":null,"sourceIPAddress":"REDACTED","userAgent":"REDACTED","userIdentity":{"accessKeyId":"REDACTED","accountId":"REDACTED","arn":"arn:aws:iam::REDACTED:user/REDACTED","principalId":"REDACTED","sessionContext":{"attributes":{"creationDate":"REDACTED","mfaAuthenticated":"true"},"sessionIssuer":{},"webIdFederationData":{}},"type":"IAMUser","userName":"REDACTED"}}
diff --git a/...lebeat/module/aws/cloudtrail/test/describe_configuration_recorders-json.log-expected.json b/...lebeat/module/aws/cloudtrail/test/describe_configuration_recorders-json.log-expected.json
@@ -0,0 +1,39 @@
+[
+    {
+        "@timestamp": "2014-03-25T21:08:19.000Z",
+        "aws.cloudtrail.event_category": "Management",
+        "aws.cloudtrail.event_type": "AwsApiCall",
+        "aws.cloudtrail.event_version": "1.07",
+        "aws.cloudtrail.management_event": true,
+        "aws.cloudtrail.read_only": true,
+        "aws.cloudtrail.recipient_account_id": "REDACTED",
+        "aws.cloudtrail.user_identity.access_key_id": "REDACTED",
+        "aws.cloudtrail.user_identity.arn": "arn:aws:iam::REDACTED:user/REDACTED",
+        "aws.cloudtrail.user_identity.session_context.mfa_authenticated": "true",
+        "aws.cloudtrail.user_identity.type": "IAMUser",
+        "cloud.account.id": "REDACTED",
+        "cloud.region": "us-east-1",
+        "event.action": "DescribeConfigurationRecorders",
+        "event.dataset": "aws.cloudtrail",
+        "event.id": "REDACTED",
+        "event.kind": "event",
+        "event.module": "aws",
+        "event.original": "{\"awsRegion\":\"us-east-1\",\"eventCategory\":\"Management\",\"eventID\":\"REDACTED\",\"eventName\":\"DescribeConfigurationRecorders\",\"eventSource\":\"config.amazonaws.com\",\"eventTime\":\"2014-03-25T21:08:19Z\",\"eventType\":\"AwsApiCall\",\"eventVersion\":\"1.07\",\"managementEvent\":true,\"readOnly\":true,\"recipientAccountId\":\"REDACTED\",\"requestID\":\"REDACTED\",\"requestParameters\":null,\"responseElements\":null,\"sourceIPAddress\":\"REDACTED\",\"userAgent\":\"REDACTED\",\"userIdentity\":{\"accessKeyId\":\"REDACTED\",\"accountId\":\"REDACTED\",\"arn\":\"arn:aws:iam::REDACTED:user/REDACTED\",\"principalId\":\"REDACTED\",\"sessionContext\":{\"attributes\":{\"creationDate\":\"REDACTED\",\"mfaAuthenticated\":\"true\"},\"sessionIssuer\":{},\"webIdFederationData\":{}},\"type\":\"IAMUser\",\"userName\":\"REDACTED\"}}",
+        "event.outcome": "success",
+        "event.provider": "config.amazonaws.com",
+        "event.type": "info",
+        "fileset.name": "cloudtrail",
+        "input.type": "log",
+        "log.offset": 0,
+        "service.type": "aws",
+        "source.address": "REDACTED",
+        "tags": [
+            "forwarded"
+        ],
+        "user.id": "REDACTED",
+        "user.name": "REDACTED",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Other",
+        "user_agent.original": "REDACTED"
+    }
+]
diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json
@@ -1,6 +1,7 @@
 [
     {
         "@timestamp": "2020-09-09T23:00:00.000Z",
+        "aws.cloudtrail.event_category": "Insight",
         "aws.cloudtrail.event_type": "AwsCloudTrailInsight",
         "aws.cloudtrail.event_version": "1.07",
         "aws.cloudtrail.insight_details.eventName": "AttachUserPolicy",
@@ -45,7 +46,6 @@
         "aws.cloudtrail.insight_details.state": "End",
         "aws.cloudtrail.recipient_account_id": "123456789012",
         "cloud.region": "us-east-1",
-        "event.action": "Insight",
         "event.dataset": "aws.cloudtrail",
         "event.id": "41ed77ca-d659-b45a-8e9a-74e504300007",
         "event.kind": "event",

diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go