From 9b783d340c5c3c24eef08d3663e17e7fe86e8cfe Mon Sep 17 00:00:00 2001
From: Lee Hinman <57081003+leehinman@users.noreply.github.com>
Date: Wed, 30 Sep 2020 13:09:57 -0500
Subject: [PATCH] Add field limit check for AWS Cloudtrail flattened fields
 (#21388)

add 32k length check for
  - aws.cloudtrail.flattened.request_parameters
  - aws.cloudtrail.flattened.response_elements
  - aws.cloudtrail.flattened.additional_eventdata
  - aws.cloudtrail.flattened.service_event_details

Closes #21382

(cherry picked from commit eae9f5c156467f2f23df5afc4e73b63c01e3091b)
---
 CHANGELOG.next.asciidoc                       |  1 +
 .../module/aws/cloudtrail/ingest/pipeline.yml | 62 ++++++++-----------
 2 files changed, 26 insertions(+), 37 deletions(-)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index d85eaf585d4..eb46131cfba 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -70,6 +70,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed `cloudfoundry.access` to have the correct `cloudfoundry.app.id` contents. {pull}17847[17847]
 - Fixing `ingress_controller.` fields to be of type keyword instead of text. {issue}17834[17834]
 - Fixed typo in log message. {pull}17897[17897]
+- Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382]
 
 *Heartbeat*
 
diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
index 3fd015d0252..9cc35e42d7f 100644
--- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
+++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
@@ -146,37 +146,36 @@ processors:
       field: "json.errorMessage"
       target_field: "aws.cloudtrail.error_message"
       ignore_failure: true
-  - rename:
-      field: json.requestParameters
-      target_field: "aws.cloudtrail.flattened.request_parameters"
-      if: ctx?.json?.requestParameters != null
   - script:
       lang: painless
       source: |
-        if (ctx.aws.cloudtrail.flattened.request_parameters != null) {
-          ctx.aws.cloudtrail.request_parameters = ctx.aws.cloudtrail.flattened.request_parameters.toString();
+        if (ctx.aws.cloudtrail?.flattened == null) {
+            Map map = new HashMap();
+            ctx.aws.cloudtrail.put("flattened", map);
+          }
+        if (ctx.json.requestParameters != null) {
+          ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
+          if (ctx.aws.cloudtrail.request_parameters.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("request_parameters", ctx.json.requestParameters);
+          }
         }
-      ignore_failure: true
-  - rename:
-      field: json.responseElements
-      target_field: "aws.cloudtrail.flattened.response_elements"
-      if: ctx?.json?.responseElements != null
-  - script:
-      lang: painless
-      source: |
-        if (ctx.aws.cloudtrail.flattened.response_elements != null) {
-          ctx.aws.cloudtrail.response_elements = ctx.aws.cloudtrail.flattened.response_elements.toString();
+        if (ctx.json.responseElements != null) {
+          ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
+          if (ctx.aws.cloudtrail.response_elements.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("response_elements", ctx.json.responseElements);
+          }
         }
-      ignore_failure: true
-  - rename:
-      field: json.additionalEventData
-      target_field: "aws.cloudtrail.flattened.additional_eventdata"
-      if: ctx?.json?.additionalEventData != null
-  - script:
-      lang: painless
-      source: |
-        if (ctx.aws.cloudtrail.flattened.additional_eventdata != null) {
-          ctx.aws.cloudtrail.additional_eventdata = ctx.aws.cloudtrail.flattened.additional_eventdata.toString();
+        if (ctx.json.additionalEventData != null) {
+          ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
+          if (ctx.aws.cloudtrail.additional_eventdata.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("additional_eventdata", ctx.json.additionalEventData);
+          }
+        }
+        if (ctx.json.serviceEventDetails != null) {
+          ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
+          if (ctx.aws.cloudtrail.service_event_details.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("service_event_details", ctx.json.serviceEventDetails);
+          }
         }
       ignore_failure: true
   - rename:
@@ -219,17 +218,6 @@ processors:
       field: "json.recipientAccountId"
       target_field: "aws.cloudtrail.recipient_account_id"
       ignore_failure: true
-  - rename:
-      field: json.serviceEventDetails
-      target_field: "aws.cloudtrail.flattened.service_event_details"
-      if: ctx?.json?.serviceEventDetails != null
-  - script:
-      lang: painless
-      source: |
-        if (ctx.aws.cloudtrail.flattened.service_event_details != null) {
-          ctx.aws.cloudtrail.service_event_details = ctx.aws.cloudtrail.flattened.service_event_details.toString();
-        }
-      ignore_failure: true
   - rename:
       field: "json.sharedEventId"
       target_field: "aws.cloudtrail.shared_event_id"