[Filebeat] Add ECS tls fields to zeek and aws modules (#15936) (#16043)

* Add ECS tls fields to zeek and aws modules - zeek/smtp + tls.established - zeek/rdp + tls.established - zeek/ssl + tls.cipher + tls.curve + tls.client.issuer + tls.established + tls.resumed + tls.server.issuer + tls.version + tls.version_protocol - aws/elb + tls.cipher + tls.version + tls.version_protocol - aws/s3access + tls.cipher + tls.version + tls.version_protocol Closes #15757 (cherry picked from commit b6baae0)
elastic · Feb 5, 2020 · 064f4ee · 064f4ee
1 parent 960a877
commit 064f4ee
Show file tree

Hide file tree

Showing 17 changed files with 388 additions and 399 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -176,6 +176,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove Beta label from google-pubsub input. {issue}13346[13346] {pull}14715[14715]
 - Set event.outcome field based on googlecloud audit log output. {pull}15731[15731]
 - Add dashboard for AWS vpcflow fileset. {pull}16007[16007]
+- Add ECS tls fields to zeek:smtp,rdp,ssl and aws:s3access,elb {issue}15757[15757] {pull}15935[15936]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml
@@ -139,6 +139,26 @@ processors:
       target_field: source.as.organization.name
       ignore_missing: true
 
+  - set:
+      field: tls.cipher
+      value: '{{aws.elb.ssl_cipher}}'
+      if: ctx.aws?.elb?.ssl_cipher != null
+
+  - script:
+      lang: painless
+      if: ctx.aws?.elb?.ssl_protocol != null
+      source: >-
+        def parts = ctx.aws.elb.ssl_protocol.splitOnToken("v");
+        if (parts.length != 2) {
+          return;
+        }
+        if (parts[1].contains(".")) {
+          ctx.tls.version = parts[1];
+        } else {
+          ctx.tls.version = parts[1].substring(0,1) + "." + parts[1].substring(1);
+        }
+        ctx.tls.version_protocol = parts[0].toLowerCase();
+
   - remove:
       field:
         - message

diff --git a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json
@@ -72,6 +72,9 @@
         "service.type": "aws",
         "source.ip": "192.168.131.39",
         "source.port": "2817",
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls",
         "user_agent.original": "curl/7.46.0"
     },
     {
@@ -110,6 +113,9 @@
         "service.type": "aws",
         "source.ip": "10.0.1.252",
         "source.port": "48160",
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls",
         "user_agent.original": "curl/7.46.0"
     },
     {
@@ -174,6 +180,9 @@
         "service.type": "aws",
         "source.ip": "10.0.0.140",
         "source.port": "44244",
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls",
         "user_agent.original": "-"
     },
     {

diff --git a/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json
@@ -26,6 +26,9 @@
         "service.type": "aws",
         "source.ip": "192.168.131.39",
         "source.port": "2817",
+        "tls.cipher": "DHE-RSA-AES128-SHA",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls",
         "user_agent.original": "curl/7.38.0"
     }
 ]
diff --git a/x-pack/filebeat/module/aws/elb/test/example-nlb-tcp.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-nlb-tcp.log-expected.json
@@ -32,6 +32,9 @@
         "source.geo.region_iso_code": "US-VA",
         "source.geo.region_name": "Virginia",
         "source.ip": "72.21.218.154",
-        "source.port": "51341"
+        "source.port": "51341",
+        "tls.cipher": "ECDHE-RSA-AES128-SHA",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls"
     }
 ]
diff --git a/x-pack/filebeat/module/aws/elb/test/example-ssl.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-ssl.log-expected.json
@@ -20,6 +20,9 @@
         "service.type": "aws",
         "source.bytes": 57,
         "source.ip": "192.168.131.39",
-        "source.port": "2817"
+        "source.port": "2817",
+        "tls.cipher": "ECDHE-ECDSA-AES128-GCM-SHA256",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls"
     }
 ]
diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml
@@ -30,6 +30,21 @@ processors:
       ignore_failure: true
       formats:
         - "dd/MMM/yyyy:H:m:s Z"
+  - set:
+      field: tls.cipher
+      value: '{{aws.s3access.cipher_suite}}'
+      if: ctx.aws?.s3access?.cipher_suite != null
+
+  - script:
+      lang: painless
+      if: ctx.aws?.s3access?.tls_version != null
+      source: >-
+        def parts = ctx.aws.s3access.tls_version.toLowerCase().splitOnToken("v");
+        if (parts.length != 2) {
+          return;
+        }
+        ctx.tls.version = parts[1];
+        ctx.tls.version_protocol = parts[0]
 
   #
   # Remove temporary fields

diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json
@@ -24,7 +24,10 @@
         "fileset.name": "s3access",
         "input.type": "log",
         "log.offset": 0,
-        "service.type": "aws"
+        "service.type": "aws",
+        "tls.cipher": "ECDHE-RSA-AES128-SHA",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls"
     },
     {
         "@timestamp": "2019-08-01T00:24:42.000Z",
@@ -51,7 +54,10 @@
         "fileset.name": "s3access",
         "input.type": "log",
         "log.offset": 715,
-        "service.type": "aws"
+        "service.type": "aws",
+        "tls.cipher": "ECDHE-RSA-AES128-SHA",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls"
     },
     {
         "@timestamp": "2019-08-01T00:24:43.000Z",
@@ -79,7 +85,10 @@
         "fileset.name": "s3access",
         "input.type": "log",
         "log.offset": 1429,
-        "service.type": "aws"
+        "service.type": "aws",
+        "tls.cipher": "ECDHE-RSA-AES128-SHA",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls"
     },
     {
         "@timestamp": "2019-08-01T00:24:43.000Z",
@@ -106,7 +115,10 @@
         "fileset.name": "s3access",
         "input.type": "log",
         "log.offset": 2161,
-        "service.type": "aws"
+        "service.type": "aws",
+        "tls.cipher": "ECDHE-RSA-AES128-SHA",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls"
     },
     {
         "@timestamp": "2019-09-10T15:11:07.000Z",
@@ -130,7 +142,10 @@
         "fileset.name": "s3access",
         "input.type": "log",
         "log.offset": 2875,
-        "service.type": "aws"
+        "service.type": "aws",
+        "tls.cipher": "ECDHE-RSA-AES128-SHA",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls"
     },
     {
         "@timestamp": "2019-09-19T17:06:39.000Z",
@@ -154,6 +169,9 @@
         "fileset.name": "s3access",
         "input.type": "log",
         "log.offset": 3280,
-        "service.type": "aws"
+        "service.type": "aws",
+        "tls.cipher": "ECDHE-RSA-AES128-SHA",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls"
     }
 ]
diff --git a/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json
@@ -24,7 +24,10 @@
         "fileset.name": "s3access",
         "input.type": "log",
         "log.offset": 0,
-        "service.type": "aws"
+        "service.type": "aws",
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.1",
+        "tls.version_protocol": "tls"
     },
     {
         "@timestamp": "2019-02-06T00:00:38.000Z",
@@ -51,7 +54,10 @@
         "fileset.name": "s3access",
         "input.type": "log",
         "log.offset": 471,
-        "service.type": "aws"
+        "service.type": "aws",
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.1",
+        "tls.version_protocol": "tls"
     },
     {
         "@timestamp": "2019-02-06T00:00:38.000Z",
@@ -79,7 +85,10 @@
         "fileset.name": "s3access",
         "input.type": "log",
         "log.offset": 944,
-        "service.type": "aws"
+        "service.type": "aws",
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.1",
+        "tls.version_protocol": "tls"
     },
     {
         "@timestamp": "2019-02-06T00:01:00.000Z",
@@ -106,7 +115,10 @@
         "fileset.name": "s3access",
         "input.type": "log",
         "log.offset": 1431,
-        "service.type": "aws"
+        "service.type": "aws",
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.1",
+        "tls.version_protocol": "tls"
     },
     {
         "@timestamp": "2019-02-06T00:01:57.000Z",
@@ -135,6 +147,9 @@
         "fileset.name": "s3access",
         "input.type": "log",
         "log.offset": 1903,
-        "service.type": "aws"
+        "service.type": "aws",
+        "tls.cipher": "ECDHE-RSA-AES128-SHA",
+        "tls.version": "1.1",
+        "tls.version_protocol": "tls"
     }
 ]
diff --git a/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.json b/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.json
@@ -25,6 +25,14 @@
         "if": "ctx.zeek.session_id != null"
       }
     },
+    {
+      "convert": {
+        "field": "zeek.rdp.ssl",
+        "target_field": "tls.established",
+        "type": "boolean",
+        "ignore_missing": true
+      }
+    },
     {
       "set": {
         "field": "source.ip",

diff --git a/x-pack/filebeat/module/zeek/rdp/test/rdp-json.log-expected.json b/x-pack/filebeat/module/zeek/rdp/test/rdp-json.log-expected.json
@@ -20,6 +20,7 @@
         "tags": [
             "zeek.rdp"
         ],
+        "tls.established": true,
         "zeek.rdp.cert.count": 0,
         "zeek.rdp.result": "encrypted",
         "zeek.rdp.security_protocol": "HYBRID",

diff --git a/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.json b/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.json
@@ -37,6 +37,14 @@
         "value": "{{destination.address}}"
       }
     },
+    {
+      "convert": {
+        "field": "zeek.smtp.tls",
+        "target_field": "tls.established",
+        "type": "boolean",
+        "ignore_missing": true
+      }
+    },
     {
       "date": {
         "field": "zeek.smtp.date",

diff --git a/x-pack/filebeat/module/zeek/smtp/test/smtp-json.log-expected.json b/x-pack/filebeat/module/zeek/smtp/test/smtp-json.log-expected.json
@@ -20,6 +20,7 @@
         "tags": [
             "zeek.smtp"
         ],
+        "tls.established": true,
         "zeek.session_id": "CWWzPB3RjqhFf528c",
         "zeek.smtp.fuids": [],
         "zeek.smtp.helo": "EXAMPLE.COM",