elastic · leehinman · Jul 2, 2020 · Jun 29, 2020 · Jun 30, 2020 · Jun 30, 2020
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -199,6 +199,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix date and timestamp formats for fortigate module {pull}19316[19316]
 - Fix memory leak in tcp and unix input sources. {pull}19459[19459]
 - Add missing `default_field: false` to aws filesets fields.yml. {pull}19568[19568]
+- Fix tls mapping in suricata module {issue}19492[19492] {pull}19494[19494]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -34596,6 +34596,36 @@ type: keyword
 
 --
 
+
+*`suricata.eve.tls.ja3s.string`*::
++
+--
+type: keyword
+
+--
+
+*`suricata.eve.tls.ja3s.hash`*::
++
+--
+type: keyword
+
+--
+
+
+*`suricata.eve.tls.ja3.string`*::
++
+--
+type: keyword
+
+--
+
+*`suricata.eve.tls.ja3.hash`*::
++
+--
+type: keyword
+
+--
+
 *`suricata.eve.app_proto_ts`*::
 +
 --

diff --git a/x-pack/filebeat/module/suricata/eve/_meta/fields.yml b/x-pack/filebeat/module/suricata/eve/_meta/fields.yml
@@ -665,6 +665,24 @@
       - name: subject
         type: keyword
 
+      - name: ja3s
+        type: group
+        default_field: false
+        fields:
+          - name: string
+            type: keyword
+          - name: hash
+            type: keyword
+
+      - name: ja3
+        type: group
+        default_field: false
+        fields:
+          - name: string
+            type: keyword
+          - name: hash
+            type: keyword
+
     - name: app_proto_ts
       type: keyword
 

diff --git a/x-pack/filebeat/module/suricata/eve/config/eve.yml b/x-pack/filebeat/module/suricata/eve/config/eve.yml
@@ -373,26 +373,23 @@ processors:
           addTlsVersion(evt);
           cleanupTlsSni(evt);
         }
-  - if:
-      equals:
-        suricata.eve.event_type: tls
-    then:
-      - convert:
-          ignore_missing: true
-          ignore_failure: true
-          mode: copy
-          fields:
-            - {from: suricata.eve.tls.subject, to: tls.server.subject}
-            - {from: suricata.eve.tls.issuerdn, to: tls.server.issuer}
-            - {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean}
-            - {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1}
-            - {from: suricata.eve.tls.sni, to: tls.client.server_name}
-            - {from: suricata.eve.tls.sni, to: destination.domain}
-            - {from: suricata.eve.tls.notbefore, to: tls.server.not_before}
-            - {from: suricata.eve.tls.notafter, to: tls.server.not_after}
-            - {from: suricata.eve.tls.ja3s, to: tls.server.ja3s}
-            - {from: suricata.eve.tls.certificate, to: tls.server.certificate}
-            - {from: suricata.eve.tls.chain, to: tls.server.certificate_chain}
+  - convert:
+      ignore_missing: true
+      fail_on_error: false
+      mode: copy
+      fields:
+        - {from: suricata.eve.tls.subject, to: tls.server.subject}
+        - {from: suricata.eve.tls.issuerdn, to: tls.server.issuer}
+        - {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean}
+        - {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1}
+        - {from: suricata.eve.tls.sni, to: tls.client.server_name}
+        - {from: suricata.eve.tls.sni, to: destination.domain}
+        - {from: suricata.eve.tls.notbefore, to: tls.server.not_before}
+        - {from: suricata.eve.tls.notafter, to: tls.server.not_after}
+        - {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s}
+        - {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3}
+        - {from: suricata.eve.tls.certificate, to: tls.server.certificate}
+        - {from: suricata.eve.tls.chain, to: tls.server.certificate_chain}
   - drop_fields:
       ignore_missing: true
       fields:

diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log
@@ -18,3 +18,5 @@
 {"timestamp":"2018-10-04T09:35:00.897009+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":7,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/binary-amd64\/by-hash\/SHA256\/5190f7afbee38b3cb32225db478fdbabd46f76eaa9c5921a13091891bf3e9bbc","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2687},"app_proto":"http","flow":{"pkts_toserver":330,"pkts_toclient":591,"bytes_toserver":23758,"bytes_toclient":884342,"start":"2018-10-04T09:34:58.926006+0000"}}
 {"timestamp":"2018-10-04T09:35:01.362208+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":8,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/i18n\/by-hash\/SHA256\/9fe539b7036e51327cd85ca5e0a4dd4eb47f69168875de2ac9842a5e36ebd4a4","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":524,"pkts_toclient":979,"bytes_toserver":36819,"bytes_toclient":1467603,"start":"2018-10-04T09:34:58.926006+0000"}}
 {"timestamp":"2018-10-04T09:35:01.575088+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":9,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/multiverse\/binary-amd64\/by-hash\/SHA256\/8ab8cb220c0e50521c589acc2bc2b43a3121210f0b035a0605972bcffd73dd16","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":575,"pkts_toclient":1079,"bytes_toserver":40452,"bytes_toclient":1618380,"start":"2018-10-04T09:34:58.926006+0000"}}
+{"tls":{"ja3s":{"string":"333,55555,66666-22","hash":"0993626a07ad09e1ce91293be7aa5721"},"ja3":{"string":"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0","hash":"d92325c876e7279f4eb8c62415e3a6b7"},"notafter":"2024-07-16T14:52:35","notbefore":"2019-07-17T14:52:35","version":"TLS 1.2","sni":"hostname.domain.net","fingerprint":"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33","serial":"00:11:22:33:44:55:66:77:88","issuerdn":"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN","subject":"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com"},"proto":"TCP","dest_port":9080,"dest_ip":"10.232.0.237","src_port":45884,"src_ip":"10.126.2.140","event_type":"tls","in_iface":"enp5s0","flow_id":1091813059495729,"timestamp":"2018-10-04T09:35:02.796615+0000"}
+{"flow":{"start":"2020-06-26T11:00:02.970011-0400","bytes_toclient":4660,"bytes_toserver":1074,"pkts_toclient":8,"pkts_toserver":7},"app_proto":"tls","tls":{"ja3s":{"string":"742,48172,30210-30","hash":"391231ba5675e42807b9e1f457b2614e"},"ja3":{"string":"718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3","hash":"3f1ea03f5822e8021b60cc3e4b233181"},"notafter":"2026-06-25T17:36:29","notbefore":"2016-06-27T17:36:29","version":"TLS 1.2","sni":"host.domain.net","fingerprint":"36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc","serial":"72:A9:2C:51","issuerdn":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown","subject":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown"},"alert":{"severity":3,"category":"","signature":"SURICATA TLS on unusual port","rev":1,"signature_id":2610003,"gid":1,"action":"allowed"},"proto":"TCP","dest_port":8443,"dest_ip":"10.128.2.48","src_port":64389,"src_ip":"10.137.3.54","event_type":"alert","in_iface":"enp0s31f6","flow_id":991192778198299,"timestamp":"2020-06-26T11:00:03.342282-0400"}
diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json
@@ -1536,5 +1536,146 @@
         "user_agent.original": "Debian APT-HTTP/1.3 (1.6.3ubuntu0.1)",
         "user_agent.os.name": "Debian",
         "user_agent.version": "1.3"
+    },
+    {
+        "@timestamp": "2018-10-04T09:35:02.796Z",
+        "destination.address": "10.232.0.237",
+        "destination.domain": "hostname.domain.net",
+        "destination.ip": "10.232.0.237",
+        "destination.port": 9080,
+        "event.category": [
+            "network"
+        ],
+        "event.dataset": "suricata.eve",
+        "event.kind": "event",
+        "event.module": "suricata",
+        "event.original": "{\"tls\":{\"ja3s\":{\"string\":\"333,55555,66666-22\",\"hash\":\"0993626a07ad09e1ce91293be7aa5721\"},\"ja3\":{\"string\":\"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0\",\"hash\":\"d92325c876e7279f4eb8c62415e3a6b7\"},\"notafter\":\"2024-07-16T14:52:35\",\"notbefore\":\"2019-07-17T14:52:35\",\"version\":\"TLS 1.2\",\"sni\":\"hostname.domain.net\",\"fingerprint\":\"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33\",\"serial\":\"00:11:22:33:44:55:66:77:88\",\"issuerdn\":\"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN\",\"subject\":\"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com\"},\"proto\":\"TCP\",\"dest_port\":9080,\"dest_ip\":\"10.232.0.237\",\"src_port\":45884,\"src_ip\":\"10.126.2.140\",\"event_type\":\"tls\",\"in_iface\":\"enp5s0\",\"flow_id\":1091813059495729,\"timestamp\":\"2018-10-04T09:35:02.796615+0000\"}",
+        "event.type": [
+            "protocol"
+        ],
+        "fileset.name": "eve",
+        "input.type": "log",
+        "log.offset": 16546,
+        "network.community_id": "1:qsGDjYDIWp+kHhxotTdhPbUaWSo=",
+        "network.protocol": "tls",
+        "network.transport": "tcp",
+        "related.hash": [
+            "00112233445566778899AABBCCDDEEFF00112233"
+        ],
+        "related.ip": [
+            "10.126.2.140",
+            "10.232.0.237"
+        ],
+        "service.type": "suricata",
+        "source.address": "10.126.2.140",
+        "source.ip": "10.126.2.140",
+        "source.port": 45884,
+        "suricata.eve.event_type": "tls",
+        "suricata.eve.flow_id": 1091813059495729,
+        "suricata.eve.in_iface": "enp5s0",
+        "suricata.eve.tls.fingerprint": "00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33",
+        "suricata.eve.tls.issuerdn": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN",
+        "suricata.eve.tls.ja3.hash": "d92325c876e7279f4eb8c62415e3a6b7",
+        "suricata.eve.tls.ja3.string": "001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0",
+        "suricata.eve.tls.ja3s.hash": "0993626a07ad09e1ce91293be7aa5721",
+        "suricata.eve.tls.ja3s.string": "333,55555,66666-22",
+        "suricata.eve.tls.notafter": "2024-07-16T14:52:35",
+        "suricata.eve.tls.notbefore": "2019-07-17T14:52:35",
+        "suricata.eve.tls.serial": "00:11:22:33:44:55:66:77:88",
+        "suricata.eve.tls.sni": "hostname.domain.net",
+        "suricata.eve.tls.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com",
+        "suricata.eve.tls.version": "TLS 1.2",
+        "tags": [
+            "suricata"
+        ],
+        "tls.client.ja3": "d92325c876e7279f4eb8c62415e3a6b7",
+        "tls.client.server_name": "hostname.domain.net",
+        "tls.server.hash.sha1": "00112233445566778899AABBCCDDEEFF00112233",
+        "tls.server.issuer": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN",
+        "tls.server.ja3s": "0993626a07ad09e1ce91293be7aa5721",
+        "tls.server.not_after": "2024-07-16T14:52:35",
+        "tls.server.not_before": "2019-07-17T14:52:35",
+        "tls.server.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls"
+    },
+    {
+        "@timestamp": "2020-06-26T15:00:03.342Z",
+        "destination.address": "10.128.2.48",
+        "destination.bytes": 4660,
+        "destination.domain": "host.domain.net",
+        "destination.ip": "10.128.2.48",
+        "destination.packets": 8,
+        "destination.port": 8443,
+        "event.category": [
+            "network",
+            "intrusion_detection"
+        ],
+        "event.dataset": "suricata.eve",
+        "event.kind": "alert",
+        "event.module": "suricata",
+        "event.original": "{\"flow\":{\"start\":\"2020-06-26T11:00:02.970011-0400\",\"bytes_toclient\":4660,\"bytes_toserver\":1074,\"pkts_toclient\":8,\"pkts_toserver\":7},\"app_proto\":\"tls\",\"tls\":{\"ja3s\":{\"string\":\"742,48172,30210-30\",\"hash\":\"391231ba5675e42807b9e1f457b2614e\"},\"ja3\":{\"string\":\"718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3\",\"hash\":\"3f1ea03f5822e8021b60cc3e4b233181\"},\"notafter\":\"2026-06-25T17:36:29\",\"notbefore\":\"2016-06-27T17:36:29\",\"version\":\"TLS 1.2\",\"sni\":\"host.domain.net\",\"fingerprint\":\"36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc\",\"serial\":\"72:A9:2C:51\",\"issuerdn\":\"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown\",\"subject\":\"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown\"},\"alert\":{\"severity\":3,\"category\":\"\",\"signature\":\"SURICATA TLS on unusual port\",\"rev\":1,\"signature_id\":2610003,\"gid\":1,\"action\":\"allowed\"},\"proto\":\"TCP\",\"dest_port\":8443,\"dest_ip\":\"10.128.2.48\",\"src_port\":64389,\"src_ip\":\"10.137.3.54\",\"event_type\":\"alert\",\"in_iface\":\"enp0s31f6\",\"flow_id\":991192778198299,\"timestamp\":\"2020-06-26T11:00:03.342282-0400\"}",
+        "event.severity": 3,
+        "event.start": "2020-06-26T15:00:02.970Z",
+        "event.type": [
+            "allowed"
+        ],
+        "fileset.name": "eve",
+        "input.type": "log",
+        "log.offset": 17606,
+        "message": "",
+        "network.bytes": 5734,
+        "network.community_id": "1:W6fjhboFUwyEchJ3ELaqSBzDEJE=",
+        "network.packets": 15,
+        "network.protocol": "tls",
+        "network.transport": "tcp",
+        "related.hash": [
+            "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC"
+        ],
+        "related.ip": [
+            "10.137.3.54",
+            "10.128.2.48"
+        ],
+        "rule.id": "2610003",
+        "rule.name": "SURICATA TLS on unusual port",
+        "service.type": "suricata",
+        "source.address": "10.137.3.54",
+        "source.bytes": 1074,
+        "source.ip": "10.137.3.54",
+        "source.packets": 7,
+        "source.port": 64389,
+        "suricata.eve.alert.category": "",
+        "suricata.eve.alert.gid": 1,
+        "suricata.eve.alert.rev": 1,
+        "suricata.eve.alert.signature": "SURICATA TLS on unusual port",
+        "suricata.eve.alert.signature_id": 2610003,
+        "suricata.eve.event_type": "alert",
+        "suricata.eve.flow_id": 991192778198299,
+        "suricata.eve.in_iface": "enp0s31f6",
+        "suricata.eve.tls.fingerprint": "36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc",
+        "suricata.eve.tls.issuerdn": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
+        "suricata.eve.tls.ja3.hash": "3f1ea03f5822e8021b60cc3e4b233181",
+        "suricata.eve.tls.ja3.string": "718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3",
+        "suricata.eve.tls.ja3s.hash": "391231ba5675e42807b9e1f457b2614e",
+        "suricata.eve.tls.ja3s.string": "742,48172,30210-30",
+        "suricata.eve.tls.notafter": "2026-06-25T17:36:29",
+        "suricata.eve.tls.notbefore": "2016-06-27T17:36:29",
+        "suricata.eve.tls.serial": "72:A9:2C:51",
+        "suricata.eve.tls.sni": "host.domain.net",
+        "suricata.eve.tls.subject": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
+        "suricata.eve.tls.version": "TLS 1.2",
+        "tags": [
+            "suricata"
+        ],
+        "tls.client.ja3": "3f1ea03f5822e8021b60cc3e4b233181",
+        "tls.client.server_name": "host.domain.net",
+        "tls.server.hash.sha1": "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC",
+        "tls.server.issuer": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
+        "tls.server.ja3s": "391231ba5675e42807b9e1f457b2614e",
+        "tls.server.not_after": "2026-06-25T17:36:29",
+        "tls.server.not_before": "2016-06-27T17:36:29",
+        "tls.server.subject": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls"
     }
 ]
diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json
@@ -43,6 +43,7 @@
         "@timestamp": "2018-07-05T19:07:20.910Z",
         "destination.address": "192.168.156.70",
         "destination.bytes": 343,
+        "destination.domain": "l2.io",
         "destination.ip": "192.168.156.70",
         "destination.packets": 3,
         "destination.port": 443,
@@ -96,6 +97,8 @@
         "tags": [
             "suricata"
         ],
+        "tls.client.server_name": "l2.io",
+        "tls.resumed": true,
         "tls.version": "1.2",
         "tls.version_protocol": "tls"
     },