Skip to content

Commit

Permalink
[Filebeat] Fix tls mapping in suricata module (elastic#19494) (elasti…
Browse files Browse the repository at this point in the history 
…c#19608)

* Fix tls mapping in suricata module

- add suricata.eve.tls.ja3s.string field
- add suricata.eve.tls.ja3s.hash field
- add suricata.eve.tls.ja3.string field
- add suricata.eve.tls.ja3.hash field
- set default_field to false for ja3 & ja3s fields
- map suricata.eve.tls.ja3.hash to tls.client.ja3
- map suricata.eve.tls.ja3s.hash to tls.server.ja3s
- perform suricata.eve.tls.* -> tls.* mappings for all event types

Closes elastic#19492

(cherry picked from commit 362016d)
  • Loading branch information
@leehinman
leehinman authored Jul 6, 2020
1 parent 9aeb33e commit da3007a
Show file tree
Hide file tree
Showing 8 changed files with 214 additions and 21 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Fix date and timestamp formats for fortigate module {pull}19316[19316]
- Fix `googlecloud.audit` pipeline to only take in fields that are explicitly defined by the dataset. {issue}18465[18465] {pull}18472[18472]
- Fix a rate limit related issue in httpjson input for Okta module. {issue}18530[18530] {pull}18534[18534]
- Fix tls mapping in suricata module {issue}19492[19492] {pull}19494[19494]

*Heartbeat*

Expand Down
30 changes: 30 additions & 0 deletions filebeat/docs/fields.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34552,6 +34552,36 @@ type: keyword
--
*`suricata.eve.tls.ja3s.string`*::
+
--
type: keyword
--
*`suricata.eve.tls.ja3s.hash`*::
+
--
type: keyword
--
*`suricata.eve.tls.ja3.string`*::
+
--
type: keyword
--
*`suricata.eve.tls.ja3.hash`*::
+
--
type: keyword
--
*`suricata.eve.app_proto_ts`*::
+
--
Expand Down
18 changes: 18 additions & 0 deletions x-pack/filebeat/module/suricata/eve/_meta/fields.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -665,6 +665,24 @@
- name: subject
type: keyword

- name: ja3s
type: group
default_field: false
fields:
- name: string
type: keyword
- name: hash
type: keyword

- name: ja3
type: group
default_field: false
fields:
- name: string
type: keyword
- name: hash
type: keyword

- name: app_proto_ts
type: keyword

Expand Down
37 changes: 17 additions & 20 deletions x-pack/filebeat/module/suricata/eve/config/eve.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,26 +372,23 @@ processors:
addTlsVersion(evt);
cleanupTlsSni(evt);
}
- if:
equals:
suricata.eve.event_type: tls
then:
- convert:
ignore_missing: true
ignore_failure: true
mode: copy
fields:
- {from: suricata.eve.tls.subject, to: tls.server.subject}
- {from: suricata.eve.tls.issuerdn, to: tls.server.issuer}
- {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean}
- {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1}
- {from: suricata.eve.tls.sni, to: tls.client.server_name}
- {from: suricata.eve.tls.sni, to: destination.domain}
- {from: suricata.eve.tls.notbefore, to: tls.server.not_before}
- {from: suricata.eve.tls.notafter, to: tls.server.not_after}
- {from: suricata.eve.tls.ja3s, to: tls.server.ja3s}
- {from: suricata.eve.tls.certificate, to: tls.server.certificate}
- {from: suricata.eve.tls.chain, to: tls.server.certificate_chain}
- convert:
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.tls.subject, to: tls.server.subject}
- {from: suricata.eve.tls.issuerdn, to: tls.server.issuer}
- {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean}
- {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1}
- {from: suricata.eve.tls.sni, to: tls.client.server_name}
- {from: suricata.eve.tls.sni, to: destination.domain}
- {from: suricata.eve.tls.notbefore, to: tls.server.not_before}
- {from: suricata.eve.tls.notafter, to: tls.server.not_after}
- {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s}
- {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3}
- {from: suricata.eve.tls.certificate, to: tls.server.certificate}
- {from: suricata.eve.tls.chain, to: tls.server.certificate_chain}
- drop_fields:
ignore_missing: true
fields:
Expand Down
2 changes: 2 additions & 0 deletions x-pack/filebeat/module/suricata/eve/test/eve-alerts.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,5 @@
{"timestamp":"2018-10-04T09:35:00.897009+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":7,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/binary-amd64\/by-hash\/SHA256\/5190f7afbee38b3cb32225db478fdbabd46f76eaa9c5921a13091891bf3e9bbc","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2687},"app_proto":"http","flow":{"pkts_toserver":330,"pkts_toclient":591,"bytes_toserver":23758,"bytes_toclient":884342,"start":"2018-10-04T09:34:58.926006+0000"}}
{"timestamp":"2018-10-04T09:35:01.362208+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":8,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/i18n\/by-hash\/SHA256\/9fe539b7036e51327cd85ca5e0a4dd4eb47f69168875de2ac9842a5e36ebd4a4","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":524,"pkts_toclient":979,"bytes_toserver":36819,"bytes_toclient":1467603,"start":"2018-10-04T09:34:58.926006+0000"}}
{"timestamp":"2018-10-04T09:35:01.575088+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":9,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/multiverse\/binary-amd64\/by-hash\/SHA256\/8ab8cb220c0e50521c589acc2bc2b43a3121210f0b035a0605972bcffd73dd16","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":575,"pkts_toclient":1079,"bytes_toserver":40452,"bytes_toclient":1618380,"start":"2018-10-04T09:34:58.926006+0000"}}
{"tls":{"ja3s":{"string":"333,55555,66666-22","hash":"0993626a07ad09e1ce91293be7aa5721"},"ja3":{"string":"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0","hash":"d92325c876e7279f4eb8c62415e3a6b7"},"notafter":"2024-07-16T14:52:35","notbefore":"2019-07-17T14:52:35","version":"TLS 1.2","sni":"hostname.domain.net","fingerprint":"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33","serial":"00:11:22:33:44:55:66:77:88","issuerdn":"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN","subject":"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com"},"proto":"TCP","dest_port":9080,"dest_ip":"10.232.0.237","src_port":45884,"src_ip":"10.126.2.140","event_type":"tls","in_iface":"enp5s0","flow_id":1091813059495729,"timestamp":"2018-10-04T09:35:02.796615+0000"}
{"flow":{"start":"2020-06-26T11:00:02.970011-0400","bytes_toclient":4660,"bytes_toserver":1074,"pkts_toclient":8,"pkts_toserver":7},"app_proto":"tls","tls":{"ja3s":{"string":"742,48172,30210-30","hash":"391231ba5675e42807b9e1f457b2614e"},"ja3":{"string":"718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3","hash":"3f1ea03f5822e8021b60cc3e4b233181"},"notafter":"2026-06-25T17:36:29","notbefore":"2016-06-27T17:36:29","version":"TLS 1.2","sni":"host.domain.net","fingerprint":"36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc","serial":"72:A9:2C:51","issuerdn":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown","subject":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown"},"alert":{"severity":3,"category":"","signature":"SURICATA TLS on unusual port","rev":1,"signature_id":2610003,"gid":1,"action":"allowed"},"proto":"TCP","dest_port":8443,"dest_ip":"10.128.2.48","src_port":64389,"src_ip":"10.137.3.54","event_type":"alert","in_iface":"enp0s31f6","flow_id":991192778198299,"timestamp":"2020-06-26T11:00:03.342282-0400"}
142 changes: 142 additions & 0 deletions x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1536,5 +1536,147 @@
"user_agent.original": "Debian APT-HTTP/1.3 (1.6.3ubuntu0.1)",
"user_agent.os.name": "Debian",
"user_agent.version": "1.3"
},
{
"@timestamp": "2018-10-04T09:35:02.796Z",
"destination.address": "10.232.0.237",
"destination.domain": "hostname.domain.net",
"destination.ip": "10.232.0.237",
"destination.port": 9080,
"event.category": [
"network"
],
"event.dataset": "suricata.eve",
"event.kind": "event",
"event.module": "suricata",
"event.original": "{\"tls\":{\"ja3s\":{\"string\":\"333,55555,66666-22\",\"hash\":\"0993626a07ad09e1ce91293be7aa5721\"},\"ja3\":{\"string\":\"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0\",\"hash\":\"d92325c876e7279f4eb8c62415e3a6b7\"},\"notafter\":\"2024-07-16T14:52:35\",\"notbefore\":\"2019-07-17T14:52:35\",\"version\":\"TLS 1.2\",\"sni\":\"hostname.domain.net\",\"fingerprint\":\"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33\",\"serial\":\"00:11:22:33:44:55:66:77:88\",\"issuerdn\":\"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN\",\"subject\":\"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com\"},\"proto\":\"TCP\",\"dest_port\":9080,\"dest_ip\":\"10.232.0.237\",\"src_port\":45884,\"src_ip\":\"10.126.2.140\",\"event_type\":\"tls\",\"in_iface\":\"enp5s0\",\"flow_id\":1091813059495729,\"timestamp\":\"2018-10-04T09:35:02.796615+0000\"}",
"event.type": [
"protocol"
],
"fileset.name": "eve",
"input.type": "log",
"log.offset": 16546,
"network.community_id": "1:qsGDjYDIWp+kHhxotTdhPbUaWSo=",
"network.protocol": "tls",
"network.transport": "tcp",
"related.hash": [
"00112233445566778899AABBCCDDEEFF00112233"
],
"related.ip": [
"10.126.2.140",
"10.232.0.237"
],
"service.type": "suricata",
"source.address": "10.126.2.140",
"source.ip": "10.126.2.140",
"source.port": 45884,
"suricata.eve.event_type": "tls",
"suricata.eve.flow_id": 1091813059495729,
"suricata.eve.in_iface": "enp5s0",
"suricata.eve.tls.fingerprint": "00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33",
"suricata.eve.tls.issuerdn": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN",
"suricata.eve.tls.ja3.hash": "d92325c876e7279f4eb8c62415e3a6b7",
"suricata.eve.tls.ja3.string": "001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0",
"suricata.eve.tls.ja3s.hash": "0993626a07ad09e1ce91293be7aa5721",
"suricata.eve.tls.ja3s.string": "333,55555,66666-22",
"suricata.eve.tls.notafter": "2024-07-16T14:52:35",
"suricata.eve.tls.notbefore": "2019-07-17T14:52:35",
"suricata.eve.tls.serial": "00:11:22:33:44:55:66:77:88",
"suricata.eve.tls.sni": "hostname.domain.net",
"suricata.eve.tls.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com",
"suricata.eve.tls.version": "TLS 1.2",
"tags": [
"suricata"
],
"tls.client.ja3": "d92325c876e7279f4eb8c62415e3a6b7",
"tls.client.server_name": "hostname.domain.net",
"tls.server.hash.sha1": "00112233445566778899AABBCCDDEEFF00112233",
"tls.server.issuer": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN",
"tls.server.ja3s": "0993626a07ad09e1ce91293be7aa5721",
"tls.server.not_after": "2024-07-16T14:52:35",
"tls.server.not_before": "2019-07-17T14:52:35",
"tls.server.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com",
"tls.version": "1.2",
"tls.version_protocol": "tls"
},
{
"@timestamp": "2020-06-26T15:00:03.342Z",
"destination.address": "10.128.2.48",
"destination.bytes": 4660,
"destination.domain": "host.domain.net",
"destination.ip": "10.128.2.48",
"destination.packets": 8,
"destination.port": 8443,
"event.category": [
"network",
"intrusion_detection"
],
"event.dataset": "suricata.eve",
"event.kind": "alert",
"event.module": "suricata",
"event.original": "{\"flow\":{\"start\":\"2020-06-26T11:00:02.970011-0400\",\"bytes_toclient\":4660,\"bytes_toserver\":1074,\"pkts_toclient\":8,\"pkts_toserver\":7},\"app_proto\":\"tls\",\"tls\":{\"ja3s\":{\"string\":\"742,48172,30210-30\",\"hash\":\"391231ba5675e42807b9e1f457b2614e\"},\"ja3\":{\"string\":\"718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3\",\"hash\":\"3f1ea03f5822e8021b60cc3e4b233181\"},\"notafter\":\"2026-06-25T17:36:29\",\"notbefore\":\"2016-06-27T17:36:29\",\"version\":\"TLS 1.2\",\"sni\":\"host.domain.net\",\"fingerprint\":\"36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc\",\"serial\":\"72:A9:2C:51\",\"issuerdn\":\"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown\",\"subject\":\"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown\"},\"alert\":{\"severity\":3,\"category\":\"\",\"signature\":\"SURICATA TLS on unusual port\",\"rev\":1,\"signature_id\":2610003,\"gid\":1,\"action\":\"allowed\"},\"proto\":\"TCP\",\"dest_port\":8443,\"dest_ip\":\"10.128.2.48\",\"src_port\":64389,\"src_ip\":\"10.137.3.54\",\"event_type\":\"alert\",\"in_iface\":\"enp0s31f6\",\"flow_id\":991192778198299,\"timestamp\":\"2020-06-26T11:00:03.342282-0400\"}",
"event.severity": 3,
"event.start": "2020-06-26T15:00:02.970Z",
"event.type": [
"allowed"
],
"fileset.name": "eve",
"input.type": "log",
"log.offset": 17606,
"message": "",
"network.bytes": 5734,
"network.community_id": "1:W6fjhboFUwyEchJ3ELaqSBzDEJE=",
"network.packets": 15,
"network.protocol": "tls",
"network.transport": "tcp",
"related.hash": [
"363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC"
],
"related.ip": [
"10.137.3.54",
"10.128.2.48"
],
"rule.category": "",
"rule.id": "2610003",
"rule.name": "SURICATA TLS on unusual port",
"service.type": "suricata",
"source.address": "10.137.3.54",
"source.bytes": 1074,
"source.ip": "10.137.3.54",
"source.packets": 7,
"source.port": 64389,
"suricata.eve.alert.category": "",
"suricata.eve.alert.gid": 1,
"suricata.eve.alert.rev": 1,
"suricata.eve.alert.signature": "SURICATA TLS on unusual port",
"suricata.eve.alert.signature_id": 2610003,
"suricata.eve.event_type": "alert",
"suricata.eve.flow_id": 991192778198299,
"suricata.eve.in_iface": "enp0s31f6",
"suricata.eve.tls.fingerprint": "36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc",
"suricata.eve.tls.issuerdn": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
"suricata.eve.tls.ja3.hash": "3f1ea03f5822e8021b60cc3e4b233181",
"suricata.eve.tls.ja3.string": "718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3",
"suricata.eve.tls.ja3s.hash": "391231ba5675e42807b9e1f457b2614e",
"suricata.eve.tls.ja3s.string": "742,48172,30210-30",
"suricata.eve.tls.notafter": "2026-06-25T17:36:29",
"suricata.eve.tls.notbefore": "2016-06-27T17:36:29",
"suricata.eve.tls.serial": "72:A9:2C:51",
"suricata.eve.tls.sni": "host.domain.net",
"suricata.eve.tls.subject": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
"suricata.eve.tls.version": "TLS 1.2",
"tags": [
"suricata"
],
"tls.client.ja3": "3f1ea03f5822e8021b60cc3e4b233181",
"tls.client.server_name": "host.domain.net",
"tls.server.hash.sha1": "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC",
"tls.server.issuer": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
"tls.server.ja3s": "391231ba5675e42807b9e1f457b2614e",
"tls.server.not_after": "2026-06-25T17:36:29",
"tls.server.not_before": "2016-06-27T17:36:29",
"tls.server.subject": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
"tls.version": "1.2",
"tls.version_protocol": "tls"
}
]
3 changes: 3 additions & 0 deletions x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
"@timestamp": "2018-07-05T19:07:20.910Z",
"destination.address": "192.168.156.70",
"destination.bytes": 343,
"destination.domain": "l2.io",
"destination.ip": "192.168.156.70",
"destination.packets": 3,
"destination.port": 443,
Expand Down Expand Up @@ -96,6 +97,8 @@
"tags": [
"suricata"
],
"tls.client.server_name": "l2.io",
"tls.resumed": true,
"tls.version": "1.2",
"tls.version_protocol": "tls"
},
Expand Down
Loading

0 comments on commit da3007a

Please sign in to comment.