Skip to content

Commit

Permalink
[Filebeat][suricata] Map x509 for suricata/eve fileset (elastic#20973)
Browse files Browse the repository at this point in the history 
* Map x509 for suricata/eve fileset

* Fix not_before condition and bump ecs version

(cherry picked from commit 70d6bde)
  • Loading branch information
@marc-gr
marc-gr committed Sep 9, 2020
1 parent adc63fe commit 09eb206
Show file tree
Hide file tree
Showing 7 changed files with 160 additions and 22 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -641,6 +641,7 @@ field. You can revert this change by configuring tags for the module and omittin
- Improve Zeek Kerberos module with `x509` ECS mappings {pull}20958[20958]
- Improve Fortinet firewall module with `x509` ECS mappings {pull}20983[20983]
- Improve Santa module with `x509` ECS mappings {pull}20976[20976]
- Improve Suricata Eve module with `x509` ECS mappings {pull}20973[20973]

*Heartbeat*

Expand Down
4 changes: 1 addition & 3 deletions x-pack/filebeat/module/suricata/eve/config/eve.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,8 +384,6 @@ processors:
- {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1}
- {from: suricata.eve.tls.sni, to: tls.client.server_name}
- {from: suricata.eve.tls.sni, to: destination.domain}
- {from: suricata.eve.tls.notbefore, to: tls.server.not_before}
- {from: suricata.eve.tls.notafter, to: tls.server.not_after}
- {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s}
- {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3}
- {from: suricata.eve.tls.certificate, to: tls.server.certificate}
Expand All @@ -404,4 +402,4 @@ processors:
- add_fields:
target: ''
fields:
ecs.version: 1.5.0
ecs.version: 1.6.0
101 changes: 101 additions & 0 deletions x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,13 +234,114 @@ processors:
field: related.hash
value: "{{tls.server.hash.sha1}}"
if: "ctx?.tls?.server?.hash?.sha1 != null"
- gsub:
field: suricata.eve.tls.issuerdn
pattern: \\,
replacement: ""
ignore_missing: true
- kv:
field: suricata.eve.tls.issuerdn
field_split: ', '
value_split: '='
target_field: suricata.eve.tls.kv_issuerdn
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_issuerdn.C
target_field: file.x509.issuer.country
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_issuerdn.CN
target_field: file.x509.issuer.common_name
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_issuerdn.L
target_field: file.x509.issuer.locality
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_issuerdn.O
target_field: file.x509.issuer.organization
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_issuerdn.OU
target_field: file.x509.issuer.organizational_unit
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_issuerdn.ST
target_field: file.x509.issuer.state_or_province
ignore_missing: true
- gsub:
field: suricata.eve.tls.subject
pattern: \\,
replacement: ""
ignore_missing: true
- kv:
field: suricata.eve.tls.subject
field_split: ', '
value_split: '='
target_field: suricata.eve.tls.kv_subject
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_subject.C
target_field: file.x509.subject.country
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_subject.CN
target_field: file.x509.subject.common_name
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_subject.L
target_field: file.x509.subject.locality
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_subject.O
target_field: file.x509.subject.organization
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_subject.OU
target_field: file.x509.subject.organizational_unit
ignore_missing: true
- rename:
field: suricata.eve.tls.kv_subject.ST
target_field: file.x509.subject.state_or_province
ignore_missing: true
- set:
field: file.x509.serial_number
value: '{{suricata.eve.tls.serial}}'
ignore_empty_value: true
- gsub:
field: file.x509.serial_number
pattern: ':'
replacement: ''
ignore_missing: true
- date:
field: suricata.eve.tls.notafter
target_field: tls.server.not_after
formats:
- ISO8601
if: ctx.suricata?.eve?.tls?.notafter != null
- date:
field: suricata.eve.tls.notbefore
target_field: tls.server.not_before
formats:
- ISO8601
if: ctx.suricata?.eve?.tls?.notbefore != null
- set:
field: file.x509.not_after
value: '{{tls.server.not_after}}'
ignore_empty_value: true
- set:
field: file.x509.not_before
value: '{{tls.server.not_before}}'
ignore_empty_value: true
- remove:
field:
- suricata.eve.app_proto
- suricata.eve.flow.end
- suricata.eve.flow.start
- suricata.eve.http.http_method
- suricata.eve.http.http_user_agent
- suricata.eve.tls.kv_issuerdn
- suricata.eve.tls.kv_subject
ignore_missing: true
on_failure:
- set:
Expand Down
2 changes: 1 addition & 1 deletion x-pack/filebeat/module/suricata/eve/test/eve-alerts.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,5 @@
{"timestamp":"2018-10-04T09:35:00.897009+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":7,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/binary-amd64\/by-hash\/SHA256\/5190f7afbee38b3cb32225db478fdbabd46f76eaa9c5921a13091891bf3e9bbc","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2687},"app_proto":"http","flow":{"pkts_toserver":330,"pkts_toclient":591,"bytes_toserver":23758,"bytes_toclient":884342,"start":"2018-10-04T09:34:58.926006+0000"}}
{"timestamp":"2018-10-04T09:35:01.362208+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":8,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/i18n\/by-hash\/SHA256\/9fe539b7036e51327cd85ca5e0a4dd4eb47f69168875de2ac9842a5e36ebd4a4","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":524,"pkts_toclient":979,"bytes_toserver":36819,"bytes_toclient":1467603,"start":"2018-10-04T09:34:58.926006+0000"}}
{"timestamp":"2018-10-04T09:35:01.575088+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":9,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/multiverse\/binary-amd64\/by-hash\/SHA256\/8ab8cb220c0e50521c589acc2bc2b43a3121210f0b035a0605972bcffd73dd16","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":575,"pkts_toclient":1079,"bytes_toserver":40452,"bytes_toclient":1618380,"start":"2018-10-04T09:34:58.926006+0000"}}
{"tls":{"ja3s":{"string":"333,55555,66666-22","hash":"0993626a07ad09e1ce91293be7aa5721"},"ja3":{"string":"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0","hash":"d92325c876e7279f4eb8c62415e3a6b7"},"notafter":"2024-07-16T14:52:35","notbefore":"2019-07-17T14:52:35","version":"TLS 1.2","sni":"hostname.domain.net","fingerprint":"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33","serial":"00:11:22:33:44:55:66:77:88","issuerdn":"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN","subject":"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com"},"proto":"TCP","dest_port":9080,"dest_ip":"10.232.0.237","src_port":45884,"src_ip":"10.126.2.140","event_type":"tls","in_iface":"enp5s0","flow_id":1091813059495729,"timestamp":"2018-10-04T09:35:02.796615+0000"}
{"tls":{"ja3s":{"string":"333,55555,66666-22","hash":"0993626a07ad09e1ce91293be7aa5721"},"ja3":{"string":"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0","hash":"d92325c876e7279f4eb8c62415e3a6b7"},"notafter":"2024-07-16T14:52:35","notbefore":"2019-07-17T14:52:35","version":"TLS 1.2","sni":"hostname.domain.net","fingerprint":"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33","serial":"00:11:22:33:44:55:66:77:88","issuerdn":"C=US, O=Google Inc, CN=Google Internet Authority G2","subject":"C=US, ST=California, L=Mountain View, O=Google Inc, CN=*.google.com"},"proto":"TCP","dest_port":9080,"dest_ip":"10.232.0.237","src_port":45884,"src_ip":"10.126.2.140","event_type":"tls","in_iface":"enp5s0","flow_id":1091813059495729,"timestamp":"2018-10-04T09:35:02.796615+0000"}
{"flow":{"start":"2020-06-26T11:00:02.970011-0400","bytes_toclient":4660,"bytes_toserver":1074,"pkts_toclient":8,"pkts_toserver":7},"app_proto":"tls","tls":{"ja3s":{"string":"742,48172,30210-30","hash":"391231ba5675e42807b9e1f457b2614e"},"ja3":{"string":"718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3","hash":"3f1ea03f5822e8021b60cc3e4b233181"},"notafter":"2026-06-25T17:36:29","notbefore":"2016-06-27T17:36:29","version":"TLS 1.2","sni":"host.domain.net","fingerprint":"36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc","serial":"72:A9:2C:51","issuerdn":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown","subject":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown"},"alert":{"severity":3,"category":"","signature":"SURICATA TLS on unusual port","rev":1,"signature_id":2610003,"gid":1,"action":"allowed"},"proto":"TCP","dest_port":8443,"dest_ip":"10.128.2.48","src_port":64389,"src_ip":"10.137.3.54","event_type":"alert","in_iface":"enp0s31f6","flow_id":991192778198299,"timestamp":"2020-06-26T11:00:03.342282-0400"}
46 changes: 36 additions & 10 deletions x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1549,10 +1549,21 @@
"event.dataset": "suricata.eve",
"event.kind": "event",
"event.module": "suricata",
"event.original": "{\"tls\":{\"ja3s\":{\"string\":\"333,55555,66666-22\",\"hash\":\"0993626a07ad09e1ce91293be7aa5721\"},\"ja3\":{\"string\":\"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0\",\"hash\":\"d92325c876e7279f4eb8c62415e3a6b7\"},\"notafter\":\"2024-07-16T14:52:35\",\"notbefore\":\"2019-07-17T14:52:35\",\"version\":\"TLS 1.2\",\"sni\":\"hostname.domain.net\",\"fingerprint\":\"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33\",\"serial\":\"00:11:22:33:44:55:66:77:88\",\"issuerdn\":\"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN\",\"subject\":\"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com\"},\"proto\":\"TCP\",\"dest_port\":9080,\"dest_ip\":\"10.232.0.237\",\"src_port\":45884,\"src_ip\":\"10.126.2.140\",\"event_type\":\"tls\",\"in_iface\":\"enp5s0\",\"flow_id\":1091813059495729,\"timestamp\":\"2018-10-04T09:35:02.796615+0000\"}",
"event.original": "{\"tls\":{\"ja3s\":{\"string\":\"333,55555,66666-22\",\"hash\":\"0993626a07ad09e1ce91293be7aa5721\"},\"ja3\":{\"string\":\"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0\",\"hash\":\"d92325c876e7279f4eb8c62415e3a6b7\"},\"notafter\":\"2024-07-16T14:52:35\",\"notbefore\":\"2019-07-17T14:52:35\",\"version\":\"TLS 1.2\",\"sni\":\"hostname.domain.net\",\"fingerprint\":\"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33\",\"serial\":\"00:11:22:33:44:55:66:77:88\",\"issuerdn\":\"C=US, O=Google Inc, CN=Google Internet Authority G2\",\"subject\":\"C=US, ST=California, L=Mountain View, O=Google Inc, CN=*.google.com\"},\"proto\":\"TCP\",\"dest_port\":9080,\"dest_ip\":\"10.232.0.237\",\"src_port\":45884,\"src_ip\":\"10.126.2.140\",\"event_type\":\"tls\",\"in_iface\":\"enp5s0\",\"flow_id\":1091813059495729,\"timestamp\":\"2018-10-04T09:35:02.796615+0000\"}",
"event.type": [
"protocol"
],
"file.x509.issuer.common_name": "Google Internet Authority G2",
"file.x509.issuer.country": "US",
"file.x509.issuer.organization": "Google Inc",
"file.x509.not_after": "2024-07-16T14:52:35.000Z",
"file.x509.not_before": "2019-07-17T14:52:35.000Z",
"file.x509.serial_number": "001122334455667788",
"file.x509.subject.common_name": "*.google.com",
"file.x509.subject.country": "US",
"file.x509.subject.locality": "Mountain View",
"file.x509.subject.organization": "Google Inc",
"file.x509.subject.state_or_province": "California",
"fileset.name": "eve",
"input.type": "log",
"log.offset": 16546,
Expand All @@ -1574,7 +1585,7 @@
"suricata.eve.flow_id": 1091813059495729,
"suricata.eve.in_iface": "enp5s0",
"suricata.eve.tls.fingerprint": "00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33",
"suricata.eve.tls.issuerdn": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN",
"suricata.eve.tls.issuerdn": "C=US, O=Google Inc, CN=Google Internet Authority G2",
"suricata.eve.tls.ja3.hash": "d92325c876e7279f4eb8c62415e3a6b7",
"suricata.eve.tls.ja3.string": "001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0",
"suricata.eve.tls.ja3s.hash": "0993626a07ad09e1ce91293be7aa5721",
Expand All @@ -1583,19 +1594,19 @@
"suricata.eve.tls.notbefore": "2019-07-17T14:52:35",
"suricata.eve.tls.serial": "00:11:22:33:44:55:66:77:88",
"suricata.eve.tls.sni": "hostname.domain.net",
"suricata.eve.tls.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com",
"suricata.eve.tls.subject": "C=US, ST=California, L=Mountain View, O=Google Inc, CN=*.google.com",
"suricata.eve.tls.version": "TLS 1.2",
"tags": [
"suricata"
],
"tls.client.ja3": "d92325c876e7279f4eb8c62415e3a6b7",
"tls.client.server_name": "hostname.domain.net",
"tls.server.hash.sha1": "00112233445566778899AABBCCDDEEFF00112233",
"tls.server.issuer": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN",
"tls.server.issuer": "C=US, O=Google Inc, CN=Google Internet Authority G2",
"tls.server.ja3s": "0993626a07ad09e1ce91293be7aa5721",
"tls.server.not_after": "2024-07-16T14:52:35",
"tls.server.not_before": "2019-07-17T14:52:35",
"tls.server.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com",
"tls.server.not_after": "2024-07-16T14:52:35.000Z",
"tls.server.not_before": "2019-07-17T14:52:35.000Z",
"tls.server.subject": "C=US, ST=California, L=Mountain View, O=Google Inc, CN=*.google.com",
"tls.version": "1.2",
"tls.version_protocol": "tls"
},
Expand All @@ -1620,9 +1631,24 @@
"event.type": [
"allowed"
],
"file.x509.issuer.common_name": "Unknown",
"file.x509.issuer.country": "Unknown",
"file.x509.issuer.locality": "Unknown",
"file.x509.issuer.organization": "Unknown",
"file.x509.issuer.organizational_unit": "Unknown",
"file.x509.issuer.state_or_province": "Unknown",
"file.x509.not_after": "2026-06-25T17:36:29.000Z",
"file.x509.not_before": "2016-06-27T17:36:29.000Z",
"file.x509.serial_number": "72A92C51",
"file.x509.subject.common_name": "Unknown",
"file.x509.subject.country": "Unknown",
"file.x509.subject.locality": "Unknown",
"file.x509.subject.organization": "Unknown",
"file.x509.subject.organizational_unit": "Unknown",
"file.x509.subject.state_or_province": "Unknown",
"fileset.name": "eve",
"input.type": "log",
"log.offset": 17606,
"log.offset": 17541,
"message": "",
"network.bytes": 5734,
"network.community_id": "1:W6fjhboFUwyEchJ3ELaqSBzDEJE=",
Expand Down Expand Up @@ -1672,8 +1698,8 @@
"tls.server.hash.sha1": "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC",
"tls.server.issuer": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
"tls.server.ja3s": "391231ba5675e42807b9e1f457b2614e",
"tls.server.not_after": "2026-06-25T17:36:29",
"tls.server.not_before": "2016-06-27T17:36:29",
"tls.server.not_after": "2026-06-25T17:36:29.000Z",
"tls.server.not_before": "2016-06-27T17:36:29.000Z",
"tls.server.subject": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown",
"tls.version": "1.2",
"tls.version_protocol": "tls"
Expand Down
Loading

0 comments on commit 09eb206

Please sign in to comment.