Skip to content

Commit

Permalink
[Filebeat] Refactor Suricata pipeline to use Ingest Node more (elasti…
Browse files Browse the repository at this point in the history 
…c#22291)

Refactor the Suricata module pipeline to use Ingest Node more. I moved most
processing from Beat processors to Ingest Node. I created a DNS and TLS
pipeline to house all of the processing for those protocols respectively.

Other changes
- Added DNS response IPs to `related.ip`.
- Some DNS events were missing `dns.header_flags`.
  • Loading branch information
@andrewkroh
andrewkroh committed Nov 5, 2020
1 parent a399d21 commit 2fe9e2a
Show file tree
Hide file tree
Showing 10 changed files with 629 additions and 473 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -656,6 +656,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add max_number_of_messages config into s3 input. {pull}21993[21993]
- Update Okta documentation for new stateful restarts. {pull}22091[22091]
- Copy tag names from MISP data into events. {pull}21664[21664]
- Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291]
- Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696]
- Added `event.ingested` field to data from the Netflow module. {pull}22412[22412]

Expand Down
365 changes: 9 additions & 356 deletions x-pack/filebeat/module/suricata/eve/config/eve.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,363 +42,16 @@ processors:
{{ if .community_id }}
- community_id:
{{ end }}
- if:
equals:
suricata.eve.event_type: dns
then:
- convert:
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.dns.id, to: dns.id, type: string}
- {from: suricata.eve.dns.rcode, to: dns.response_code}
- {from: suricata.eve.dns.type, to: dns.type}
- convert:
when.equals.dns.type: query
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.dns.rrname, to: dns.question.name}
- {from: suricata.eve.dns.rrtype, to: dns.question.type}
- if:
and:
- equals.dns.type: answer
- equals.suricata.eve.dns.version: 2
then:
- convert:
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.dns.rrname, to: dns.question.name}
- {from: suricata.eve.dns.rrtype, to: dns.question.type}
- registered_domain:
ignore_missing: true
ignore_failure: true
field: dns.question.name
target_field: dns.question.registered_domain
- script:
id: eve_process
lang: javascript
source: >-
function addEcsCategorization(evt) {
var event_type = evt.Get("suricata.eve.event_type");
if (event_type == null) {
return;
}
var catArray = [];
var typeArray = [];
evt.Put("suricata.eve.event_type", event_type.toLowerCase());
switch (event_type.toLowerCase()) {
case "alert":
evt.Put("event.kind", "alert");
catArray.push("network");
catArray.push("intrusion_detection");
break;
case "anomaly":
evt.Put("event.kind", "event");
catArray.push("network");
break;
case "http":
evt.Put("event.kind", "event");
catArray.push("network");
catArray.push("web");
typeArray.push("access");
typeArray.push("protocol");
evt.Put("network.protocol", "http");
var status = evt.Get("suricata.eve.http.status");
if (status == null) {
break;
}
if (status < 400) {
evt.Put("event.outcome", "success");
}
if (status >= 400 ) {
evt.Put("event.outcome", "failure");
}
break;
case "dns":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "dns");
break;
case "ftp":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "ftp");
break;
case "ftp_data":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "ftp");
break;
case "tls":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "tls");
break;
case "tftp":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "tftp");
break;
case "smb":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "smb");
break;
case "ssh":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "ssh");
break;
case "flow":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("connection");
var state = evt.Get("suricata.eve.flow.state");
if (state == null) {
break;
}
switch (state) {
case "new":
typeArray.push("start");
break;
case "closed":
typeArray.push("end");
break;
}
break;
case "rdp":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "rdp");
break;
case "stats":
evt.Put("event.kind", "metric");
break;
default:
evt.Put("event.kind", "event");
catArray.push("network");
}
if (catArray.length > 0) {
evt.Put("event.category", catArray);
}
if (typeArray.length > 0) {
evt.Put("event.type", typeArray);
}
}
function setDnsV1Answers(evt) {
var dns_type = evt.Get("dns.type")
if (dns_type != "answer") {
return;
}
var version = evt.Get("suricata.eve.dns.version")
if (version == "2") {
return;
}
var name = evt.Get("suricata.eve.dns.rrname");
var data = evt.Get("suricata.eve.dns.rdata");
var type = evt.Get("suricata.eve.dns.rrtype");
var ttl = evt.Get("suricata.eve.dns.ttl");
var answer = {};
if (name) {
answer.name = name;
}
if (data) {
answer.data = data;
}
if (type) {
answer.type = type;
}
if (ttl) {
answer.ttl = ttl;
}
if (Object.keys(answer).length === 0) {
return;
}
evt.Put("dns.answers", [answer]);
}
function addDnsV2Answers(evt) {
var type = evt.Get("dns.type")
if (type != "answer") {
return;
}
var version = evt.Get("suricata.eve.dns.version")
if (version != 2) {
return;
}
var answers = evt.Get("suricata.eve.dns.answers");
if (!answers) {
return;
}
evt.Delete("suricata.eve.dns.answers");
var resolvedIps = [];
for (var i = 0; i < answers.length; i++) {
var answer = answers[i];
// Rename properties.
var name = answer["rrname"];
delete answer["rrname"];
var type = answer["rrtype"];
delete answer["rrtype"];
var data = answer["rdata"];
delete answer["rdata"];
answer["name"] = name;
answer["type"] = type;
answer["data"] = data;
// Append IP addresses to dns.resolved_ip.
if (type === "A" || type === "AAAA") {
resolvedIps.push(data);
}
}
evt.Put("dns.answers", answers);
if (resolvedIps.length > 0) {
evt.Put("dns.resolved_ip", resolvedIps);
}
}
function addDnsV2HeaderFlags(evt) {
var type = evt.Get("dns.type")
if (type != "answer") {
return;
}
var version = evt.Get("suricata.eve.dns.version")
if (version != 2) {
return;
}
var flag = evt.Get("suricata.eve.dns.aa");
if (flag === true) {
evt.AppendTo("dns.header_flags", "AA");
}
flag = evt.Get("suricata.eve.dns.tc");
if (flag === true) {
evt.AppendTo("dns.header_flags", "TC");
}
flag = evt.Get("suricata.eve.dns.rd");
if (flag === true) {
evt.AppendTo("dns.header_flags", "RD");
}
flag = evt.Get("suricata.eve.dns.ra");
if (flag === true) {
evt.AppendTo("dns.header_flags", "RA");
}
}
function addTopLevelDomain(evt) {
var rd = evt.Get("dns.question.registered_domain");
if (rd == null) {
return;
}
var firstPeriod = rd.indexOf(".");
if (firstPeriod == -1) {
return;
}
evt.Put("dns.question.top_level_domain", rd.substr(firstPeriod + 1));
}
function cleanupAppProto(evt) {
var proto = evt.Get("suricata.eve.app_proto");
if (proto == null){
return;
}
switch (proto.toLowerCase()) {
case "failed":
case "template":
case "template-rust":
break;
case "ftp-data":
evt.Put("network.protocol", "ftp");
break;
default:
evt.Put("network.protocol", proto.toLowerCase());
}
evt.Delete("suricata.eve.app_proto");
}
function addRelatedIps(evt) {
var src_ip = evt.Get("source.ip");
if (src_ip != null) {
evt.AppendTo("related.ip", src_ip);
}
var dst_ip = evt.Get("destination.ip");
if (dst_ip != null) {
evt.AppendTo("related.ip", dst_ip);
}
}
function addTlsVersion(evt) {
var tls_version = evt.Get("suricata.eve.tls.version");
if (tls_version == null) {
return;
}
var parts = tls_version.split(" ");
if (parts.length < 2) {
return;
}
evt.Put("tls.version_protocol", parts[0].toLowerCase());
evt.Put("tls.version", parts[1]);
}
function cleanupTlsSni(evt) {
var sni = evt.Get("suricata.eve.tls.sni");
if (sni == null) {
return;
}
if ("." == sni.charAt(sni.length - 1)) {
evt.Put("suricata.eve.tls.sni", sni.substring(0, sni.length - 1));
}
}
function process(evt) {
var event_type = evt.Get("suricata.eve.event_type");
addEcsCategorization(evt);
if (event_type == "dns") {
setDnsV1Answers(evt);
addDnsV2Answers(evt);
addDnsV2HeaderFlags(evt);
addTopLevelDomain(evt);
}
cleanupAppProto(evt);
addRelatedIps(evt);
addTlsVersion(evt);
cleanupTlsSni(evt);
}
- convert:
- registered_domain:
when:
or:
- equals.suricata.eve.dns.type: query
# V2 events always include the query data.
- equals.suricata.eve.dns.version: 2
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.tls.subject, to: tls.server.subject}
- {from: suricata.eve.tls.issuerdn, to: tls.server.issuer}
- {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean}
- {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1}
- {from: suricata.eve.tls.sni, to: tls.client.server_name}
- {from: suricata.eve.tls.sni, to: destination.domain}
- {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s}
- {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3}
- {from: suricata.eve.tls.certificate, to: tls.server.certificate}
- {from: suricata.eve.tls.chain, to: tls.server.certificate_chain}
- drop_fields:
ignore_missing: true
fields:
- suricata.eve.dns.aa
- suricata.eve.dns.tc
- suricata.eve.dns.rd
- suricata.eve.dns.ra
- suricata.eve.dns.qr
- suricata.eve.dns.version
- suricata.eve.dns.flags
- suricata.eve.dns.grouped
ignore_failure: true
field: suricata.eve.dns.rrname
target_field: dns.question.registered_domain
- add_fields:
target: ''
fields:
Expand Down
Loading

0 comments on commit 2fe9e2a

Please sign in to comment.