From b4c44f3ac1ba258d862e0e1064402342ecbad0af Mon Sep 17 00:00:00 2001 From: Matthew Budge Date: Sun, 15 Sep 2019 09:47:53 +0100 Subject: [PATCH 1/3] Added question.subdomain field Added to question.subdomain field for security use cases such as looking for dns-exfil. --- code/go/ecs/dns.go | 3 +++ docs/field-details.asciidoc | 11 +++++++++++ generated/beats/fields.ecs.yml | 6 ++++++ generated/csv/fields.csv | 1 + generated/ecs/ecs_flat.yml | 24 +++++++++++++++++------- generated/ecs/ecs_nested.yml | 24 +++++++++++++++++------- generated/elasticsearch/6/template.json | 4 ++++ generated/elasticsearch/7/template.json | 4 ++++ generated/legacy/template.json | 4 ++++ schema.json | 10 ++++++++++ schemas/dns.yml | 8 ++++++++ 11 files changed, 85 insertions(+), 14 deletions(-) diff --git a/code/go/ecs/dns.go b/code/go/ecs/dns.go index 17b930f84c..44836df4fc 100644 --- a/code/go/ecs/dns.go +++ b/code/go/ecs/dns.go @@ -72,6 +72,9 @@ type Dns struct { // "co.uk". QuestionRegisteredDomain string `ecs:"question.registered_domain"` + // A subdomain is a hostname under it's parent domain. + QuestionSubdomain string `ecs:"question.subdomain"` + // An array containing an object for each answer section returned by the // server. // The main keys that should be present in these objects are defined by diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index c9366c1a9b..986ee15594 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -890,6 +890,17 @@ example: `google.com` // =============================================================== +| dns.question.subdomain +| A subdomain is a hostname under it's parent domain. + +type: keyword + +example: `www` + +| extended + +// =============================================================== + | dns.question.type | The type of record being queried. diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index a0656ab401..d10bbcb4dc 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -735,6 +735,12 @@ list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk".' example: google.com + - name: question.subdomain + level: extended + type: keyword + ignore_above: 1024 + description: A subdomain is a hostname under it's parent domain. + example: www - name: question.type level: extended type: keyword diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index 3bce47e736..b4d3105540 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -91,6 +91,7 @@ dns.op_code,keyword,extended,QUERY,1.2.0-dev dns.question.class,keyword,extended,IN,1.2.0-dev dns.question.name,keyword,extended,www.google.com,1.2.0-dev dns.question.registered_domain,keyword,extended,google.com,1.2.0-dev +dns.question.subdomain,keyword,extended,www,1.2.0-dev dns.question.type,keyword,extended,AAAA,1.2.0-dev dns.resolved_ip,ip,extended,"['10.10.10.10', '10.10.10.11']",1.2.0-dev dns.response_code,keyword,extended,NOERROR,1.2.0-dev diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 54cf0ca76a..9d5b78c443 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -869,7 +869,7 @@ dns.answers: level: extended name: answers object_type: keyword - order: 9 + order: 10 short: Array of DNS answers. type: object dns.answers.class: @@ -879,7 +879,7 @@ dns.answers.class: ignore_above: 1024 level: extended name: answers.class - order: 12 + order: 13 short: The class of DNS data contained in this resource record. type: keyword dns.answers.data: @@ -891,7 +891,7 @@ dns.answers.data: ignore_above: 1024 level: extended name: answers.data - order: 14 + order: 15 short: The data describing the resource. type: keyword dns.answers.name: @@ -905,7 +905,7 @@ dns.answers.name: ignore_above: 1024 level: extended name: answers.name - order: 10 + order: 11 short: The domain name to which this resource record pertains. type: keyword dns.answers.ttl: @@ -915,7 +915,7 @@ dns.answers.ttl: flat_name: dns.answers.ttl level: extended name: answers.ttl - order: 13 + order: 14 short: The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached. type: long @@ -926,7 +926,7 @@ dns.answers.type: ignore_above: 1024 level: extended name: answers.type - order: 11 + order: 12 short: The type of data contained in this resource record. type: keyword dns.header_flags: @@ -1008,6 +1008,16 @@ dns.question.registered_domain: order: 8 short: The highest registered domain, stripped of the subdomain. type: keyword +dns.question.subdomain: + description: A subdomain is a hostname under it's parent domain. + example: www + flat_name: dns.question.subdomain + ignore_above: 1024 + level: extended + name: question.subdomain + order: 9 + short: The subdomain of the domain. + type: keyword dns.question.type: description: The type of record being queried. example: AAAA @@ -1031,7 +1041,7 @@ dns.resolved_ip: flat_name: dns.resolved_ip level: extended name: resolved_ip - order: 15 + order: 16 short: Array containing all IPs seen in answers.data type: ip dns.response_code: diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 24d1686bbe..71829f5695 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1037,7 +1037,7 @@ dns: level: extended name: answers object_type: keyword - order: 9 + order: 10 short: Array of DNS answers. type: object answers.class: @@ -1047,7 +1047,7 @@ dns: ignore_above: 1024 level: extended name: answers.class - order: 12 + order: 13 short: The class of DNS data contained in this resource record. type: keyword answers.data: @@ -1059,7 +1059,7 @@ dns: ignore_above: 1024 level: extended name: answers.data - order: 14 + order: 15 short: The data describing the resource. type: keyword answers.name: @@ -1073,7 +1073,7 @@ dns: ignore_above: 1024 level: extended name: answers.name - order: 10 + order: 11 short: The domain name to which this resource record pertains. type: keyword answers.ttl: @@ -1084,7 +1084,7 @@ dns: flat_name: dns.answers.ttl level: extended name: answers.ttl - order: 13 + order: 14 short: The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached. @@ -1096,7 +1096,7 @@ dns: ignore_above: 1024 level: extended name: answers.type - order: 11 + order: 12 short: The type of data contained in this resource record. type: keyword header_flags: @@ -1179,6 +1179,16 @@ dns: order: 8 short: The highest registered domain, stripped of the subdomain. type: keyword + question.subdomain: + description: A subdomain is a hostname under it's parent domain. + example: www + flat_name: dns.question.subdomain + ignore_above: 1024 + level: extended + name: question.subdomain + order: 9 + short: The subdomain of the domain. + type: keyword question.type: description: The type of record being queried. example: AAAA @@ -1202,7 +1212,7 @@ dns: flat_name: dns.resolved_ip level: extended name: resolved_ip - order: 15 + order: 16 short: Array containing all IPs seen in answers.data type: ip response_code: diff --git a/generated/elasticsearch/6/template.json b/generated/elasticsearch/6/template.json index ebd7821c2f..39f755af9f 100644 --- a/generated/elasticsearch/6/template.json +++ b/generated/elasticsearch/6/template.json @@ -452,6 +452,10 @@ "ignore_above": 1024, "type": "keyword" }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, "type": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/elasticsearch/7/template.json b/generated/elasticsearch/7/template.json index 6ddcb32921..ca3c142812 100644 --- a/generated/elasticsearch/7/template.json +++ b/generated/elasticsearch/7/template.json @@ -451,6 +451,10 @@ "ignore_above": 1024, "type": "keyword" }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, "type": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/legacy/template.json b/generated/legacy/template.json index 0e6cab1b38..222c9cad4b 100644 --- a/generated/legacy/template.json +++ b/generated/legacy/template.json @@ -273,6 +273,10 @@ "ignore_above": 1024, "type": "keyword" }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, "type": { "ignore_above": 1024, "type": "keyword" diff --git a/schema.json b/schema.json index 97863730c7..7522fa621b 100644 --- a/schema.json +++ b/schema.json @@ -625,6 +625,16 @@ "required": false, "type": "keyword" }, + "dns.question.subdomain": { + "description": "A subdomain is a hostname under it's parent domain.", + "example": "www", + "footnote": "", + "group": 2, + "level": "extended", + "name": "dns.question.subdomain", + "required": false, + "type": "keyword" + }, "dns.question.type": { "description": "The type of record being queried.", "example": "AAAA", diff --git a/schemas/dns.yml b/schemas/dns.yml index 9f5087ee90..eccdcd0d68 100644 --- a/schemas/dns.yml +++ b/schemas/dns.yml @@ -101,6 +101,14 @@ simply taking the last two labels will not work well for TLDs such as "co.uk". example: google.com + - name: question.subdomain + level: extended + type: keyword + short: The subdomain of the domain. + description: > + A subdomain is a hostname under it's parent domain. + example: www + - name: answers level: extended type: object From 46519b7bfb6750bf854627d0778cf0ecbd03030d Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Tue, 1 Oct 2019 16:17:44 -0400 Subject: [PATCH 2/3] Apply PR #561 feedback --- code/go/ecs/dns.go | 5 ++++- docs/field-details.asciidoc | 4 +++- generated/beats/fields.ecs.yml | 5 ++++- generated/ecs/ecs_flat.yml | 5 ++++- generated/ecs/ecs_nested.yml | 5 ++++- schema.json | 2 +- schemas/dns.yml | 5 ++++- 7 files changed, 24 insertions(+), 7 deletions(-) diff --git a/code/go/ecs/dns.go b/code/go/ecs/dns.go index 44836df4fc..65aaab10c2 100644 --- a/code/go/ecs/dns.go +++ b/code/go/ecs/dns.go @@ -72,7 +72,10 @@ type Dns struct { // "co.uk". QuestionRegisteredDomain string `ecs:"question.registered_domain"` - // A subdomain is a hostname under it's parent domain. + // The subdomain is all of the labels under the registered_domain. + // If the domain has multiple levels of subdomain, such as + // "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", + // with no trailing period. QuestionSubdomain string `ecs:"question.subdomain"` // An array containing an object for each answer section returned by the diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index c07d2a7c74..e31ed7f433 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -891,7 +891,9 @@ example: `google.com` // =============================================================== | dns.question.subdomain -| A subdomain is a hostname under it's parent domain. +| The subdomain is all of the labels under the registered_domain. + +If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. type: keyword diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 1402098cb4..cb02f05888 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -753,7 +753,10 @@ level: extended type: keyword ignore_above: 1024 - description: A subdomain is a hostname under it's parent domain. + description: 'The subdomain is all of the labels under the registered_domain. + + If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", + the subdomain field should contain "sub2.sub1", with no trailing period.' example: www - name: question.type level: extended diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index b28ed256da..9de3115899 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -1033,7 +1033,10 @@ dns.question.registered_domain: short: The highest registered domain, stripped of the subdomain. type: keyword dns.question.subdomain: - description: A subdomain is a hostname under it's parent domain. + description: 'The subdomain is all of the labels under the registered_domain. + + If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", + the subdomain field should contain "sub2.sub1", with no trailing period.' example: www flat_name: dns.question.subdomain ignore_above: 1024 diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index f25cdbd754..da72e7d47a 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1204,7 +1204,10 @@ dns: short: The highest registered domain, stripped of the subdomain. type: keyword question.subdomain: - description: A subdomain is a hostname under it's parent domain. + description: 'The subdomain is all of the labels under the registered_domain. + + If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", + the subdomain field should contain "sub2.sub1", with no trailing period.' example: www flat_name: dns.question.subdomain ignore_above: 1024 diff --git a/schema.json b/schema.json index a39ca968c2..6e97076dc7 100644 --- a/schema.json +++ b/schema.json @@ -626,7 +626,7 @@ "type": "keyword" }, "dns.question.subdomain": { - "description": "A subdomain is a hostname under it's parent domain.", + "description": "The subdomain is all of the labels under the registered_domain.\nIf the domain has multiple levels of subdomain, such as \"sub2.sub1.example.com\", the subdomain field should contain \"sub2.sub1\", with no trailing period.", "example": "www", "footnote": "", "group": 2, diff --git a/schemas/dns.yml b/schemas/dns.yml index eccdcd0d68..7d7900d088 100644 --- a/schemas/dns.yml +++ b/schemas/dns.yml @@ -106,7 +106,10 @@ type: keyword short: The subdomain of the domain. description: > - A subdomain is a hostname under it's parent domain. + The subdomain is all of the labels under the registered_domain. + + If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", + the subdomain field should contain "sub2.sub1", with no trailing period. example: www - name: answers From 131a83fa06c3fec776596650509e11c02a713060 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Tue, 1 Oct 2019 16:19:35 -0400 Subject: [PATCH 3/3] Changelog --- CHANGELOG.next.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 8beee53e20..615a2cd8ce 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -13,6 +13,7 @@ Thanks, you're awesome :-) --> * Added fields in `log.*` to allow for full Syslog mapping. #525 * Add group.domain field #547 +* Added `dns.question.subdomain` field. #561, #574 * Added `error.stack_trace` field. #562 * Added `log.origin.file.name`, `log.origin.function` and `log.origin.file.line` fields. #563 * Added `service.node.name` to allow distinction between different nodes of the same service running on the same host. #565