diff --git a/.appveyor.yml b/.appveyor.yml index 6ebbc412912..466804f57f3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -33,8 +33,11 @@ install: - ps: c:\gopath\src\github.com\elastic\beats\libbeat\scripts\install-go.ps1 -version 1.5.3 - set PATH=%GOROOT%\bin;%PATH% # AppVeyor installed mingw is 32-bit only. - - cinst mingw > mingw-install.txt - - ps: Push-AppveyorArtifact mingw-install.txt + - ps: >- + if(!(Test-Path "C:\tools\mingw64\bin\gcc.exe")) { + cinst mingw > mingw-install.txt + Push-AppveyorArtifact mingw-install.txt + } - set PATH=C:\tools\mingw64\bin;%GOROOT%\bin;%PATH% - set PATH=%GOPATH%\bin;%PATH% - go get github.com/pierrre/gotestcover diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 5a13675e3d4..c6c806a0b61 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -6,7 +6,7 @@ This file is generated! See etc/fields.yml and scripts/generate_field_docs.py [[exported-fields]] == Exported Fields -This document describes the fields that are exported by Packetbeat. They are +This document describes the fields that are exported by Docs/Fields.Asciidoc. They are grouped in the following categories: * <> @@ -477,7 +477,7 @@ The DNS status code. ==== dns.question.name -example: www.google.com +example: www.google.com. The domain name being queried. If the name field contains non-printable characters (below 32 or above 126), then those characters are represented as escaped base 10 integers (\DDD). Back slashes and quotes are escaped. Tabs, carriage returns, and line feeds are converted to \t, \r, and \n respectively. @@ -494,6 +494,12 @@ example: IN The class of of records being queried. +==== dns.question.etld_plus_one + +example: amazon.co.uk. + +The effective top-level domain (eTLD) plus one more label. For example, the eTLD+1 for "foo.bar.golang.org." is "golang.org.". The data for determining the eTLD comes from an embedded copy of the data from http://publicsuffix.org. + ==== dns.answers_count type: int @@ -503,7 +509,7 @@ The number of resource records contained in the `dns.answers` field. ==== dns.answers.name -example: example.com +example: example.com. The domain name to which this resource record pertains. @@ -547,7 +553,7 @@ The number of resource records contained in the `dns.authorities` field. The `dn ==== dns.authorities.name -example: example.com +example: example.com. The domain name to which this resource record pertains. @@ -598,7 +604,7 @@ The number of resource records contained in the `dns.additionals` field. The `dn ==== dns.additionals.name -example: example.com +example: example.com. The domain name to which this resource record pertains. @@ -626,6 +632,284 @@ The time interval in seconds that this resource record may be cached before it s The data describing the resource. The meaning of this data depends on the type and class of the resource record. +=== amqp Fields + +AMQP specific event fields. + + +==== amqp.reply-code + +type: int + +example: 404 + +AMQP reply code to an error, similar to http reply-code + + +==== amqp.reply-text + +type: string + +Text expliciting the error. + + +==== amqp.class-id + +type: int + +Failing method class. + + +==== amqp.method-id + +type: int + +Failing method ID. + + +==== amqp.exchange + +type: string + +Name of the exchange. + + +==== amqp.exchange-type + +type: string + +example: fanout + +Exchange type. + + +==== amqp.passive + +type: bool + +If set, do not create exchange/queue. + + +==== amqp.durable + +type: bool + +If set, request a durable exchange/queue. + + +==== amqp.exclusive + +type: bool + +If set, request an exclusive queue. + + +==== amqp.auto-delete + +type: bool + +If set, auto-delete queue when unused. + + +==== amqp.no-wait + +type: bool + +If set, the server will not respond to the method. + + +==== amqp.consumer-tag + +Identifier for the consumer, valid within the current channel. + + +==== amqp.delivery-tag + +type: int + +The server-assigned and channel-specific delivery tag. + + +==== amqp.message-count + +type: int + +The number of messages in the queue, which will be zero for newly-declared queues. + + +==== amqp.consumer-count + +type: int + +The number of consumers of a queue. + + +==== amqp.routing-key + +type: int + +Message routing key. + + +==== amqp.no-ack + +type: bool + +If set, the server does not expect acknowledgements for messages. + + +==== amqp.no-local + +type: bool + +If set, the server will not send messages to the connection that published them. + + +==== amqp.if-unused + +type: bool + +Delete only if unused. + + +==== amqp.if-empty + +type: bool + +Delete only if empty. + + +==== amqp.queue + +type: string + +The queue name identifies the queue within the vhost. + + +==== amqp.redelivered + +type: bool + +Indicates that the message has been previously delivered to this or another client. + + +==== amqp.multiple + +type: bool + +Acknowledge multiple messages. + + +==== amqp.arguments.* + +Optional additional arguments passed to some methods. Can be of various types. + + +==== amqp.mandatory + +type: bool + +Indicates mandatory routing. + + +==== amqp.immediate + +type: bool + +Request immediate delivery. + + +==== amqp.content-type + +type: string + +example: text/plain + +MIME content type. + + +==== amqp.content-encoding + +type: string + +MIME content encoding. + + +==== amqp.headers.* + +Message header field table. + + +==== amqp.delivery-mode + +type: int + +Non-persistent (1) or persistent (2). + + +==== amqp.priority + +type: int + +Message priority, 0 to 9. + + +==== amqp.correlation-id + +type: string + +Application correlation identifier. + + +==== amqp.reply-to + +type: string + +Address to reply to. + + +==== amqp.expiration + +type: string + +Message expiration specification. + + +==== amqp.message-id + +type: string + +Application message identifier. + + +==== amqp.timestamp + +type: string + +Message timestamp. + + +==== amqp.type + +type: string + +Message type name. + + +==== amqp.user-id + +type: string + +Creating user id. + + +==== amqp.app-id + +type: string + +Creating application id. + + [[exported-fields-http]] === Http Fields diff --git a/packetbeat/etc/fields.yml b/packetbeat/etc/fields.yml index e158c38798b..4c1b61b1d90 100644 --- a/packetbeat/etc/fields.yml +++ b/packetbeat/etc/fields.yml @@ -516,7 +516,7 @@ trans_event: as escaped base 10 integers (\DDD). Back slashes and quotes are escaped. Tabs, carriage returns, and line feeds are converted to \t, \r, and \n respectively. - example: www.google.com + example: www.google.com. - name: dns.question.type description: The type of records being queried. @@ -526,6 +526,13 @@ trans_event: description: The class of of records being queried. example: IN + - name: dns.question.etld_plus_one + description: The effective top-level domain (eTLD) plus one more label. + For example, the eTLD+1 for "foo.bar.golang.org." is "golang.org.". + The data for determining the eTLD comes from an embedded copy of the + data from http://publicsuffix.org. + example: amazon.co.uk. + - name: dns.answers_count type: int description: > @@ -533,7 +540,7 @@ trans_event: - name: dns.answers.name description: The domain name to which this resource record pertains. - example: example.com + example: example.com. - name: dns.answers.type description: The type of data contained in this resource record. @@ -570,7 +577,7 @@ trans_event: - name: dns.authorities.name description: The domain name to which this resource record pertains. - example: example.com + example: example.com. - name: dns.authorities.type description: The type of data contained in this resource record. @@ -613,7 +620,7 @@ trans_event: - name: dns.additionals.name description: The domain name to which this resource record pertains. - example: example.com + example: example.com. - name: dns.additionals.type description: The type of data contained in this resource record. @@ -653,12 +660,12 @@ trans_event: - name: amqp.class-id type: int description: > - Failing method class. + Failing method class. - name: amqp.method-id type: int description: > - Failing method ID. + Failing method ID. - name: amqp.exchange type: string @@ -694,27 +701,27 @@ trans_event: - name: amqp.no-wait type: bool description: > - If set, the server will not respond to the method. + If set, the server will not respond to the method. - name: amqp.consumer-tag description: > - Identifier for the consumer, valid within the current channel. + Identifier for the consumer, valid within the current channel. - name: amqp.delivery-tag type: int description: > - The server-­assigned and channel-­specific delivery tag. + The server-assigned and channel-specific delivery tag. - name: amqp.message-count type: int description: > - The number of messages in the queue, which will be zero for - newly-­declared queues. + The number of messages in the queue, which will be zero for + newly-declared queues. - name: amqp.consumer-count type: int description: > - The number of consumers of a queue. + The number of consumers of a queue. - name: amqp.routing-key type: int @@ -724,13 +731,13 @@ trans_event: - name: amqp.no-ack type: bool description: > - If set, the server does not expect acknowledgements for messages. + If set, the server does not expect acknowledgements for messages. - name: amqp.no-local type: bool description: > - If set,  the server will not send messages to the connection that - published them. + If set, the server will not send messages to the connection that + published them. - name: amqp.if-unused type: bool @@ -745,13 +752,13 @@ trans_event: - name: amqp.queue type: string description: > - The queue name identifies the queue within the vhost. + The queue name identifies the queue within the vhost. - name: amqp.redelivered type: bool description: > - Indicates that the message has been previously delivered to this - or another client. + Indicates that the message has been previously delivered to this + or another client. - name: amqp.multiple type: bool @@ -801,7 +808,7 @@ trans_event: - name: amqp.correlation-id type: string description: > - Application correlation identifier. + Application correlation identifier. - name: amqp.reply-to type: string @@ -816,7 +823,7 @@ trans_event: - name: amqp.message-id type: string description: > - Application message identifier. + Application message identifier. - name: amqp.timestamp type: string diff --git a/packetbeat/protos/dns/dns.go b/packetbeat/protos/dns/dns.go index 59e5370e0e4..efcc5d0c9f0 100644 --- a/packetbeat/protos/dns/dns.go +++ b/packetbeat/protos/dns/dns.go @@ -22,6 +22,7 @@ import ( "github.com/elastic/beats/packetbeat/publish" mkdns "github.com/miekg/dns" + "golang.org/x/net/publicsuffix" ) const MaxDnsTupleRawSize = 16 + 16 + 2 + 2 + 4 + 1 @@ -406,11 +407,17 @@ func addDnsToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional if len(dns.Question) > 0 { q := dns.Question[0] - m["question"] = common.MapStr{ + qMapStr := common.MapStr{ "name": q.Name, "type": dnsTypeToString(q.Qtype), "class": dnsClassToString(q.Qclass), } + m["question"] = qMapStr + + eTLDPlusOne, err := publicsuffix.EffectiveTLDPlusOne(strings.TrimRight(q.Name, ".")) + if err == nil { + qMapStr["etld_plus_one"] = eTLDPlusOne + "." + } } m["answers_count"] = len(dns.Answer) diff --git a/packetbeat/protos/dns/dns_tcp_test.go b/packetbeat/protos/dns/dns_tcp_test.go index ec21ebb6969..2d5c2a659ee 100644 --- a/packetbeat/protos/dns/dns_tcp_test.go +++ b/packetbeat/protos/dns/dns_tcp_test.go @@ -41,6 +41,7 @@ var ( q_class: "IN", q_type: "A", q_name: "elastic.co.", + q_etld: "elastic.co.", answers: []string{"54.201.204.244", "54.200.185.88"}, authorities: []string{"NS-835.AWSDNS-40.NET.", "NS-1183.AWSDNS-19.ORG.", "NS-2007.AWSDNS-58.CO.UK.", "NS-66.AWSDNS-08.COM."}, request: []byte{ @@ -71,6 +72,7 @@ var ( q_class: "IN", q_type: "AXFR", q_name: "etas.com.", + q_etld: "etas.com.", answers: []string{"training2003p.", "training2003p.", "1.1.1.1", "training2003p."}, request: []byte{ 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x65, @@ -101,6 +103,7 @@ var ( q_class: "IN", q_type: "PTR", q_name: "131.252.30.192.in-addr.arpa.", + q_etld: "192.in-addr.arpa.", answers: []string{"github.com."}, authorities: []string{"ns1.p16.dynect.net.", "ns3.p16.dynect.net.", "ns4.p16.dynect.net.", "ns2.p16.dynect.net."}, request: []byte{ @@ -132,6 +135,7 @@ var ( q_name: "3.1o19ss00s2s17s4qp375sp49r830n2n4n923s8839052s7p7768s53365226pp3.659p1r741os37393" + "648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" + "1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac.sophosxl.net.", + q_etld: "sophosxl.net.", request: []byte{ 0x00, 0xed, 0x88, 0xc1, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x33, 0x3f, 0x31, 0x6f, 0x31, 0x39, 0x73, 0x73, 0x30, 0x30, 0x73, 0x32, 0x73, 0x31, 0x37, 0x73, 0x34, diff --git a/packetbeat/protos/dns/dns_test.go b/packetbeat/protos/dns/dns_test.go index dc0a93d613d..b0df466c19d 100644 --- a/packetbeat/protos/dns/dns_test.go +++ b/packetbeat/protos/dns/dns_test.go @@ -36,6 +36,7 @@ type DnsTestMessage struct { q_class string q_type string q_name string + q_etld string answers []string authorities []string additionals []string @@ -214,6 +215,7 @@ func assertRequest(t testing.TB, m common.MapStr, q DnsTestMessage) { assert.Equal(t, q.q_class, mapValue(t, m, "dns.question.class")) assert.Equal(t, q.q_type, mapValue(t, m, "dns.question.type")) assert.Equal(t, q.q_name, mapValue(t, m, "dns.question.name")) + assert.Equal(t, q.q_etld, mapValue(t, m, "dns.question.etld_plus_one")) } // Assert that the specified flags are set. diff --git a/packetbeat/protos/dns/dns_udp_test.go b/packetbeat/protos/dns/dns_udp_test.go index 91e4430da8f..8127a36a554 100644 --- a/packetbeat/protos/dns/dns_udp_test.go +++ b/packetbeat/protos/dns/dns_udp_test.go @@ -54,6 +54,7 @@ var ( q_class: "IN", q_type: "A", q_name: "elastic.co.", + q_etld: "elastic.co.", answers: []string{"54.148.130.30", "54.69.104.66"}, request: []byte{ 0x21, 0x51, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x65, 0x6c, 0x61, @@ -75,6 +76,7 @@ var ( q_class: "IN", q_type: "IXFR", q_name: "etas.com.", + q_etld: "etas.com.", answers: []string{"training2003p.", "training2003p.", "training2003p.", "training2003p.", "1.1.1.100"}, request: []byte{ @@ -111,6 +113,7 @@ var ( q_class: "IN", q_type: "PTR", q_name: "131.252.30.192.in-addr.arpa.", + q_etld: "192.in-addr.arpa.", answers: []string{"github.com."}, authorities: []string{"a.root-servers.net.", "b.root-servers.net.", "c.root-servers.net.", "d.root-servers.net.", "e.root-servers.net.", "f.root-servers.net.", "g.root-servers.net.", @@ -153,6 +156,7 @@ var ( q_name: "3.1o19ss00s2s17s4qp375sp49r830n2n4n923s8839052s7p7768s53365226pp3.659p1r741os37393" + "648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" + "1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac.sophosxl.net.", + q_etld: "sophosxl.net.", request: []byte{ 0x20, 0x2e, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x33, 0x3f, 0x31, 0x6f, 0x31, 0x39, 0x73, 0x73, 0x30, 0x30, 0x73, 0x32, 0x73, 0x31, 0x37, 0x73, 0x34, 0x71, 0x70,