From bacbc0fc2ff42ffd5b50402ae961c76738922053 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Fri, 12 Jun 2020 12:35:02 -0400 Subject: [PATCH 1/5] Add packetbeat changes --- packetbeat/docs/fields.asciidoc | 628 +++++++++++ packetbeat/pb/ecs.go | 58 ++ packetbeat/pb/event.go | 25 +- packetbeat/pb/event_test.go | 3 +- packetbeat/protos/amqp/amqp.go | 1 + packetbeat/protos/dhcpv4/dhcpv4.go | 3 + packetbeat/protos/dhcpv4/dhcpv4_test.go | 13 +- packetbeat/protos/dns/dns.go | 9 +- packetbeat/protos/dns/names_test.go | 3 +- packetbeat/protos/icmp/icmp.go | 2 + packetbeat/protos/memcache/memcache.go | 28 +- packetbeat/protos/nfs/request_handler.go | 10 +- packetbeat/protos/redis/redis.go | 6 + packetbeat/protos/tls/_meta/fields.yml | 986 ++++++++++++------ packetbeat/protos/tls/fields.go | 2 +- packetbeat/protos/tls/parse.go | 17 +- packetbeat/protos/tls/tls.go | 6 + packetbeat/protos/tls/tls_test.go | 2 +- .../golden/established_tls-expected.json | 57 +- .../golden/non_established_tls-expected.json | 13 +- .../tests/system/golden/tls_1_3-expected.json | 13 +- .../golden/tls_all_options-expected.json | 57 +- .../system/golden/tls_no_certs-expected.json | 13 +- .../golden/tls_not_detailed-expected.json | 13 +- .../test_0040_memcache_tcp_bin_basic.py | 2 +- packetbeat/tests/system/test_0050_icmp.py | 4 +- .../tests/system/test_0051_amqp_publish.py | 68 +- .../system/test_0052_amqp_emit_receive.py | 145 +-- .../system/test_0053_amqp_channel_error.py | 81 +- 29 files changed, 1746 insertions(+), 522 deletions(-) create mode 100644 packetbeat/pb/ecs.go diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 28ccba25556..cb66e939a76 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -10345,6 +10345,550 @@ Detailed TLS-specific event fields. + +*`tls.client.x509.version`*:: ++ +-- +Version of x509 format. + +type: keyword + +example: 3 + +-- + +*`tls.client.x509.version_number`*:: ++ +-- +Version of x509 format. + +type: keyword + +example: 3 + +-- + +*`tls.client.x509.serial_number`*:: ++ +-- +Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters. + + +type: keyword + +example: 55FBB9C7DEBF09809D12CCAA + +-- + +*`tls.client.x509.issuer.distinguished_name`*:: ++ +-- +Distinguished name (DN) of issuing certificate authority. + +type: keyword + +example: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert SHA2 High Assurance Server CA + +-- + +*`tls.client.x509.issuer.common_name`*:: ++ +-- +List of common name (CN) of issuing certificate authority. + +type: keyword + +example: DigiCert SHA2 High Assurance Server CA + +-- + +*`tls.client.x509.issuer.organizational_unit`*:: ++ +-- +List of organizational units (OU) of issuing certificate authority. + +type: keyword + +example: www.digicert.com + +-- + +*`tls.client.x509.issuer.organization`*:: ++ +-- +List of organizations (O) of issuing certificate authority. + +type: keyword + +example: DigiCert Inc + +-- + +*`tls.client.x509.issuer.locality`*:: ++ +-- +List of locality names (L) + +type: keyword + +example: Mountain View + +-- + +*`tls.client.x509.issuer.province`*:: ++ +-- +Province or region within country. + +type: keyword + +-- + +*`tls.client.x509.issuer.state_or_province`*:: ++ +-- +List of state or province names (ST, S, or P) + +type: keyword + +example: California + +-- + +*`tls.client.x509.issuer.country`*:: ++ +-- +List of country (C) codes + +type: keyword + +example: US + +-- + +*`tls.client.x509.signature_algorithm`*:: ++ +-- +Identifier for certificate signature algorithm. Recommend using names found in Go Lang Crypto library (See https://github.com/golang/go/blob/go1.14/src/crypto/x509/x509.go#L337-L353). + +type: keyword + +example: SHA256-RSA + +-- + +*`tls.client.x509.not_before`*:: ++ +-- +Time at which the certificate is first considered valid. + +type: date + +example: 2019-08-16 01:40:25 + +-- + +*`tls.client.x509.not_after`*:: ++ +-- +Time at which the certificate is no longer considered valid. + +type: date + +example: 2020-07-16 03:15:39 + +-- + +*`tls.client.x509.subject.distinguished_name`*:: ++ +-- +Distinguished name (DN) of the certificate subject entity. + +type: keyword + +example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + +-- + +*`tls.client.x509.subject.common_name`*:: ++ +-- +List of common names (CN) of subject. + +type: keyword + +example: r2.shared.global.fastly.net + +-- + +*`tls.client.x509.subject.organizational_unit`*:: ++ +-- +List of organizational units (OU) of subject. + +type: keyword + +-- + +*`tls.client.x509.subject.organization`*:: ++ +-- +List of organizations (O) of subject. + +type: keyword + +example: Fastly, Inc. + +-- + +*`tls.client.x509.subject.locality`*:: ++ +-- +List of locality names (L) + +type: keyword + +example: San Francisco + +-- + +*`tls.client.x509.subject.province`*:: ++ +-- +Province or region within country. + +type: keyword + +-- + +*`tls.client.x509.subject.state_or_province`*:: ++ +-- +List of state or province names (ST, S, or P) + +type: keyword + +example: California + +-- + +*`tls.client.x509.subject.country`*:: ++ +-- +List of country (C) code + +type: keyword + +example: US + +-- + +*`tls.client.x509.public_key_algorithm`*:: ++ +-- +Algorithm used to generate the public key. + +type: keyword + +example: RSA + +-- + +*`tls.client.x509.public_key_size`*:: ++ +-- +The size of the public key space in bits. + +type: long + +example: 2048 + +-- + +*`tls.client.x509.alternative_names`*:: ++ +-- +List of subject alternative names (SAN). Name types vary by certificate authority and certificate type but commonly contain IP addresses, DNS names (and wildcards), and email addresses. + +type: keyword + +example: *.elastic.co + +-- + + + +*`tls.server.x509.version`*:: ++ +-- +Version of x509 format. + +type: keyword + +example: 3 + +-- + +*`tls.server.x509.version_number`*:: ++ +-- +Version of x509 format. + +type: keyword + +example: 3 + +-- + +*`tls.server.x509.serial_number`*:: ++ +-- +Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters. + + +type: keyword + +example: 55FBB9C7DEBF09809D12CCAA + +-- + +*`tls.server.x509.issuer.distinguished_name`*:: ++ +-- +Distinguished name (DN) of issuing certificate authority. + +type: keyword + +example: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert SHA2 High Assurance Server CA + +-- + +*`tls.server.x509.issuer.common_name`*:: ++ +-- +List of common name (CN) of issuing certificate authority. + +type: keyword + +example: DigiCert SHA2 High Assurance Server CA + +-- + +*`tls.server.x509.issuer.organizational_unit`*:: ++ +-- +List of organizational units (OU) of issuing certificate authority. + +type: keyword + +example: www.digicert.com + +-- + +*`tls.server.x509.issuer.organization`*:: ++ +-- +List of organizations (O) of issuing certificate authority. + +type: keyword + +example: DigiCert Inc + +-- + +*`tls.server.x509.issuer.locality`*:: ++ +-- +List of locality names (L) + +type: keyword + +example: Mountain View + +-- + +*`tls.server.x509.issuer.province`*:: ++ +-- +Province or region within country. + +type: keyword + +-- + +*`tls.server.x509.issuer.state_or_province`*:: ++ +-- +List of state or province names (ST, S, or P) + +type: keyword + +example: California + +-- + +*`tls.server.x509.issuer.country`*:: ++ +-- +List of country (C) codes + +type: keyword + +example: US + +-- + +*`tls.server.x509.signature_algorithm`*:: ++ +-- +Identifier for certificate signature algorithm. Recommend using names found in Go Lang Crypto library (See https://github.com/golang/go/blob/go1.14/src/crypto/x509/x509.go#L337-L353). + +type: keyword + +example: SHA256-RSA + +-- + +*`tls.server.x509.not_before`*:: ++ +-- +Time at which the certificate is first considered valid. + +type: date + +example: 2019-08-16 01:40:25 + +-- + +*`tls.server.x509.not_after`*:: ++ +-- +Time at which the certificate is no longer considered valid. + +type: date + +example: 2020-07-16 03:15:39 + +-- + +*`tls.server.x509.subject.distinguished_name`*:: ++ +-- +Distinguished name (DN) of the certificate subject entity. + +type: keyword + +example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + +-- + +*`tls.server.x509.subject.common_name`*:: ++ +-- +List of common names (CN) of subject. + +type: keyword + +example: r2.shared.global.fastly.net + +-- + +*`tls.server.x509.subject.organizational_unit`*:: ++ +-- +List of organizational units (OU) of subject. + +type: keyword + +-- + +*`tls.server.x509.subject.organization`*:: ++ +-- +List of organizations (O) of subject. + +type: keyword + +example: Fastly, Inc. + +-- + +*`tls.server.x509.subject.locality`*:: ++ +-- +List of locality names (L) + +type: keyword + +example: San Francisco + +-- + +*`tls.server.x509.subject.province`*:: ++ +-- +Province or region within country. + +type: keyword + +-- + +*`tls.server.x509.subject.state_or_province`*:: ++ +-- +List of state or province names (ST, S, or P) + +type: keyword + +example: California + +-- + +*`tls.server.x509.subject.country`*:: ++ +-- +List of country (C) code + +type: keyword + +example: US + +-- + +*`tls.server.x509.public_key_algorithm`*:: ++ +-- +Algorithm used to generate the public key. + +type: keyword + +example: RSA + +-- + +*`tls.server.x509.public_key_size`*:: ++ +-- +The size of the public key space in bits. + +type: long + +example: 2048 + +-- + +*`tls.server.x509.alternative_names`*:: ++ +-- +List of subject alternative names (SAN). Name types vary by certificate authority and certificate type but commonly contain IP addresses, DNS names (and wildcards), and email addresses. + +type: keyword + +example: *.elastic.co + +-- + + *`tls.detailed.version`*:: + -- @@ -10594,6 +11138,17 @@ type: long -- +*`tls.detailed.client_certificate.version_number`*:: ++ +-- +Version of x509 format. + +type: keyword + +example: 3 + +-- + *`tls.detailed.client_certificate.serial_number`*:: + -- @@ -10719,6 +11274,17 @@ type: keyword -- +*`tls.detailed.client_certificate.subject.distinguished_name`*:: ++ +-- +Distinguished name (DN) of the certificate subject entity. + +type: keyword + +example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + +-- + [float] === issuer @@ -10779,6 +11345,17 @@ type: keyword -- +*`tls.detailed.client_certificate.issuer.distinguished_name`*:: ++ +-- +Distinguished name (DN) of the certificate issuer entity. + +type: keyword + +example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + +-- + [float] === server_certificate @@ -10794,6 +11371,17 @@ type: long -- +*`tls.detailed.server_certificate.version_number`*:: ++ +-- +Version of x509 format. + +type: keyword + +example: 3 + +-- + *`tls.detailed.server_certificate.serial_number`*:: + -- @@ -10901,6 +11489,15 @@ type: keyword -- +*`tls.detailed.server_certificate.subject.state_or_province`*:: ++ +-- +Province or region within country. + +type: keyword + +-- + *`tls.detailed.server_certificate.subject.common_name`*:: + -- @@ -10919,6 +11516,17 @@ type: keyword -- +*`tls.detailed.server_certificate.subject.distinguished_name`*:: ++ +-- +Distinguished name (DN) of the certificate subject entity. + +type: keyword + +example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + +-- + [float] === issuer @@ -10961,6 +11569,15 @@ type: keyword -- +*`tls.detailed.server_certificate.issuer.state_or_province`*:: ++ +-- +Province or region within country. + +type: keyword + +-- + *`tls.detailed.server_certificate.issuer.common_name`*:: + -- @@ -10979,6 +11596,17 @@ type: keyword -- +*`tls.detailed.server_certificate.issuer.distinguished_name`*:: ++ +-- +Distinguished name (DN) of the certificate issuer entity. + +type: keyword + +example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + +-- + *`tls.detailed.server_certificate_chain`*:: + -- diff --git a/packetbeat/pb/ecs.go b/packetbeat/pb/ecs.go new file mode 100644 index 00000000000..05dd32e5b71 --- /dev/null +++ b/packetbeat/pb/ecs.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package pb + +import "time" + +type ecsEvent struct { + ID string `ecs:"id"` + Code string `ecs:"code"` + Kind string `ecs:"kind"` + // overridden because this needs to be an array + Category []string `ecs:"category"` + Action string `ecs:"action"` + Outcome string `ecs:"outcome"` + // overridden because this needs to be an array + Type []string `ecs:"type"` + Module string `ecs:"module"` + Dataset string `ecs:"dataset"` + Provider string `ecs:"provider"` + Severity int64 `ecs:"severity"` + Original string `ecs:"original"` + Hash string `ecs:"hash"` + Duration time.Duration `ecs:"duration"` + Sequence int64 `ecs:"sequence"` + Timezone string `ecs:"timezone"` + Created time.Time `ecs:"created"` + Start time.Time `ecs:"start"` + End time.Time `ecs:"end"` + RiskScore float64 `ecs:"risk_score"` + RiskScoreNorm float64 `ecs:"risk_score_norm"` + Ingested time.Time `ecs:"ingested"` + Reference string `ecs:"reference"` + Url string `ecs:"url"` +} + +type ecsRelated struct { + IP []string `ecs:"ip"` + User []string `ecs:"user"` + Hash []string `ecs:"hash"` + + // for de-dup + ipSet map[string]struct{} +} diff --git a/packetbeat/pb/event.go b/packetbeat/pb/event.go index d4330b542ea..cbc615e5539 100644 --- a/packetbeat/pb/event.go +++ b/packetbeat/pb/event.go @@ -54,8 +54,9 @@ type Fields struct { Destination *ecs.Destination `ecs:"destination"` Client *ecs.Client `ecs:"client"` Server *ecs.Server `ecs:"server"` + Related *ecsRelated `ecs:"related"` Network ecs.Network `ecs:"network"` - Event ecs.Event `ecs:"event"` + Event ecsEvent `ecs:"event"` SourceProcess *ecs.Process `ecs:"source.process"` DestinationProcess *ecs.Process `ecs:"destination.process"` @@ -72,10 +73,11 @@ type Fields struct { // NewFields returns a new Fields value. func NewFields() *Fields { return &Fields{ - Event: ecs.Event{ + Event: ecsEvent{ Duration: -1, Kind: "event", - Category: "network_traffic", + Type: []string{"connection", "protocol"}, + Category: []string{"network_traffic", "network"}, }, } } @@ -112,6 +114,7 @@ func (f *Fields) SetSource(endpoint *common.Endpoint) { if f.Source == nil { f.Source = &ecs.Source{} } + f.AddIP(endpoint.IP) f.Source.IP = endpoint.IP f.Source.Port = int64(endpoint.Port) f.Source.Domain = endpoint.Domain @@ -126,6 +129,7 @@ func (f *Fields) SetDestination(endpoint *common.Endpoint) { if f.Destination == nil { f.Destination = &ecs.Destination{} } + f.AddIP(endpoint.IP) f.Destination.IP = endpoint.IP f.Destination.Port = int64(endpoint.Port) f.Destination.Domain = endpoint.Domain @@ -135,6 +139,21 @@ func (f *Fields) SetDestination(endpoint *common.Endpoint) { } } +// AddIP adds the given ip addresses to the related ECS IP field +func (f *Fields) AddIP(ip ...string) { + if f.Related == nil { + f.Related = &ecsRelated{ + ipSet: make(map[string]struct{}), + } + } + for _, ipAddress := range ip { + if _, ok := f.Related.ipSet[ipAddress]; !ok { + f.Related.ipSet[ipAddress] = struct{}{} + f.Related.IP = append(f.Related.IP, ipAddress) + } + } +} + func makeProcess(p *common.Process) *ecs.Process { return &ecs.Process{ Name: p.Name, diff --git a/packetbeat/pb/event_test.go b/packetbeat/pb/event_test.go index b44bd4c5c4c..ff21e8769b6 100644 --- a/packetbeat/pb/event_test.go +++ b/packetbeat/pb/event_test.go @@ -41,7 +41,8 @@ func TestMarshalMapStr(t *testing.T) { assert.Equal(t, common.MapStr{ "event": common.MapStr{ "kind": "event", - "category": "network_traffic", + "category": []string{"network_traffic", "network"}, + "type": []string{"connection", "protocol"}, }, "source": common.MapStr{"ip": "127.0.0.1"}, }, m) diff --git a/packetbeat/protos/amqp/amqp.go b/packetbeat/protos/amqp/amqp.go index 8b5689e4f2e..c361c3e7fe6 100644 --- a/packetbeat/protos/amqp/amqp.go +++ b/packetbeat/protos/amqp/amqp.go @@ -435,6 +435,7 @@ func (amqp *amqpPlugin) publishTransaction(t *amqpTransaction) { pbf.Event.Start = t.ts pbf.Event.End = t.endTime pbf.Event.Dataset = "amqp" + pbf.Event.Action = "amqp." + t.method pbf.Network.Protocol = pbf.Event.Dataset pbf.Network.Transport = "tcp" pbf.Error.Message = t.notes diff --git a/packetbeat/protos/dhcpv4/dhcpv4.go b/packetbeat/protos/dhcpv4/dhcpv4.go index a50108a83c6..10d299aea76 100644 --- a/packetbeat/protos/dhcpv4/dhcpv4.go +++ b/packetbeat/protos/dhcpv4/dhcpv4.go @@ -133,12 +133,15 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event { if !v4.ClientIPAddr().IsUnspecified() { dhcpData.Put("client_ip", v4.ClientIPAddr().String()) + pbf.AddIP(v4.ClientIPAddr().String()) } if !v4.YourIPAddr().IsUnspecified() { dhcpData.Put("assigned_ip", v4.YourIPAddr().String()) + pbf.AddIP(v4.YourIPAddr().String()) } if !v4.GatewayIPAddr().IsUnspecified() { dhcpData.Put("relay_ip", v4.GatewayIPAddr().String()) + pbf.AddIP(v4.GatewayIPAddr().String()) } if serverName := v4.ServerHostNameToString(); serverName != "" { dhcpData.Put("server_name", serverName) diff --git a/packetbeat/protos/dhcpv4/dhcpv4_test.go b/packetbeat/protos/dhcpv4/dhcpv4_test.go index 72aade5014e..704c4d2bece 100644 --- a/packetbeat/protos/dhcpv4/dhcpv4_test.go +++ b/packetbeat/protos/dhcpv4/dhcpv4_test.go @@ -117,7 +117,8 @@ func TestParseDHCPRequest(t *testing.T) { "port": 67, }, "event": common.MapStr{ - "category": "network_traffic", + "category": []string{"network_traffic", "network"}, + "type": []string{"connection", "protocol"}, "dataset": "dhcpv4", "kind": "event", "start": pkt.Ts, @@ -129,6 +130,9 @@ func TestParseDHCPRequest(t *testing.T) { "bytes": 272, "community_id": "1:t9O1j0qj71O4wJM7gnaHtgmfev8=", }, + "related": common.MapStr{ + "ip": []string{"0.0.0.0", "255.255.255.255"}, + }, "dhcpv4": common.MapStr{ "client_mac": "00:0b:82:01:fc:42", "flags": "unicast", @@ -197,7 +201,8 @@ func TestParseDHCPACK(t *testing.T) { "bytes": 300, }, "event": common.MapStr{ - "category": "network_traffic", + "category": []string{"network_traffic", "network"}, + "type": []string{"connection", "protocol"}, "dataset": "dhcpv4", "kind": "event", "start": pkt.Ts, @@ -209,7 +214,9 @@ func TestParseDHCPACK(t *testing.T) { "bytes": 300, "community_id": "1:VbRSZnvQqvLiQRhYHLrdVI17sLQ=", }, - + "related": common.MapStr{ + "ip": []string{"192.168.0.1", "192.168.0.10"}, + }, "dhcpv4": common.MapStr{ "assigned_ip": "192.168.0.10", "client_mac": "00:0b:82:01:fc:42", diff --git a/packetbeat/protos/dns/dns.go b/packetbeat/protos/dns/dns.go index 29efcc9bcdc..8fbf402b6b4 100644 --- a/packetbeat/protos/dns/dns.go +++ b/packetbeat/protos/dns/dns.go @@ -391,7 +391,7 @@ func (dns *dnsPlugin) publishTransaction(t *dnsTransaction) { fields["query"] = dnsQuestionToString(t.request.data.Question[0]) fields["resource"] = t.request.data.Question[0].Name } - addDNSToMapStr(dnsEvent, t.response.data, dns.includeAuthorities, + addDNSToMapStr(dnsEvent, pbf, t.response.data, dns.includeAuthorities, dns.includeAdditionals) if t.response.data.Rcode == 0 { @@ -414,7 +414,7 @@ func (dns *dnsPlugin) publishTransaction(t *dnsTransaction) { fields["query"] = dnsQuestionToString(t.request.data.Question[0]) fields["resource"] = t.request.data.Question[0].Name } - addDNSToMapStr(dnsEvent, t.request.data, dns.includeAuthorities, + addDNSToMapStr(dnsEvent, pbf, t.request.data, dns.includeAuthorities, dns.includeAdditionals) if dns.sendRequest { @@ -430,7 +430,7 @@ func (dns *dnsPlugin) publishTransaction(t *dnsTransaction) { fields["query"] = dnsQuestionToString(t.response.data.Question[0]) fields["resource"] = t.response.data.Question[0].Name } - addDNSToMapStr(dnsEvent, t.response.data, dns.includeAuthorities, + addDNSToMapStr(dnsEvent, pbf, t.response.data, dns.includeAuthorities, dns.includeAdditionals) if dns.sendResponse { fields["response"] = dnsToString(t.response.data) @@ -448,7 +448,7 @@ func (dns *dnsPlugin) expireTransaction(t *dnsTransaction) { } // Adds the DNS message data to the supplied MapStr. -func addDNSToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional bool) { +func addDNSToMapStr(m common.MapStr, pbf *pb.Fields, dns *mkdns.Msg, authority bool, additional bool) { m["id"] = dns.Id m["op_code"] = dnsOpCodeToString(dns.Opcode) @@ -533,6 +533,7 @@ func addDNSToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional m["answers"], resolvedIPs = rrsToMapStrs(dns.Answer, true) if len(resolvedIPs) > 0 { m["resolved_ip"] = resolvedIPs + pbf.AddIP(resolvedIPs...) } } diff --git a/packetbeat/protos/dns/names_test.go b/packetbeat/protos/dns/names_test.go index 9066082cab8..5d5d9323eb5 100644 --- a/packetbeat/protos/dns/names_test.go +++ b/packetbeat/protos/dns/names_test.go @@ -32,6 +32,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/packetbeat/pb" ) type dnsTestMsg struct { @@ -110,7 +111,7 @@ func assertDNSMessage(t testing.TB, q dnsTestMsg) { } mapStr := common.MapStr{} - addDNSToMapStr(mapStr, dns, true, true) + addDNSToMapStr(mapStr, pb.NewFields(), dns, true, true) if q.question != nil { for k, v := range q.question { assert.NotNil(t, mapStr["question"].(common.MapStr)[k]) diff --git a/packetbeat/protos/icmp/icmp.go b/packetbeat/protos/icmp/icmp.go index 3c4b692ca82..6fb210fd871 100644 --- a/packetbeat/protos/icmp/icmp.go +++ b/packetbeat/protos/icmp/icmp.go @@ -287,7 +287,9 @@ func (icmp *icmpPlugin) publishTransaction(trans *icmpTransaction) { evt, pbf := pb.NewBeatEvent(trans.ts) pbf.Source = &ecs.Source{IP: trans.tuple.srcIP.String()} pbf.Destination = &ecs.Destination{IP: trans.tuple.dstIP.String()} + pbf.AddIP(trans.tuple.srcIP.String(), trans.tuple.dstIP.String()) pbf.Event.Dataset = "icmp" + pbf.Event.Type = []string{"connection"} pbf.Error.Message = trans.notes // common fields - group "event" diff --git a/packetbeat/protos/memcache/memcache.go b/packetbeat/protos/memcache/memcache.go index 74bbd0f3f89..e59550287a5 100644 --- a/packetbeat/protos/memcache/memcache.go +++ b/packetbeat/protos/memcache/memcache.go @@ -22,6 +22,7 @@ package memcache import ( "encoding/json" "math" + "strings" "time" "github.com/elastic/beats/v7/libbeat/beat" @@ -388,12 +389,18 @@ func (t *transaction) Event(event *beat.Event) error { mc := common.MapStr{} event.Fields["memcache"] = mc + msg := t.request + if msg == nil { + msg = t.response + } + if t.request != nil { _, err := t.request.SubEvent("request", mc) if err != nil { logp.Warn("error filling transaction request: %v", err) return err } + event.Fields["event.action"] = "memcache." + strings.ToLower(t.request.command.typ.String()) } if t.response != nil { _, err := t.response.SubEvent("response", mc) @@ -401,12 +408,12 @@ func (t *transaction) Event(event *beat.Event) error { logp.Warn("error filling transaction response: %v", err) return err } + normalized := normalizeEventOutcome(memcacheStatusCode(t.response.status).String()) + if normalized != "" { + event.Fields["event.outcome"] = normalized + } } - msg := t.request - if msg == nil { - msg = t.response - } if msg == nil { mc["protocol_type"] = "unknown" } else { @@ -420,6 +427,19 @@ func (t *transaction) Event(event *beat.Event) error { return nil } +func normalizeEventOutcome(outcome string) string { + switch outcome { + case "Fail": + return "failure" + case "UNKNOWN": + return "unknown" + case "Success": + return "success" + default: + return "" + } +} + func computeTransactionStatus(requ, resp *message) string { switch { case requ == nil && resp != nil: diff --git a/packetbeat/protos/nfs/request_handler.go b/packetbeat/protos/nfs/request_handler.go index 241e0dca482..b6ce1cdbdb5 100644 --- a/packetbeat/protos/nfs/request_handler.go +++ b/packetbeat/protos/nfs/request_handler.go @@ -143,7 +143,12 @@ func (r *rpc) handleCall(xid string, xdr *xdr, ts time.Time, tcptuple *common.TC pbf: pbf, event: evt, } - fields["nfs"] = nfs.getRequestInfo(xdr) + info := nfs.getRequestInfo(xdr) + fields["nfs"] = info + + if opcode, ok := info["opcode"].(string); ok && opcode != "" { + pbf.Event.Action = "nfs." + opcode + } // use xid+src ip to uniquely identify request reqID := xid + tcptuple.SrcIP.String() @@ -190,7 +195,8 @@ func (r *rpc) handleReply(xid string, xdr *xdr, ts time.Time, tcptuple *common.T if status == 0 { nfsInfo := fields["nfs"].(common.MapStr) nfsInfo["status"] = nfs.getNFSReplyStatus(xdr) + } else { + nfs.pbf.Event.Outcome = "failure" } - r.results(nfs.event) } } diff --git a/packetbeat/protos/redis/redis.go b/packetbeat/protos/redis/redis.go index e7feac645c5..9012f4ebda2 100644 --- a/packetbeat/protos/redis/redis.go +++ b/packetbeat/protos/redis/redis.go @@ -19,6 +19,7 @@ package redis import ( "bytes" + "strings" "time" "github.com/elastic/beats/v7/libbeat/beat" @@ -326,6 +327,11 @@ func (redis *redisPlugin) newTransaction(requ, resp *redisMessage) beat.Event { fields["response"] = resp.message } + pbf.Event.Action = "redis." + strings.ToLower(fields["method"].(string)) + if resp.isError { + pbf.Event.Outcome = "failure" + } + return evt } diff --git a/packetbeat/protos/tls/_meta/fields.yml b/packetbeat/protos/tls/_meta/fields.yml index 89f0c0c7b6f..259540c695a 100644 --- a/packetbeat/protos/tls/_meta/fields.yml +++ b/packetbeat/protos/tls/_meta/fields.yml @@ -1,354 +1,654 @@ - key: tls_detailed - title: "Detailed TLS" + title: 'Detailed TLS' description: > Detailed TLS-specific event fields. fields: - name: tls type: group fields: + # get rid of this when we upgrade to ECS 1.6 + - name: client + type: group + fields: + - name: x509 + type: group + default_fields: false + fields: + - name: version + type: keyword + description: Version of x509 format. + example: 3 + + - name: version_number + type: keyword + description: Version of x509 format. + example: 3 + + - name: serial_number + type: keyword + description: > + Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be + formatted without colons and uppercase characters. + example: 55FBB9C7DEBF09809D12CCAA + + - name: issuer.distinguished_name + type: keyword + description: Distinguished name (DN) of issuing certificate authority. + example: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert SHA2 High Assurance Server CA + + - name: issuer.common_name + type: keyword + description: List of common name (CN) of issuing certificate authority. + example: DigiCert SHA2 High Assurance Server CA + + - name: issuer.organizational_unit + type: keyword + description: List of organizational units (OU) of issuing certificate authority. + example: www.digicert.com + + - name: issuer.organization + type: keyword + description: List of organizations (O) of issuing certificate authority. + example: DigiCert Inc + + - name: issuer.locality + type: keyword + description: List of locality names (L) + example: Mountain View + + - name: issuer.province + type: keyword + description: Province or region within country. + + - name: issuer.state_or_province + type: keyword + description: List of state or province names (ST, S, or P) + example: California + + - name: issuer.country + type: keyword + description: List of country (C) codes + example: US + + - name: signature_algorithm + type: keyword + description: Identifier for certificate signature algorithm. Recommend using names found in Go Lang Crypto library (See https://github.com/golang/go/blob/go1.14/src/crypto/x509/x509.go#L337-L353). + example: SHA256-RSA + + - name: not_before + type: date + description: Time at which the certificate is first considered valid. + example: 2019-08-16T01:40:25Z + + - name: not_after + type: date + description: Time at which the certificate is no longer considered valid. + example: 2020-07-16T03:15:39Z + + - name: subject.distinguished_name + type: keyword + description: Distinguished name (DN) of the certificate subject entity. + example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + + - name: subject.common_name + type: keyword + description: List of common names (CN) of subject. + example: r2.shared.global.fastly.net + + - name: subject.organizational_unit + type: keyword + description: List of organizational units (OU) of subject. + + - name: subject.organization + type: keyword + description: List of organizations (O) of subject. + example: Fastly, Inc. + + - name: subject.locality + type: keyword + description: List of locality names (L) + example: San Francisco + + - name: subject.province + type: keyword + description: Province or region within country. + + - name: subject.state_or_province + type: keyword + description: List of state or province names (ST, S, or P) + example: California + + - name: subject.country + type: keyword + description: List of country (C) code + example: US + + - name: public_key_algorithm + type: keyword + description: Algorithm used to generate the public key. + example: RSA + + - name: public_key_size + type: long + description: The size of the public key space in bits. + example: 2048 + + - name: alternative_names + type: keyword + description: List of subject alternative names (SAN). Name types vary by certificate authority and certificate type but commonly contain IP addresses, DNS names (and wildcards), and email addresses. + example: '*.elastic.co' + + # get rid of this when we upgrade to ECS 1.6 + - name: server + type: group + fields: + - name: x509 + type: group + default_fields: false + fields: + - name: version + type: keyword + description: Version of x509 format. + example: 3 + + - name: version_number + type: keyword + description: Version of x509 format. + example: 3 + + - name: serial_number + type: keyword + description: > + Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be + formatted without colons and uppercase characters. + example: 55FBB9C7DEBF09809D12CCAA + + - name: issuer.distinguished_name + type: keyword + description: Distinguished name (DN) of issuing certificate authority. + example: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert SHA2 High Assurance Server CA + + - name: issuer.common_name + type: keyword + description: List of common name (CN) of issuing certificate authority. + example: DigiCert SHA2 High Assurance Server CA + + - name: issuer.organizational_unit + type: keyword + description: List of organizational units (OU) of issuing certificate authority. + example: www.digicert.com + + - name: issuer.organization + type: keyword + description: List of organizations (O) of issuing certificate authority. + example: DigiCert Inc + + - name: issuer.locality + type: keyword + description: List of locality names (L) + example: Mountain View + + - name: issuer.province + type: keyword + description: Province or region within country. + + - name: issuer.state_or_province + type: keyword + description: List of state or province names (ST, S, or P) + example: California + + - name: issuer.country + type: keyword + description: List of country (C) codes + example: US + + - name: signature_algorithm + type: keyword + description: Identifier for certificate signature algorithm. Recommend using names found in Go Lang Crypto library (See https://github.com/golang/go/blob/go1.14/src/crypto/x509/x509.go#L337-L353). + example: SHA256-RSA + + - name: not_before + type: date + description: Time at which the certificate is first considered valid. + example: 2019-08-16T01:40:25Z + + - name: not_after + type: date + description: Time at which the certificate is no longer considered valid. + example: 2020-07-16T03:15:39Z + + - name: subject.distinguished_name + type: keyword + description: Distinguished name (DN) of the certificate subject entity. + example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + + - name: subject.common_name + type: keyword + description: List of common names (CN) of subject. + example: r2.shared.global.fastly.net + + - name: subject.organizational_unit + type: keyword + description: List of organizational units (OU) of subject. + + - name: subject.organization + type: keyword + description: List of organizations (O) of subject. + example: Fastly, Inc. + + - name: subject.locality + type: keyword + description: List of locality names (L) + example: San Francisco + + - name: subject.province + type: keyword + description: Province or region within country. + + - name: subject.state_or_province + type: keyword + description: List of state or province names (ST, S, or P) + example: California + + - name: subject.country + type: keyword + description: List of country (C) code + example: US + + - name: public_key_algorithm + type: keyword + description: Algorithm used to generate the public key. + example: RSA + + - name: public_key_size + type: long + description: The size of the public key space in bits. + example: 2048 + + - name: alternative_names + type: keyword + description: List of subject alternative names (SAN). Name types vary by certificate authority and certificate type but commonly contain IP addresses, DNS names (and wildcards), and email addresses. + example: '*.elastic.co' + - name: detailed type: group default_fields: false fields: - - name: version - type: keyword - description: > - The version of the TLS protocol used. - example: "TLS 1.3" - - - name: resumption_method - type: keyword - description: > - If the session has been resumed, the underlying method used. One of - "id" for TLS session ID or "ticket" for TLS ticket extension. - - - name: client_certificate_requested - type: boolean - description: > - Whether the server has requested the client to authenticate itself - using a client certificate. - - - name: client_hello - type: group - fields: - - name: version - type: keyword - description: > - The version of the TLS protocol by which the client wishes to - communicate during this session. - - - name: session_id - type: keyword - description: > - Unique number to identify the session for the corresponding - connection with the client. - - - name: supported_compression_methods - type: keyword - description: > - The list of compression methods the client supports. - See https://www.iana.org/assignments/comp-meth-ids/comp-meth-ids.xhtml - - - name: extensions - type: group - description: The hello extensions provided by the client. - fields: - - name: server_name_indication - type: keyword - description: List of hostnames - - - name: application_layer_protocol_negotiation - type: keyword - description: > - List of application-layer protocols the client is willing to use. - - - name: session_ticket - type: keyword - description: > - Length of the session ticket, if provided, or an empty string - to advertise support for tickets. - - - name: supported_versions - type: keyword - description: > - List of TLS versions that the client is willing to use. - - - name: supported_groups - type: keyword - description: > - List of Elliptic Curve Cryptography (ECC) curve groups - supported by the client. - - - name: signature_algorithms - type: keyword - description: > - List of signature algorithms that may be use in digital - signatures. - - - name: ec_points_formats - type: keyword - description: > - List of Elliptic Curve (EC) point formats. Indicates the - set of point formats that the client can parse. - - - name: _unparsed_ - type: keyword - description: > - List of extensions that were left unparsed by Packetbeat. - - - name: server_hello - type: group - fields: - - name: version - type: keyword - description: > - The version of the TLS protocol that is used for this session. - It is the highest version supported by the server not exceeding - the version requested in the client hello. - - - name: selected_compression_method - type: keyword - description: > - The compression method selected by the server from the list - provided in the client hello. - - - name: session_id - type: keyword - description: > - Unique number to identify the session for the corresponding - connection with the client. - - - name: extensions - type: group - description: The hello extensions provided by the server. - fields: - - name: application_layer_protocol_negotiation - type: keyword - description: Negotiated application layer protocol - - - name: session_ticket - type: keyword - description: > - Used to announce that a session ticket will be provided - by the server. Always an empty string. - - - name: supported_versions - type: keyword - description: > - Negotiated TLS version to be used. - - - name: ec_points_formats - type: keyword - description: > - List of Elliptic Curve (EC) point formats. Indicates the - set of point formats that the server can parse. - - - name: _unparsed_ - type: keyword - description: > - List of extensions that were left unparsed by Packetbeat. - - - name: client_certificate - type: group - description: Certificate provided by the client for authentication. - fields: - - - name: version - type: long - description: X509 format version. - - - name: serial_number - type: keyword - description: The certificate's serial number. - - - name: not_before - type: date - description: Date before which the certificate is not valid. - - - name: not_after - type: date - description: Date after which the certificate expires. - - - name: public_key_algorithm - type: keyword - description: > - The algorithm used for this certificate's public key. - One of RSA, DSA or ECDSA. - - - name: public_key_size - type: long - description: Size of the public key. - - - name: signature_algorithm - type: keyword - description: > - The algorithm used for the certificate's signature. - - - name: alternative_names - type: keyword - description: Subject Alternative Names for this certificate. - - - name: subject - type: group - description: Subject represented by this certificate. - fields: - - name: country - type: keyword - description: Country code. - - - name: organization - type: keyword - description: Organization name. - - - name: organizational_unit - type: keyword - description: Unit within organization. - - - name: province - type: keyword - description: Province or region within country. - - - name: common_name - type: keyword - description: Name or host name identified by the certificate. - - - name: locality - type: keyword - description: Locality. - - - name: issuer - type: group - description: Entity that issued and signed this certificate. - fields: - - name: country - type: keyword - description: Country code. - - - name: organization - type: keyword - description: Organization name. - - - name: organizational_unit - type: keyword - description: Unit within organization. - - - name: province - type: keyword - description: Province or region within country. - - - name: common_name - type: keyword - description: Name or host name identified by the certificate. - - - name: locality - type: keyword - description: Locality. - - - name: server_certificate - type: group - description: Certificate provided by the server for authentication. - fields: - - - name: version - type: long - description: X509 format version. - - - name: serial_number - type: keyword - description: The certificate's serial number. - - - name: not_before - type: date - description: Date before which the certificate is not valid. - - - name: not_after - type: date - description: Date after which the certificate expires. - - - name: public_key_algorithm - type: keyword - description: > - The algorithm used for this certificate's public key. - One of RSA, DSA or ECDSA. - - - name: public_key_size - type: long - description: Size of the public key. - - - name: signature_algorithm - type: keyword - description: > - The algorithm used for the certificate's signature. - - - name: alternative_names - type: keyword - description: Subject Alternative Names for this certificate. - - - name: subject - type: group - description: Subject represented by this certificate. - fields: - - name: country - type: keyword - description: Country code. - - - name: organization - type: keyword - description: Organization name. - - - name: organizational_unit - type: keyword - description: Unit within organization. - - - name: province - type: keyword - description: Province or region within country. - - - name: common_name - type: keyword - description: Name or host name identified by the certificate. - - - name: locality - type: keyword - description: Locality. - - - name: issuer - type: group - description: Entity that issued and signed this certificate. - fields: - - name: country - type: keyword - description: Country code. - - - name: organization - type: keyword - description: Organization name. - - - name: organizational_unit - type: keyword - description: Unit within organization. - - - name: province - type: keyword - description: Province or region within country. - - - name: common_name - type: keyword - description: Name or host name identified by the certificate. - - - name: locality - type: keyword - description: Locality. - - - name: server_certificate_chain - type: array - description: Chain of trust for the server certificate. - - - name: client_certificate_chain - type: array - description: Chain of trust for the client certificate. - - - name: alert_types - type: keyword - description: > - An array containing the TLS alert type for every alert received. + - name: version + type: keyword + description: > + The version of the TLS protocol used. + example: 'TLS 1.3' + + - name: resumption_method + type: keyword + description: > + If the session has been resumed, the underlying method used. One of + "id" for TLS session ID or "ticket" for TLS ticket extension. + + - name: client_certificate_requested + type: boolean + description: > + Whether the server has requested the client to authenticate itself + using a client certificate. + + - name: client_hello + type: group + fields: + - name: version + type: keyword + description: > + The version of the TLS protocol by which the client wishes to + communicate during this session. + + - name: session_id + type: keyword + description: > + Unique number to identify the session for the corresponding + connection with the client. + + - name: supported_compression_methods + type: keyword + description: > + The list of compression methods the client supports. + See https://www.iana.org/assignments/comp-meth-ids/comp-meth-ids.xhtml + + - name: extensions + type: group + description: The hello extensions provided by the client. + fields: + - name: server_name_indication + type: keyword + description: List of hostnames + + - name: application_layer_protocol_negotiation + type: keyword + description: > + List of application-layer protocols the client is willing to use. + + - name: session_ticket + type: keyword + description: > + Length of the session ticket, if provided, or an empty string + to advertise support for tickets. + + - name: supported_versions + type: keyword + description: > + List of TLS versions that the client is willing to use. + + - name: supported_groups + type: keyword + description: > + List of Elliptic Curve Cryptography (ECC) curve groups + supported by the client. + + - name: signature_algorithms + type: keyword + description: > + List of signature algorithms that may be use in digital + signatures. + + - name: ec_points_formats + type: keyword + description: > + List of Elliptic Curve (EC) point formats. Indicates the + set of point formats that the client can parse. + + - name: _unparsed_ + type: keyword + description: > + List of extensions that were left unparsed by Packetbeat. + + - name: server_hello + type: group + fields: + - name: version + type: keyword + description: > + The version of the TLS protocol that is used for this session. + It is the highest version supported by the server not exceeding + the version requested in the client hello. + + - name: selected_compression_method + type: keyword + description: > + The compression method selected by the server from the list + provided in the client hello. + + - name: session_id + type: keyword + description: > + Unique number to identify the session for the corresponding + connection with the client. + + - name: extensions + type: group + description: The hello extensions provided by the server. + fields: + - name: application_layer_protocol_negotiation + type: keyword + description: Negotiated application layer protocol + + - name: session_ticket + type: keyword + description: > + Used to announce that a session ticket will be provided + by the server. Always an empty string. + + - name: supported_versions + type: keyword + description: > + Negotiated TLS version to be used. + + - name: ec_points_formats + type: keyword + description: > + List of Elliptic Curve (EC) point formats. Indicates the + set of point formats that the server can parse. + + - name: _unparsed_ + type: keyword + description: > + List of extensions that were left unparsed by Packetbeat. + + - name: client_certificate + type: group + description: Certificate provided by the client for authentication. + fields: + - name: version + type: long + description: X509 format version. + + - name: version_number + type: keyword + description: Version of x509 format. + example: 3 + + - name: serial_number + type: keyword + description: The certificate's serial number. + + - name: not_before + type: date + description: Date before which the certificate is not valid. + + - name: not_after + type: date + description: Date after which the certificate expires. + + - name: public_key_algorithm + type: keyword + description: > + The algorithm used for this certificate's public key. + One of RSA, DSA or ECDSA. + + - name: public_key_size + type: long + description: Size of the public key. + + - name: signature_algorithm + type: keyword + description: > + The algorithm used for the certificate's signature. + + - name: alternative_names + type: keyword + description: Subject Alternative Names for this certificate. + + - name: subject + type: group + description: Subject represented by this certificate. + fields: + - name: country + type: keyword + description: Country code. + + - name: organization + type: keyword + description: Organization name. + + - name: organizational_unit + type: keyword + description: Unit within organization. + + - name: province + type: keyword + description: Province or region within country. + + - name: common_name + type: keyword + description: Name or host name identified by the certificate. + + - name: locality + type: keyword + description: Locality. + + - name: distinguished_name + type: keyword + description: Distinguished name (DN) of the certificate subject entity. + example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + + - name: issuer + type: group + description: Entity that issued and signed this certificate. + fields: + - name: country + type: keyword + description: Country code. + + - name: organization + type: keyword + description: Organization name. + + - name: organizational_unit + type: keyword + description: Unit within organization. + + - name: province + type: keyword + description: Province or region within country. + + - name: common_name + type: keyword + description: Name or host name identified by the certificate. + + - name: locality + type: keyword + description: Locality. + + - name: distinguished_name + type: keyword + description: Distinguished name (DN) of the certificate issuer entity. + example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + + - name: server_certificate + type: group + description: Certificate provided by the server for authentication. + fields: + - name: version + type: long + description: X509 format version. + + - name: version_number + type: keyword + description: Version of x509 format. + example: 3 + + - name: serial_number + type: keyword + description: The certificate's serial number. + + - name: not_before + type: date + description: Date before which the certificate is not valid. + + - name: not_after + type: date + description: Date after which the certificate expires. + + - name: public_key_algorithm + type: keyword + description: > + The algorithm used for this certificate's public key. + One of RSA, DSA or ECDSA. + + - name: public_key_size + type: long + description: Size of the public key. + + - name: signature_algorithm + type: keyword + description: > + The algorithm used for the certificate's signature. + + - name: alternative_names + type: keyword + description: Subject Alternative Names for this certificate. + + - name: subject + type: group + description: Subject represented by this certificate. + fields: + - name: country + type: keyword + description: Country code. + + - name: organization + type: keyword + description: Organization name. + + - name: organizational_unit + type: keyword + description: Unit within organization. + + - name: province + type: keyword + description: Province or region within country. + + - name: state_or_province + type: keyword + description: Province or region within country. + + - name: common_name + type: keyword + description: Name or host name identified by the certificate. + + - name: locality + type: keyword + description: Locality. + + - name: distinguished_name + type: keyword + description: Distinguished name (DN) of the certificate subject entity. + example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + + - name: issuer + type: group + description: Entity that issued and signed this certificate. + fields: + - name: country + type: keyword + description: Country code. + + - name: organization + type: keyword + description: Organization name. + + - name: organizational_unit + type: keyword + description: Unit within organization. + + - name: province + type: keyword + description: Province or region within country. + + - name: state_or_province + type: keyword + description: Province or region within country. + + - name: common_name + type: keyword + description: Name or host name identified by the certificate. + + - name: locality + type: keyword + description: Locality. + + - name: distinguished_name + type: keyword + description: Distinguished name (DN) of the certificate issuer entity. + example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net + + - name: server_certificate_chain + type: array + description: Chain of trust for the server certificate. + + - name: client_certificate_chain + type: array + description: Chain of trust for the client certificate. + + - name: alert_types + type: keyword + description: > + An array containing the TLS alert type for every alert received. diff --git a/packetbeat/protos/tls/fields.go b/packetbeat/protos/tls/fields.go index f56f6af273a..b0c0a67e3a2 100644 --- a/packetbeat/protos/tls/fields.go +++ b/packetbeat/protos/tls/fields.go @@ -32,5 +32,5 @@ func init() { // AssetTls returns asset data. // This is the base64 encoded gzipped contents of protos/tls. func AssetTls() string { - return "eJzsWs1u20YQvvspBr40AWy5RdFDfShgyD4YCJKgTtDeiBV3JE693GV2h5KZpy92SUor/shKLLtJKp0skZz55v+bNc/hHqtLYOUSiSxIoTwBYGKFl3B63fwEH97cnZ4ASHSppYLJ6Ev44wQAIL7l3BWY0pxSwCVqhjmhkm5yAs1fl+GJc9Aix6AzfAfgqsBLWFhTFs0v8f3xMxHG9tN/2H8kzkWpOGkEwVwoh9H1roKNiiVaR0ZHV1od91itjJVbVwY8svl8yLAVB2YOnKH3EhTWsEmNgtKhnGw9gw8iL4Lr/Y2/TH49PRmAaNGVedCZ5MiZkYcAe1sDdOgC3kw4mCHqWhnKs3C11BKtqkgvoNZc2wDvNIKZdySekjyFubHB6Fbu7TUYC6dM6T3y5nL9HfCBUfv7JkN2p4pQc5KiZZ9lgjGx+KlExzjkgpkxCoXe3wV/ZcgZ2sYPdok2uGGtI1yoQQAbECVnqDkAAWKHquuB0nlPifaZCPgu+zJUygzY083xoSyGRzJ5d4I86iHYI6tnFawySrPYWytyGTpgMyAvNXle6tqLsrTeY5yRazNmy1Oxec31hPo2PNHCj5o+lQi6zGc+GwyQ9HGeV1sV4lM3mGisRVcYLUkvBu3TGlOvDVbEsVvGTSuLwlhGmaQmL2xjaF1x7jniqcixD2akrilwF4exweUmA2LuECFjLtzlxcVqtZqQ0GJi7OJCOEcLnaNmd+Hln3vB5yQ73yYPGedqzCPrxjBmfr84esZ7Q0NtRdJ82i5JovRpG0emK2m41LaCFjpG4r8kpKVP6KHqezxiPeBvmuhkxrEX77peimGIolCN7kSJCm3SVmaicWGYDgZrKJn8p4UbITkPSNY9YiupyMGKlAp1b/w86ZUFDFR9PTCe1QzUC87aFtdWfa33DGi+Tp0zP9GEBswLrsCxHe4DAZoBIZd+DDhsq6luJEGs2236uis07bdfC4c0v4mi7+2tPuBM8BNitzYgVOuLwL9Ryt+awrS0S4SprQo2CyuKrIJXN9Ppa0jDhR2IYAO82yV2GUsLLbi0mAi1MJY4y1/E4LVe2Oit45aLCmboowSkQdKCWKgxi1spu1MS06QwpNklc2Nzwf9FSF/dTF9DQAENiAnc1v0XQ6MZMxGDtK0newmeCg2FsI8kdlLqcJNMXsL+aHoFuCu0CArnDC0Mn6bvhe8oMxQ8yDWbafWdc81gP7mwhTSELGaOfYG34XYvJ6NFho7X4ns13mwA2vi1JEUcoXccQdxsCqTjLApe3kFkFabDZO85/NnneGsEHdPn1uThu6eHA9LW3OnLrP1hafuLkdQ6Pl9DUl+OHb5txKGMlcI2Efw2mN5H3z08NdPalDrFuq2IDucLTMcP0DYaI9K2YwRXaiUq1+WH3xDRiwIVcT3vj5osyP8pA2ia4A/LAPqnaXvxgC2g083DI5t06LfRSVlvLrc966uIhTIDvXsL4d+//fx7E9tW1o7RZEmopB4gT5lOYcxuXPOTa0Q3s2kUgDaczHBubD9ja+2yG6We6msfilpGfAwXxYlc4DRLoahf2jESMedRN+wJJIgYwYEPBQ3tFy2EopwpSpN7rDYb1HPwobXwDovcDmCNxmsaopX1+Tf8eXd1Btd3V2As3Eyv7672Mc7R57FwP57ed/QZW2YcQxzL8P5S+mIu7ZVEi2UUrVCMVgumJSb1udcTsN6Vs38wZbjaCIW3XuhgxHcczQYxX0/wWhwWPQtHvabdXQBdKY+Tu9SUmm11iAk0rUVBauTuwWfsQmj6fDDa+C6SF1TsrV2opNR0EKL4URMH1k96S8VOLGH+6XSYbHwhgPeNLN9ILC7aJYR0G+OdSFKT50aHkjkIkxd5AJIZx0FDu1hRNOx3lE4MTZlUKOKD5OibRtZorZJz5egE26NUbzQTV+0hgyv9KqNlaFzhn4HHij1W7LFin1qxnRPJZ9tH2kOt4z5y3EeO+8i4ccd95LiPHPeRI7s5spvjPvIEbxwr9lix32XFju8jSZoJGnozXFgrqh1biX8ssCJbOl7Tj/YfTPu9Ffx8OPZ8O1kotJx4RW5A9Re/b36la7iQGs2CdP3eb/1+R1AVBAeQuERbNT9aTJGWKCcn/wYAAP//nnkfHg==" + return "eJzsW19v2zgSf8+nGOw+JDk4yr/N7iZAF3Cd9DZALi3qZO9wLwItjSVeaVJLUnbdT38YSrJlW5Yd20nTVn4IYkua+c1wOPxxODqCTzi+AiuMH6JlXGC4B2C5FXgF+9f5T/Bw193fAwjRBJonlit5BX/sAQCUbzkyCQa8zwPAIUoLfY4iNN4e5P9duSeOQLIBOp3uO4AdJ3gFkVZpkv9Svp8+P0OEFjQPQfXBxtzAKEYJI4Q0iTQLEayCm04XTr1fJw8VigLBUdrJz1X6qnSWRXy+OLmcubBMCH1C7LNUWD8XCH0mDM7dU6WsrHCI2nAlF64Xej/heKR0WHF9Zoz+ysSQ18gE6Cs9YNareAw/s0FCg36+twqUL9NBD/XrwmZQcyZ2BO2PihsAHiX/O8VcE2SagBuTYgi9MdgYIUBtaQIwi8BSGyvN7diDd0pDoKThxqIMxi3geRQPmUgRuAEmkpjJdICaBy3gFkysUhFCbz5wsk/mK4shjLiNVWohUEJJA0yGkCYJ6oAZhCBmmgUWtal168XFu7dvLzu/Xd+8fXdy+fvJ5fXpWafTbi/3trNaeyE3lsso5SbG0KdL23r+uizRKYOD6/tDChLSyWW0xMd19nXePHZb8P7NNY94B7WFWxm04P3jm9Fo5IU84iTSC9SgBZ376V3dP9tn8CePYmgbk2omA4Qu6iFq6Kz2TaAGA5orO3DKHTeWPJCJzL3S2dYrO7JT6YhJ/oURVCb8VHK7K3tnRQOJNnDw/nFLw+dH/UkmPodtZNSuBvNWBivtESpggtvxrmwp5DktBg7uDuug/kul0jIu4S+Oo5VYE62GXAZbz6EPuRxQGjRGtPBQ6uQSAsKjx95KKMYyi77S/q4wFf5zgglYIbhwZPehBd0WXflQ69IOE7yvtORsjbTkrN1dSnLi4KBzCIEK0dTBfOzWrN88ksymGn0mIor3eLAtxtsQJU0j1LRezkyqiTaYaPPgI1KCRVpBDc3CbBD6KpUhcAn/VHDHZAQdPU6sAsF7mpHlXUSIrU3M1fFxxG2c9iipHEdKMBkdR+q4J1TvOFKn3ukvx0YHx4ETcEykx/3xIvXz3fn5b0d35xfnh7XTnFL1xa9HH7s1eVkq6/ewr/Ty+AyZrbo4470HPkBgFkYxD+IFcsMN9Lk2NqM1IWoMiczwsBb+2cnp5dHJ70envz6cnF79cnJ1dvHfekNY39bQuZ3YIRUIJSPUT7Xl7OTo5Ddny/nV6cXV+WWNLSbt/Q8D+7KMad7YHATQtFiLMnUf3kxTSwvu3nSZhHfEELgJFFGqd8xYMW7RsuM59qTPPBMzjaEXCdVjwuu7OzyJdrV3npczmQlpKvTVeWArQ74WKZoY9iSEz8pp1vF1OYpWQ/+69GVmCqwG+xr4S4Hl2yYw0yTxrAxmUwKTpD3BA/8TjnfHYNqFIEgNhmAVRChRk7Mpt2cqSVjt7KplCyXYhn9ZHhG0RK5camNiVl+wWHum+MAkLEBiUT1u64sRZye//L4cLhMWtWSWD9EtE1V8c7MQztfGkoJJELfvDz24p5WVRBsYEu/rjau3aa4EU75Cz0DPlWhoKRJj4hluA3T7AVgYajQGTQuu77uFRhIx4iIMmA7NYcuJxAHjYnp/rQ/3/+GhYMbywAvU/t5uyqjGlQSaMmpTRm3KqJVubcqoTRm1KaOuZ2JTRm3KqN/kLqQpozZl1KaM2pRRmzJqU0ZtyqhNGfWbIzBNGbUpozZl1K3KqIUXSx2zs06bL2euKmXW1UyrS5h1g7OiBkfhNZwWDSnCHu66lHOsCpRwc2TeLVOX0K2n3vn+XiVYjSYdOM3+AG2s5qFtAfs2g2rQOOQxM9BDlJlKDFvuaipD1GJM245Mf2YNvJc0mxZk/sTDn9ymhqwqJN9eU6L9yfLgE9rp5ew74GeLku7zqj2QNRv7pcj1Nf6dorFY7YyeUgLZ/PiucMa/Y7Qx6twjrsZEDployjirg0LpjmYVsdWMq1uDYtEX2WaNFU+VDKi3NEYh1NpF/Rcq2FdXnldFfm9c3t5kfhjRhsCAnTcx+1AuSmXm1zDV5EFXkM5jqY5MZHf4vMqW3VXZ8/K6VcCzbfx4ZhZRcDtjldZoEiVDXrk+kaVSYmALwlRyUS1jShKlLYZ+oAaJzk3OZubWS87yMRbTTUuhNE8Hpjy0ObrKNQCgXI8YjUYeZ5IR3T9mxvBIDlBac0wajkj0EQ/nvnmfYzsQy30zSSTLHVE1gRbcQAa7OViSmBHIsHSgko/UoqxlExIWTv4cUfC5DCncq+corDGGCwYU1CFWxmZcpBYMSxKRI/AFG6Oj4W7++hIjZflOwVUHGX0K2CU8Rw7PJJ/MBBs3REWEyxCKVqWKaQMV+SFbdl7AHJSRjYu0WOSHTLs7aCtCym1DmAQcJHYMxuplGcMBVMDCIS0lBov5liUdJ9iscsIkf+Rpu2qu7NoR+bjSylBoBRszu9VoTgxxc/oFzbgRgm4OoJPqIeZV10izJB7DwU2HdmvuQi0umBown1HqzV6sRr+g6RXV6XwkB2wMPaRxo71UyCNumVhueyFnVbhi4CeKS2v87GD56w3zwU3nEByW/IzbeHCbZW50aWm5sejkzTy7EP4Bk5AwvTLs/VS620L/5TxRWgUd7BFqBIF9CwUYCuEPjPJPD9l8CM+ted8Ru3Xe4CargWTEr8xUq0TeugdIUsyjGI2dKFjIB/kuRCraIgWIS4mkLcGc7le4LMeX83oteRYYVNPK5/PtIpuc4JhzQl+rgftORLRS3oSfPdXuH2DT8KLEOBuxTYnxy3LR+1wkhmXFMEs6XxerfMzrrUxKlcoAsyTE5vil41K0IBejs1Te7KhBW4zY2Myz0VdIKktDV+KV5JmMhoQ/PLPIU+cPwCwWK4Rr84sZwJ1Scbx6v+8ydan2V7HOb8tY1jnP+M+0LbWQV5P+f5ju2IfZtoJ9M9sSW+OiHbWmXFPkZHLq2jps0cVRi2f7DhMHx4lZggY/J7x6E/ach4fLCSGbPVacUOrZQa0/XYT8aAI+dtstuO62QWm46Vx32+uZufVhY7fyoLGOg+6+w+yJLl6YNgWiGtQ7P/Ds5ged7dJB533e6bYYB6u7CrbjugUajbRFQTnZk8zDWJSzDs9d3iuwjt8W0HbyXoFAhavW+RX9NRupf1+S6dQ8AcPyLqSNoDxKbosGlLKaFYhqGk82grFJVwzMxEddw9lGkFyfgNLuVCBrx+NFH2rVayIrANb0Om2E7i6Xt0LtWm2KGwHYabsifL2Wxaydervkd+MMK+pb7j0iJkO3LLiT8CYHLqpvcmCTA7dE9y3lwCzNvJ4UOHfK8Ky1gKIo3dQCmlrAUgBNLaCpBTS1gA0xN7WAtf22gLbhwd8ZD17n7ZivAqwh6NVqXwVBb4oUTXJeUN8k5yY5bw6pSc61AL7L6okfxIxXvyvHtGbzozeb7OhRZ75OjZ3sL4rGlHXfh3peNGu/ncUEauu7Fy0rAWz0Dl5bZsCLVy+z952yPlOnMHtLk8DiEPU4/1FjgHyIobf3/wAAAP//PewGKg==" } diff --git a/packetbeat/protos/tls/parse.go b/packetbeat/protos/tls/parse.go index d6714a76784..c007690789d 100644 --- a/packetbeat/protos/tls/parse.go +++ b/packetbeat/protos/tls/parse.go @@ -571,12 +571,14 @@ func certToMap(cert *x509.Certificate) common.MapStr { certMap := common.MapStr{ "signature_algorithm": cert.SignatureAlgorithm.String(), "public_key_algorithm": toString(cert.PublicKeyAlgorithm), - "version": cert.Version, - "serial_number": cert.SerialNumber.Text(10), - "issuer": toMap(&cert.Issuer), - "subject": toMap(&cert.Subject), - "not_before": cert.NotBefore, - "not_after": cert.NotAfter, + // remove this in 8.x + "version": cert.Version, + "serial_number": cert.SerialNumber.Text(10), + "issuer": toMap(&cert.Issuer), + "subject": toMap(&cert.Subject), + "not_before": cert.NotBefore, + "not_after": cert.NotAfter, + "version_number": cert.Version, } if keySize := getKeySize(cert.PublicKey); keySize > 0 { certMap["public_key_size"] = keySize @@ -602,11 +604,14 @@ func toMap(name *pkix.Name) common.MapStr { {"organization", name.Organization}, {"organizational_unit", name.OrganizationalUnit}, {"locality", name.Locality}, + // remove this in 8.x {"province", name.Province}, {"postal_code", name.PostalCode}, {"serial_number", name.SerialNumber}, {"common_name", name.CommonName}, {"street_address", name.StreetAddress}, + {"state_or_province", name.Province}, + {"distinguished_name", name.String()}, } for _, field := range fields { var str string diff --git a/packetbeat/protos/tls/tls.go b/packetbeat/protos/tls/tls.go index ab068884b61..74034c4afaf 100644 --- a/packetbeat/protos/tls/tls.go +++ b/packetbeat/protos/tls/tls.go @@ -443,6 +443,12 @@ func (plugin *tlsPlugin) createEvent(conn *tlsConnectionData) beat.Event { pb.MarshalStruct(fields, "tls", tls) if plugin.includeDetailedFields { fields.Put("tls.detailed", detailed) + if cert, ok := detailed["client_certificate"]; ok { + fields.Put("tls.client.x509", cert) + } + if cert, ok := detailed["server_certificate"]; ok { + fields.Put("tls.server.x509", cert) + } } // Fixes for non-array datatypes diff --git a/packetbeat/protos/tls/tls_test.go b/packetbeat/protos/tls/tls_test.go index 608e8773883..64a79024806 100644 --- a/packetbeat/protos/tls/tls_test.go +++ b/packetbeat/protos/tls/tls_test.go @@ -39,7 +39,7 @@ type eventStore struct { } const ( - expectedClientHello = `{"client":{"ip":"192.168.0.1","port":6512},"destination":{"domain":"example.org","ip":"192.168.0.2","port":27017},"event":{"category":"network_traffic","dataset":"tls","kind":"event"},"network":{"community_id":"1:jKfewJN/czjTuEpVvsKdYXXiMzs=","protocol":"tls","transport":"tcp","type":"ipv4"},"server":{"domain":"example.org","ip":"192.168.0.2","port":27017},"source":{"ip":"192.168.0.1","port":6512},"status":"Error","tls":{"client":{"ja3":"94c485bca29d5392be53f2b8cf7f4304","server_name":"example.org","supported_ciphers":["TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256","TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256","TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","TLS_RSA_WITH_AES_128_GCM_SHA256","TLS_RSA_WITH_AES_256_GCM_SHA384","TLS_RSA_WITH_AES_128_CBC_SHA","TLS_RSA_WITH_AES_256_CBC_SHA","TLS_RSA_WITH_3DES_EDE_CBC_SHA"]},"detailed":{"client_certificate_requested":false,"client_hello":{"extensions":{"_unparsed_":["renegotiation_info","23","status_request","18","30032"],"application_layer_protocol_negotiation":["h2","http/1.1"],"ec_points_formats":["uncompressed"],"server_name_indication":["example.org"],"session_ticket":"","signature_algorithms":["ecdsa_secp256r1_sha256","rsa_pss_sha256","rsa_pkcs1_sha256","ecdsa_secp384r1_sha384","rsa_pss_sha384","rsa_pkcs1_sha384","rsa_pss_sha512","rsa_pkcs1_sha512","rsa_pkcs1_sha1"],"supported_groups":["x25519","secp256r1","secp384r1"]},"supported_compression_methods":["NULL"],"version":"3.3"},"version":"TLS 1.2"},"established":false,"resumed":false,"version":"1.2","version_protocol":"tls"},"type":"tls"}` + expectedClientHello = `{"client":{"ip":"192.168.0.1","port":6512},"destination":{"domain":"example.org","ip":"192.168.0.2","port":27017},"event":{"category":["network_traffic","network"],"dataset":"tls","kind":"event","type":["connection","protocol"]},"network":{"community_id":"1:jKfewJN/czjTuEpVvsKdYXXiMzs=","protocol":"tls","transport":"tcp","type":"ipv4"},"related":{"ip":["192.168.0.1","192.168.0.2"]},"server":{"domain":"example.org","ip":"192.168.0.2","port":27017},"source":{"ip":"192.168.0.1","port":6512},"status":"Error","tls":{"client":{"ja3":"94c485bca29d5392be53f2b8cf7f4304","server_name":"example.org","supported_ciphers":["TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256","TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256","TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","TLS_RSA_WITH_AES_128_GCM_SHA256","TLS_RSA_WITH_AES_256_GCM_SHA384","TLS_RSA_WITH_AES_128_CBC_SHA","TLS_RSA_WITH_AES_256_CBC_SHA","TLS_RSA_WITH_3DES_EDE_CBC_SHA"]},"detailed":{"client_certificate_requested":false,"client_hello":{"extensions":{"_unparsed_":["renegotiation_info","23","status_request","18","30032"],"application_layer_protocol_negotiation":["h2","http/1.1"],"ec_points_formats":["uncompressed"],"server_name_indication":["example.org"],"session_ticket":"","signature_algorithms":["ecdsa_secp256r1_sha256","rsa_pss_sha256","rsa_pkcs1_sha256","ecdsa_secp384r1_sha384","rsa_pss_sha384","rsa_pkcs1_sha384","rsa_pss_sha512","rsa_pkcs1_sha512","rsa_pkcs1_sha1"],"supported_groups":["x25519","secp256r1","secp384r1"]},"supported_compression_methods":["NULL"],"version":"3.3"},"version":"TLS 1.2"},"established":false,"resumed":false,"version":"1.2","version_protocol":"tls"},"type":"tls"}` expectedServerHello = `{"extensions":{"_unparsed_":["renegotiation_info","status_request"],"application_layer_protocol_negotiation":["h2"],"ec_points_formats":["uncompressed","ansiX962_compressed_prime","ansiX962_compressed_char2"],"session_ticket":""},"selected_compression_method":"NULL","version":"3.3"}` rawClientHello = "16030100c2010000be03033367dfae0d46ec0651e49cca2ae47317e8989df710" + "ee7570a88b9a7d5d56b3af00001c3a3ac02bc02fc02cc030cca9cca8c013c014" + diff --git a/packetbeat/tests/system/golden/established_tls-expected.json b/packetbeat/tests/system/golden/established_tls-expected.json index 2756f6f5bac..5ce92528a02 100644 --- a/packetbeat/tests/system/golden/established_tls-expected.json +++ b/packetbeat/tests/system/golden/established_tls-expected.json @@ -7,14 +7,25 @@ "destination.domain": "example.net", "destination.ip": "93.184.216.34", "destination.port": 443, - "event.category": "network_traffic", + "event.category": [ + "network_traffic", + "network" + ], "event.dataset": "tls", "event.duration": 364625000, "event.kind": "event", + "event.type": [ + "connection", + "protocol" + ], "network.community_id": "1:fx1jENdlg6r3LIvBRG3wEboWbPY=", "network.protocol": "tls", "network.transport": "tcp", "network.type": "ipv4", + "related.ip": [ + "192.168.1.35", + "93.184.216.34" + ], "server.domain": "example.net", "server.ip": "93.184.216.34", "server.port": 443, @@ -115,6 +126,7 @@ ], "tls.detailed.server_certificate.issuer.common_name": "DigiCert SHA2 Secure Server CA", "tls.detailed.server_certificate.issuer.country": "US", + "tls.detailed.server_certificate.issuer.distinguished_name": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US", "tls.detailed.server_certificate.issuer.organization": "DigiCert Inc", "tls.detailed.server_certificate.not_after": "2020-12-02T12:00:00.000Z", "tls.detailed.server_certificate.not_before": "2018-11-28T00:00:00.000Z", @@ -124,16 +136,20 @@ "tls.detailed.server_certificate.signature_algorithm": "SHA256-RSA", "tls.detailed.server_certificate.subject.common_name": "www.example.org", "tls.detailed.server_certificate.subject.country": "US", + "tls.detailed.server_certificate.subject.distinguished_name": "CN=www.example.org,OU=Technology,O=Internet Corporation for Assigned Names and Numbers,L=Los Angeles,ST=California,C=US", "tls.detailed.server_certificate.subject.locality": "Los Angeles", "tls.detailed.server_certificate.subject.organization": "Internet Corporation for Assigned Names and Numbers", "tls.detailed.server_certificate.subject.organizational_unit": "Technology", "tls.detailed.server_certificate.subject.province": "California", + "tls.detailed.server_certificate.subject.state_or_province": "California", "tls.detailed.server_certificate.version": 3, + "tls.detailed.server_certificate.version_number": 3, "tls.detailed.server_certificate_chain": [ { "issuer": { "common_name": "DigiCert Global Root CA", "country": "US", + "distinguished_name": "CN=DigiCert Global Root CA,OU=www.digicert.com,O=DigiCert Inc,C=US", "organization": "DigiCert Inc", "organizational_unit": "www.digicert.com" }, @@ -146,14 +162,17 @@ "subject": { "common_name": "DigiCert SHA2 Secure Server CA", "country": "US", + "distinguished_name": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US", "organization": "DigiCert Inc" }, - "version": 3 + "version": 3, + "version_number": 3 }, { "issuer": { "common_name": "DigiCert Global Root CA", "country": "US", + "distinguished_name": "CN=DigiCert Global Root CA,OU=www.digicert.com,O=DigiCert Inc,C=US", "organization": "DigiCert Inc", "organizational_unit": "www.digicert.com" }, @@ -166,10 +185,12 @@ "subject": { "common_name": "DigiCert Global Root CA", "country": "US", + "distinguished_name": "CN=DigiCert Global Root CA,OU=www.digicert.com,O=DigiCert Inc,C=US", "organization": "DigiCert Inc", "organizational_unit": "www.digicert.com" }, - "version": 3 + "version": 3, + "version_number": 3 } ], "tls.detailed.server_hello.extensions._unparsed_": [ @@ -196,6 +217,36 @@ "tls.server.not_after": "2020-12-02T12:00:00.000Z", "tls.server.not_before": "2018-11-28T00:00:00.000Z", "tls.server.subject": "CN=www.example.org,OU=Technology,O=Internet Corporation for Assigned Names and Numbers,L=Los Angeles,ST=California,C=US", + "tls.server.x509.alternative_names": [ + "www.example.org", + "example.com", + "example.edu", + "example.net", + "example.org", + "www.example.com", + "www.example.edu", + "www.example.net" + ], + "tls.server.x509.issuer.common_name": "DigiCert SHA2 Secure Server CA", + "tls.server.x509.issuer.country": "US", + "tls.server.x509.issuer.distinguished_name": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US", + "tls.server.x509.issuer.organization": "DigiCert Inc", + "tls.server.x509.not_after": "2020-12-02T12:00:00.000Z", + "tls.server.x509.not_before": "2018-11-28T00:00:00.000Z", + "tls.server.x509.public_key_algorithm": "RSA", + "tls.server.x509.public_key_size": 2048, + "tls.server.x509.serial_number": "21020869104500376438182461249190639870", + "tls.server.x509.signature_algorithm": "SHA256-RSA", + "tls.server.x509.subject.common_name": "www.example.org", + "tls.server.x509.subject.country": "US", + "tls.server.x509.subject.distinguished_name": "CN=www.example.org,OU=Technology,O=Internet Corporation for Assigned Names and Numbers,L=Los Angeles,ST=California,C=US", + "tls.server.x509.subject.locality": "Los Angeles", + "tls.server.x509.subject.organization": "Internet Corporation for Assigned Names and Numbers", + "tls.server.x509.subject.organizational_unit": "Technology", + "tls.server.x509.subject.province": "California", + "tls.server.x509.subject.state_or_province": "California", + "tls.server.x509.version": 3, + "tls.server.x509.version_number": 3, "tls.version": "1.2", "tls.version_protocol": "tls", "type": "tls" diff --git a/packetbeat/tests/system/golden/non_established_tls-expected.json b/packetbeat/tests/system/golden/non_established_tls-expected.json index 32a9b174e3e..573bb673774 100644 --- a/packetbeat/tests/system/golden/non_established_tls-expected.json +++ b/packetbeat/tests/system/golden/non_established_tls-expected.json @@ -7,13 +7,24 @@ "destination.domain": "www.elastic.co", "destination.ip": "151.101.134.217", "destination.port": 443, - "event.category": "network_traffic", + "event.category": [ + "network_traffic", + "network" + ], "event.dataset": "tls", "event.kind": "event", + "event.type": [ + "connection", + "protocol" + ], "network.community_id": "1:OD+OA3Fyagq6ZdQ3T94i6r2zqWA=", "network.protocol": "tls", "network.transport": "tcp", "network.type": "ipv4", + "related.ip": [ + "172.20.10.2", + "151.101.134.217" + ], "server.domain": "www.elastic.co", "server.ip": "151.101.134.217", "server.port": 443, diff --git a/packetbeat/tests/system/golden/tls_1_3-expected.json b/packetbeat/tests/system/golden/tls_1_3-expected.json index 7200da6b305..30285212e32 100644 --- a/packetbeat/tests/system/golden/tls_1_3-expected.json +++ b/packetbeat/tests/system/golden/tls_1_3-expected.json @@ -7,14 +7,25 @@ "destination.domain": "play.google.com", "destination.ip": "216.58.201.174", "destination.port": 443, - "event.category": "network_traffic", + "event.category": [ + "network_traffic", + "network" + ], "event.dataset": "tls", "event.duration": 12280000, "event.kind": "event", + "event.type": [ + "connection", + "protocol" + ], "network.community_id": "1:hfsK5r0tJm7av4j7BtSxA6oH9xA=", "network.protocol": "tls", "network.transport": "tcp", "network.type": "ipv4", + "related.ip": [ + "192.168.1.36", + "216.58.201.174" + ], "server.domain": "play.google.com", "server.ip": "216.58.201.174", "server.port": 443, diff --git a/packetbeat/tests/system/golden/tls_all_options-expected.json b/packetbeat/tests/system/golden/tls_all_options-expected.json index 78d7f74f2a4..f1ba1cf337d 100644 --- a/packetbeat/tests/system/golden/tls_all_options-expected.json +++ b/packetbeat/tests/system/golden/tls_all_options-expected.json @@ -7,14 +7,25 @@ "destination.domain": "example.net", "destination.ip": "93.184.216.34", "destination.port": 443, - "event.category": "network_traffic", + "event.category": [ + "network_traffic", + "network" + ], "event.dataset": "tls", "event.duration": 364625000, "event.kind": "event", + "event.type": [ + "connection", + "protocol" + ], "network.community_id": "1:fx1jENdlg6r3LIvBRG3wEboWbPY=", "network.protocol": "tls", "network.transport": "tcp", "network.type": "ipv4", + "related.ip": [ + "192.168.1.35", + "93.184.216.34" + ], "server.domain": "example.net", "server.ip": "93.184.216.34", "server.port": 443, @@ -115,6 +126,7 @@ ], "tls.detailed.server_certificate.issuer.common_name": "DigiCert SHA2 Secure Server CA", "tls.detailed.server_certificate.issuer.country": "US", + "tls.detailed.server_certificate.issuer.distinguished_name": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US", "tls.detailed.server_certificate.issuer.organization": "DigiCert Inc", "tls.detailed.server_certificate.not_after": "2020-12-02T12:00:00.000Z", "tls.detailed.server_certificate.not_before": "2018-11-28T00:00:00.000Z", @@ -124,16 +136,20 @@ "tls.detailed.server_certificate.signature_algorithm": "SHA256-RSA", "tls.detailed.server_certificate.subject.common_name": "www.example.org", "tls.detailed.server_certificate.subject.country": "US", + "tls.detailed.server_certificate.subject.distinguished_name": "CN=www.example.org,OU=Technology,O=Internet Corporation for Assigned Names and Numbers,L=Los Angeles,ST=California,C=US", "tls.detailed.server_certificate.subject.locality": "Los Angeles", "tls.detailed.server_certificate.subject.organization": "Internet Corporation for Assigned Names and Numbers", "tls.detailed.server_certificate.subject.organizational_unit": "Technology", "tls.detailed.server_certificate.subject.province": "California", + "tls.detailed.server_certificate.subject.state_or_province": "California", "tls.detailed.server_certificate.version": 3, + "tls.detailed.server_certificate.version_number": 3, "tls.detailed.server_certificate_chain": [ { "issuer": { "common_name": "DigiCert Global Root CA", "country": "US", + "distinguished_name": "CN=DigiCert Global Root CA,OU=www.digicert.com,O=DigiCert Inc,C=US", "organization": "DigiCert Inc", "organizational_unit": "www.digicert.com" }, @@ -146,14 +162,17 @@ "subject": { "common_name": "DigiCert SHA2 Secure Server CA", "country": "US", + "distinguished_name": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US", "organization": "DigiCert Inc" }, - "version": 3 + "version": 3, + "version_number": 3 }, { "issuer": { "common_name": "DigiCert Global Root CA", "country": "US", + "distinguished_name": "CN=DigiCert Global Root CA,OU=www.digicert.com,O=DigiCert Inc,C=US", "organization": "DigiCert Inc", "organizational_unit": "www.digicert.com" }, @@ -166,10 +185,12 @@ "subject": { "common_name": "DigiCert Global Root CA", "country": "US", + "distinguished_name": "CN=DigiCert Global Root CA,OU=www.digicert.com,O=DigiCert Inc,C=US", "organization": "DigiCert Inc", "organizational_unit": "www.digicert.com" }, - "version": 3 + "version": 3, + "version_number": 3 } ], "tls.detailed.server_hello.extensions._unparsed_": [ @@ -203,6 +224,36 @@ "tls.server.not_after": "2020-12-02T12:00:00.000Z", "tls.server.not_before": "2018-11-28T00:00:00.000Z", "tls.server.subject": "CN=www.example.org,OU=Technology,O=Internet Corporation for Assigned Names and Numbers,L=Los Angeles,ST=California,C=US", + "tls.server.x509.alternative_names": [ + "www.example.org", + "example.com", + "example.edu", + "example.net", + "example.org", + "www.example.com", + "www.example.edu", + "www.example.net" + ], + "tls.server.x509.issuer.common_name": "DigiCert SHA2 Secure Server CA", + "tls.server.x509.issuer.country": "US", + "tls.server.x509.issuer.distinguished_name": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US", + "tls.server.x509.issuer.organization": "DigiCert Inc", + "tls.server.x509.not_after": "2020-12-02T12:00:00.000Z", + "tls.server.x509.not_before": "2018-11-28T00:00:00.000Z", + "tls.server.x509.public_key_algorithm": "RSA", + "tls.server.x509.public_key_size": 2048, + "tls.server.x509.serial_number": "21020869104500376438182461249190639870", + "tls.server.x509.signature_algorithm": "SHA256-RSA", + "tls.server.x509.subject.common_name": "www.example.org", + "tls.server.x509.subject.country": "US", + "tls.server.x509.subject.distinguished_name": "CN=www.example.org,OU=Technology,O=Internet Corporation for Assigned Names and Numbers,L=Los Angeles,ST=California,C=US", + "tls.server.x509.subject.locality": "Los Angeles", + "tls.server.x509.subject.organization": "Internet Corporation for Assigned Names and Numbers", + "tls.server.x509.subject.organizational_unit": "Technology", + "tls.server.x509.subject.province": "California", + "tls.server.x509.subject.state_or_province": "California", + "tls.server.x509.version": 3, + "tls.server.x509.version_number": 3, "tls.version": "1.2", "tls.version_protocol": "tls", "type": "tls" diff --git a/packetbeat/tests/system/golden/tls_no_certs-expected.json b/packetbeat/tests/system/golden/tls_no_certs-expected.json index e730de81c10..3f4587b2586 100644 --- a/packetbeat/tests/system/golden/tls_no_certs-expected.json +++ b/packetbeat/tests/system/golden/tls_no_certs-expected.json @@ -7,14 +7,25 @@ "destination.domain": "example.net", "destination.ip": "93.184.216.34", "destination.port": 443, - "event.category": "network_traffic", + "event.category": [ + "network_traffic", + "network" + ], "event.dataset": "tls", "event.duration": 364625000, "event.kind": "event", + "event.type": [ + "connection", + "protocol" + ], "network.community_id": "1:fx1jENdlg6r3LIvBRG3wEboWbPY=", "network.protocol": "tls", "network.transport": "tcp", "network.type": "ipv4", + "related.ip": [ + "192.168.1.35", + "93.184.216.34" + ], "server.domain": "example.net", "server.ip": "93.184.216.34", "server.port": 443, diff --git a/packetbeat/tests/system/golden/tls_not_detailed-expected.json b/packetbeat/tests/system/golden/tls_not_detailed-expected.json index 4471fa2526b..ae23944e096 100644 --- a/packetbeat/tests/system/golden/tls_not_detailed-expected.json +++ b/packetbeat/tests/system/golden/tls_not_detailed-expected.json @@ -7,14 +7,25 @@ "destination.domain": "example.net", "destination.ip": "93.184.216.34", "destination.port": 443, - "event.category": "network_traffic", + "event.category": [ + "network_traffic", + "network" + ], "event.dataset": "tls", "event.duration": 364625000, "event.kind": "event", + "event.type": [ + "connection", + "protocol" + ], "network.community_id": "1:fx1jENdlg6r3LIvBRG3wEboWbPY=", "network.protocol": "tls", "network.transport": "tcp", "network.type": "ipv4", + "related.ip": [ + "192.168.1.35", + "93.184.216.34" + ], "server.domain": "example.net", "server.ip": "93.184.216.34", "server.port": 443, diff --git a/packetbeat/tests/system/test_0040_memcache_tcp_bin_basic.py b/packetbeat/tests/system/test_0040_memcache_tcp_bin_basic.py index 4e6a556bf18..82d80d1f02f 100644 --- a/packetbeat/tests/system/test_0040_memcache_tcp_bin_basic.py +++ b/packetbeat/tests/system/test_0040_memcache_tcp_bin_basic.py @@ -188,7 +188,7 @@ def test_delete(self): def test_stats(self): objs = self._run('memcache/memcache_bin_tcp_stats.pcap') - + print(objs) # all transactions succeed assert all(o['status'] == 'OK' for o in objs) diff --git a/packetbeat/tests/system/test_0050_icmp.py b/packetbeat/tests/system/test_0050_icmp.py index 86d5c231290..8500963aa9f 100644 --- a/packetbeat/tests/system/test_0050_icmp.py +++ b/packetbeat/tests/system/test_0050_icmp.py @@ -7,7 +7,6 @@ def test_2_pings(self): self.render_config_template() self.run_packetbeat(pcap="icmp/icmp_2_pings.pcap", debug_selectors=["*"]) objs = self.read_output() - assert len(objs) == 2 assert all([o["icmp.version"] == 4 for o in objs]) assert objs[0]["@timestamp"] == "2015-10-19T21:47:49.900Z" @@ -69,6 +68,8 @@ def test_icmp6_ping_over_vlan(self): def assert_common_fields(self, objs): assert all([o["type"] == "icmp" for o in objs]) assert all([o["event.dataset"] == "icmp" for o in objs]) + assert all([o["event.category"] == ['network_traffic', 'network'] for o in objs]) + assert all([o["event.type"] == ["connection"] for o in objs]) assert all([o["source.bytes"] == 4 for o in objs]) assert all([o["destination.bytes"] == 4 for o in objs]) assert all([("server.port" in o) == False for o in objs]) @@ -78,6 +79,7 @@ def assert_common_icmp4_fields(self, obj): assert obj["network.transport"] == "icmp" assert obj["server.ip"] == "10.0.0.2" assert obj["client.ip"] == "10.0.0.1" + assert obj["related.ip"] == ["10.0.0.1", "10.0.0.2"] assert obj["path"] == "10.0.0.2" assert obj["status"] == "OK" assert obj["icmp.request.message"] == "EchoRequest(0)" diff --git a/packetbeat/tests/system/test_0051_amqp_publish.py b/packetbeat/tests/system/test_0051_amqp_publish.py index cf7eeee717a..5b4d1e4f733 100644 --- a/packetbeat/tests/system/test_0051_amqp_publish.py +++ b/packetbeat/tests/system/test_0051_amqp_publish.py @@ -1,33 +1,35 @@ -from packetbeat import BaseTest - - -class Test(BaseTest): - - def test_amqp_publish(self): - self.render_config_template( - amqp_ports=[5672], - amqp_send_request=True - ) - self.run_packetbeat(pcap="amqp_publish.pcap", - debug_selectors=["amqp,tcp,publish"]) - - objs = self.read_output() - assert all([o["type"] == "amqp" for o in objs]) - assert len(objs) == 2 - assert all([o["server.port"] == 5672 for o in objs]) - - assert objs[0]["method"] == "queue.declare" - assert objs[0]["status"] == "OK" - assert objs[0]["amqp.queue"] == "hello" - assert objs[0]["amqp.durable"] == False - assert objs[0]["amqp.auto-delete"] == False - assert objs[0]["amqp.exclusive"] == False - assert objs[0]["amqp.no-wait"] == False - - assert objs[1]["method"] == "basic.publish" - assert objs[1]["status"] == "OK" - assert objs[1]["request"] == "hello" - assert objs[1]["amqp.routing-key"] == "hello" - assert objs[1]["amqp.mandatory"] == False - assert objs[1]["amqp.immediate"] == False - assert objs[1]["amqp.content-type"] == "text/plain" +from packetbeat import BaseTest + + +class Test(BaseTest): + + def test_amqp_publish(self): + self.render_config_template( + amqp_ports=[5672], + amqp_send_request=True + ) + self.run_packetbeat(pcap="amqp_publish.pcap", + debug_selectors=["amqp,tcp,publish"]) + + objs = self.read_output() + assert all([o["type"] == "amqp" for o in objs]) + assert len(objs) == 2 + assert all([o["server.port"] == 5672 for o in objs]) + + assert objs[0]["method"] == "queue.declare" + assert objs[0]["event.action"] == "amqp.queue.declare" + assert objs[0]["status"] == "OK" + assert objs[0]["amqp.queue"] == "hello" + assert objs[0]["amqp.durable"] == False + assert objs[0]["amqp.auto-delete"] == False + assert objs[0]["amqp.exclusive"] == False + assert objs[0]["amqp.no-wait"] == False + + assert objs[1]["method"] == "basic.publish" + assert objs[1]["event.action"] == "amqp.basic.publish" + assert objs[1]["status"] == "OK" + assert objs[1]["request"] == "hello" + assert objs[1]["amqp.routing-key"] == "hello" + assert objs[1]["amqp.mandatory"] == False + assert objs[1]["amqp.immediate"] == False + assert objs[1]["amqp.content-type"] == "text/plain" diff --git a/packetbeat/tests/system/test_0052_amqp_emit_receive.py b/packetbeat/tests/system/test_0052_amqp_emit_receive.py index d0649798558..3b62a5da8fb 100644 --- a/packetbeat/tests/system/test_0052_amqp_emit_receive.py +++ b/packetbeat/tests/system/test_0052_amqp_emit_receive.py @@ -1,69 +1,76 @@ -from packetbeat import BaseTest - - -class Test(BaseTest): - - def test_amqp_emit_receive(self): - self.render_config_template( - amqp_ports=[5672], - ) - self.run_packetbeat(pcap="amqp_emit_receive.pcap", - debug_selectors=["amqp,tcp,publish"]) - - objs = self.read_output() - assert all([o["type"] == "amqp" for o in objs]) - assert len(objs) == 7 - assert all([o["server.port"] == 5672 for o in objs]) - - assert objs[0]["method"] == "exchange.declare" - assert objs[0]["status"] == "OK" - assert objs[0]["amqp.exchange"] == "logs" - assert objs[0]["amqp.durable"] == True - assert objs[0]["amqp.exchange-type"] == "fanout" - assert objs[0]["amqp.passive"] == False - assert objs[0]["amqp.no-wait"] == False - - assert objs[1]["method"] == "queue.declare" - assert objs[1]["status"] == "OK" - assert objs[1]["amqp.queue"] != "" - assert objs[1]["amqp.exclusive"] == True - assert objs[1]["amqp.no-wait"] == False - assert objs[1]["amqp.durable"] == False - assert objs[1]["amqp.auto-delete"] == False - assert objs[1]["amqp.passive"] == False - - assert objs[2]["method"] == "queue.bind" - assert objs[2]["status"] == "OK" - assert objs[2]["amqp.queue"] != "" - assert objs[2]["amqp.exchange"] == "logs" - assert objs[2]["amqp.no-wait"] == False - - assert objs[3]["method"] == "basic.consume" - assert objs[3]["status"] == "OK" - assert objs[3]["amqp.queue"] != "" - assert objs[3]["amqp.no-ack"] == True - assert objs[3]["amqp.no-wait"] == False - assert objs[3]["amqp.no-local"] == False - assert objs[3]["amqp.exclusive"] == False - - assert objs[4]["method"] == "exchange.declare" - assert objs[4]["status"] == "OK" - assert objs[4]["amqp.exchange"] == "logs" - assert objs[4]["amqp.durable"] == True - assert objs[4]["amqp.exchange-type"] == "fanout" - assert objs[4]["amqp.passive"] == False - assert objs[4]["amqp.no-wait"] == False - - assert objs[5]["method"] == "basic.publish" - assert objs[5]["status"] == "OK" - assert objs[5]["amqp.content-type"] == "text/plain" - assert objs[5]["amqp.exchange"] == "logs" - assert objs[5]["amqp.immediate"] == False - assert objs[5]["amqp.mandatory"] == False - - assert objs[6]["method"] == "basic.deliver" - assert objs[6]["status"] == "OK" - assert objs[6]["amqp.content-type"] == "text/plain" - assert objs[6]["amqp.delivery-tag"] == 1 - assert objs[6]["amqp.exchange"] == "logs" - assert objs[6]["amqp.redelivered"] == False +from packetbeat import BaseTest + + +class Test(BaseTest): + + def test_amqp_emit_receive(self): + self.render_config_template( + amqp_ports=[5672], + ) + self.run_packetbeat(pcap="amqp_emit_receive.pcap", + debug_selectors=["amqp,tcp,publish"]) + + objs = self.read_output() + assert all([o["type"] == "amqp" for o in objs]) + assert len(objs) == 7 + assert all([o["server.port"] == 5672 for o in objs]) + + assert objs[0]["method"] == "exchange.declare" + assert objs[0]["event.action"] == "amqp.exchange.declare" + assert objs[0]["status"] == "OK" + assert objs[0]["amqp.exchange"] == "logs" + assert objs[0]["amqp.durable"] == True + assert objs[0]["amqp.exchange-type"] == "fanout" + assert objs[0]["amqp.passive"] == False + assert objs[0]["amqp.no-wait"] == False + + assert objs[1]["method"] == "queue.declare" + assert objs[1]["event.action"] == "amqp.queue.declare" + assert objs[1]["status"] == "OK" + assert objs[1]["amqp.queue"] != "" + assert objs[1]["amqp.exclusive"] == True + assert objs[1]["amqp.no-wait"] == False + assert objs[1]["amqp.durable"] == False + assert objs[1]["amqp.auto-delete"] == False + assert objs[1]["amqp.passive"] == False + + assert objs[2]["method"] == "queue.bind" + assert objs[2]["event.action"] == "amqp.queue.bind" + assert objs[2]["status"] == "OK" + assert objs[2]["amqp.queue"] != "" + assert objs[2]["amqp.exchange"] == "logs" + assert objs[2]["amqp.no-wait"] == False + + assert objs[3]["method"] == "basic.consume" + assert objs[3]["event.action"] == "amqp.basic.consume" + assert objs[3]["status"] == "OK" + assert objs[3]["amqp.queue"] != "" + assert objs[3]["amqp.no-ack"] == True + assert objs[3]["amqp.no-wait"] == False + assert objs[3]["amqp.no-local"] == False + assert objs[3]["amqp.exclusive"] == False + + assert objs[4]["method"] == "exchange.declare" + assert objs[4]["event.action"] == "amqp.exchange.declare" + assert objs[4]["status"] == "OK" + assert objs[4]["amqp.exchange"] == "logs" + assert objs[4]["amqp.durable"] == True + assert objs[4]["amqp.exchange-type"] == "fanout" + assert objs[4]["amqp.passive"] == False + assert objs[4]["amqp.no-wait"] == False + + assert objs[5]["method"] == "basic.publish" + assert objs[5]["event.action"] == "amqp.basic.publish" + assert objs[5]["status"] == "OK" + assert objs[5]["amqp.content-type"] == "text/plain" + assert objs[5]["amqp.exchange"] == "logs" + assert objs[5]["amqp.immediate"] == False + assert objs[5]["amqp.mandatory"] == False + + assert objs[6]["method"] == "basic.deliver" + assert objs[6]["event.action"] == "amqp.basic.deliver" + assert objs[6]["status"] == "OK" + assert objs[6]["amqp.content-type"] == "text/plain" + assert objs[6]["amqp.delivery-tag"] == 1 + assert objs[6]["amqp.exchange"] == "logs" + assert objs[6]["amqp.redelivered"] == False diff --git a/packetbeat/tests/system/test_0053_amqp_channel_error.py b/packetbeat/tests/system/test_0053_amqp_channel_error.py index 6b9ab78ea9d..13fc991af54 100644 --- a/packetbeat/tests/system/test_0053_amqp_channel_error.py +++ b/packetbeat/tests/system/test_0053_amqp_channel_error.py @@ -1,39 +1,42 @@ -from packetbeat import BaseTest - - -class Test(BaseTest): - - def test_amqp_channel_error(self): - self.render_config_template( - amqp_ports=[5672], - ) - self.run_packetbeat(pcap="amqp_channel_error.pcap", - debug_selectors=["amqp,tcp,publish"]) - - objs = self.read_output() - assert all([o["type"] == "amqp" for o in objs]) - assert len(objs) == 3 - - assert objs[0]["method"] == "exchange.declare" - assert objs[0]["status"] == "OK" - assert objs[0]["amqp.exchange"] == "titres" - assert objs[0]["amqp.durable"] == True - assert objs[0]["amqp.exchange-type"] == "fanout" - assert objs[0]["amqp.passive"] == False - assert objs[0]["amqp.no-wait"] == True - - assert objs[1]["method"] == "queue.declare" - assert objs[1]["status"] == "OK" - assert objs[1]["amqp.queue"] == "my_queue" - assert objs[1]["amqp.exclusive"] == True - assert objs[1]["amqp.no-wait"] == False - assert objs[1]["amqp.durable"] == False - assert objs[1]["amqp.auto-delete"] == False - assert objs[1]["amqp.passive"] == False - - assert objs[2]["method"] == "channel.close" - assert objs[2]["status"] == "Error" - assert objs[2]["amqp.reply-code"] == 404 - assert objs[2]["amqp.reply-text"] == "NOT_FOUND - no exchange 'plop' in vhost '/'" - assert objs[2]["amqp.class-id"] == 50 - assert objs[2]["amqp.method-id"] == 20 +from packetbeat import BaseTest + + +class Test(BaseTest): + + def test_amqp_channel_error(self): + self.render_config_template( + amqp_ports=[5672], + ) + self.run_packetbeat(pcap="amqp_channel_error.pcap", + debug_selectors=["amqp,tcp,publish"]) + + objs = self.read_output() + assert all([o["type"] == "amqp" for o in objs]) + assert len(objs) == 3 + + assert objs[0]["method"] == "exchange.declare" + assert objs[0]["event.action"] == "amqp.exchange.declare" + assert objs[0]["status"] == "OK" + assert objs[0]["amqp.exchange"] == "titres" + assert objs[0]["amqp.durable"] == True + assert objs[0]["amqp.exchange-type"] == "fanout" + assert objs[0]["amqp.passive"] == False + assert objs[0]["amqp.no-wait"] == True + + assert objs[1]["method"] == "queue.declare" + assert objs[1]["event.action"] == "amqp.queue.declare" + assert objs[1]["status"] == "OK" + assert objs[1]["amqp.queue"] == "my_queue" + assert objs[1]["amqp.exclusive"] == True + assert objs[1]["amqp.no-wait"] == False + assert objs[1]["amqp.durable"] == False + assert objs[1]["amqp.auto-delete"] == False + assert objs[1]["amqp.passive"] == False + + assert objs[2]["method"] == "channel.close" + assert objs[2]["event.action"] == "amqp.channel.close" + assert objs[2]["status"] == "Error" + assert objs[2]["amqp.reply-code"] == 404 + assert objs[2]["amqp.reply-text"] == "NOT_FOUND - no exchange 'plop' in vhost '/'" + assert objs[2]["amqp.class-id"] == 50 + assert objs[2]["amqp.method-id"] == 20 From 00b41c3d3ac9c24b6ff075a414ba1dd7e56186a5 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Fri, 12 Jun 2020 14:17:02 -0400 Subject: [PATCH 2/5] Fix up tests --- packetbeat/protos/nfs/request_handler.go | 1 + packetbeat/protos/redis/redis.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packetbeat/protos/nfs/request_handler.go b/packetbeat/protos/nfs/request_handler.go index b6ce1cdbdb5..d4656047743 100644 --- a/packetbeat/protos/nfs/request_handler.go +++ b/packetbeat/protos/nfs/request_handler.go @@ -198,5 +198,6 @@ func (r *rpc) handleReply(xid string, xdr *xdr, ts time.Time, tcptuple *common.T } else { nfs.pbf.Event.Outcome = "failure" } + r.results(nfs.event) } } diff --git a/packetbeat/protos/redis/redis.go b/packetbeat/protos/redis/redis.go index 9012f4ebda2..bf23e94836f 100644 --- a/packetbeat/protos/redis/redis.go +++ b/packetbeat/protos/redis/redis.go @@ -327,7 +327,7 @@ func (redis *redisPlugin) newTransaction(requ, resp *redisMessage) beat.Event { fields["response"] = resp.message } - pbf.Event.Action = "redis." + strings.ToLower(fields["method"].(string)) + pbf.Event.Action = "redis." + strings.ToLower(string(requ.method)) if resp.isError { pbf.Event.Outcome = "failure" } From 8560c0fd1a517903e33ad85f74fa19d63bae35d5 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Fri, 12 Jun 2020 15:50:20 -0400 Subject: [PATCH 3/5] Add changelog entry --- CHANGELOG.next.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7f1df7c0d85..199ed9a091c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -487,6 +487,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Packetbeat* +- Add ECS fields for x509 certs, event categorization, and related IP info. {pull}19167[19167] + *Functionbeat* From 33aa850f27d124fd9d5ad13a695eb26b199841c6 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Wed, 17 Jun 2020 23:31:59 -0400 Subject: [PATCH 4/5] Add compile time upgrade guard as a reminder and add inline marshal tag for ecs --- packetbeat/pb/ecs.go | 52 ++++++++++++++++++++++-------------------- packetbeat/pb/event.go | 32 ++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 29 deletions(-) diff --git a/packetbeat/pb/ecs.go b/packetbeat/pb/ecs.go index 05dd32e5b71..b7722c2c22a 100644 --- a/packetbeat/pb/ecs.go +++ b/packetbeat/pb/ecs.go @@ -17,40 +17,42 @@ package pb -import "time" +import ( + "github.com/elastic/ecs/code/go/ecs" +) + +// Fixes for non-array datatypes +// ============================= +// +// Code at github.com/elastic/ecs/code/go/ecs has some fields as string +// when they should be []string. +// +// Once the code generator is fixed, this code will no longer compile +// which reminds us to strip out the overrides below +var ( + compileTimeUpgradeCheckEvent = ecs.Event{ + Type: "remove this when we upgrade ECS", + } + compileTimeUpgradeCheckRelated = ecs.Related{ + User: "remove this when we upgrade ECS", + } +) type ecsEvent struct { - ID string `ecs:"id"` - Code string `ecs:"code"` - Kind string `ecs:"kind"` + ecs.Event `ecs:",inline"` // overridden because this needs to be an array Category []string `ecs:"category"` - Action string `ecs:"action"` - Outcome string `ecs:"outcome"` // overridden because this needs to be an array - Type []string `ecs:"type"` - Module string `ecs:"module"` - Dataset string `ecs:"dataset"` - Provider string `ecs:"provider"` - Severity int64 `ecs:"severity"` - Original string `ecs:"original"` - Hash string `ecs:"hash"` - Duration time.Duration `ecs:"duration"` - Sequence int64 `ecs:"sequence"` - Timezone string `ecs:"timezone"` - Created time.Time `ecs:"created"` - Start time.Time `ecs:"start"` - End time.Time `ecs:"end"` - RiskScore float64 `ecs:"risk_score"` - RiskScoreNorm float64 `ecs:"risk_score_norm"` - Ingested time.Time `ecs:"ingested"` - Reference string `ecs:"reference"` - Url string `ecs:"url"` + Type []string `ecs:"type"` } type ecsRelated struct { - IP []string `ecs:"ip"` + ecs.Related `ecs:",inline"` + // overridden because this needs to be an array + IP []string `ecs:"ip"` + // overridden because this needs to be an array User []string `ecs:"user"` + // overridden because this needs to be an array Hash []string `ecs:"hash"` // for de-dup diff --git a/packetbeat/pb/event.go b/packetbeat/pb/event.go index cbc615e5539..b2f39290401 100644 --- a/packetbeat/pb/event.go +++ b/packetbeat/pb/event.go @@ -18,8 +18,10 @@ package pb import ( + "fmt" "net" "reflect" + "strings" "time" "github.com/pkg/errors" @@ -74,8 +76,10 @@ type Fields struct { func NewFields() *Fields { return &Fields{ Event: ecsEvent{ - Duration: -1, - Kind: "event", + Event: ecs.Event{ + Duration: -1, + Kind: "event", + }, Type: []string{"connection", "protocol"}, Category: []string{"network_traffic", "network"}, }, @@ -336,13 +340,33 @@ func marshalStruct(m common.MapStr, key string, val reflect.Value) error { break } + inline := false + tags := strings.Split(tag, ",") + if len(tags) > 1 { + for _, flag := range tags[1:] { + switch flag { + case "inline": + inline = true + default: + return fmt.Errorf("Unsupported flag %q in tag %q of type %s", flag, tag, typ) + } + } + tag = tags[0] + } + fieldValue := val.Field(i) if !fieldValue.IsValid() || isEmptyValue(fieldValue) { continue } - if _, err := m.Put(key+"."+tag, fieldValue.Interface()); err != nil { - return err + if inline { + if err := marshalStruct(m, key, fieldValue); err != nil { + return err + } + } else { + if _, err := m.Put(key+"."+tag, fieldValue.Interface()); err != nil { + return err + } } } return nil From e65e2ccd7ce4024bb2dc73f95a9fe4efcc6273c8 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Thu, 18 Jun 2020 10:53:08 -0400 Subject: [PATCH 5/5] Fix early termination for marshaling code --- packetbeat/pb/event.go | 2 +- packetbeat/pb/event_test.go | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packetbeat/pb/event.go b/packetbeat/pb/event.go index b2f39290401..73387c7f796 100644 --- a/packetbeat/pb/event.go +++ b/packetbeat/pb/event.go @@ -337,7 +337,7 @@ func marshalStruct(m common.MapStr, key string, val reflect.Value) error { structField := typ.Field(i) tag := getTag(structField) if tag == "" { - break + continue } inline := false diff --git a/packetbeat/pb/event_test.go b/packetbeat/pb/event_test.go index ff21e8769b6..1fdb8425756 100644 --- a/packetbeat/pb/event_test.go +++ b/packetbeat/pb/event_test.go @@ -74,3 +74,24 @@ func TestIsEmptyValue(t *testing.T) { assert.False(t, isEmptyValue(reflect.ValueOf(time.Duration(0)))) assert.True(t, isEmptyValue(reflect.ValueOf(time.Duration(-1)))) } + +func TestSkipFields(t *testing.T) { + m := common.MapStr{} + if err := MarshalStruct(m, "test", &struct { + Field1 string `ecs:"field1"` + Field2 string + Field3 string `ecs:"field3"` + }{ + Field1: "field1", + Field2: "field2", + Field3: "field3", + }); err != nil { + t.Fatal(err) + } + assert.Equal(t, common.MapStr{ + "test": common.MapStr{ + "field1": "field1", + "field3": "field3", + }, + }, m) +}