diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 12959bb5147..04dab8e0ed2 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -27,6 +27,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d *Packetbeat* +- Renamed the flow event fields to follow Elastic Common Schema. {pull}9121[9121] + *Winlogbeat* *Functionbeat* diff --git a/packetbeat/_meta/fields.common.yml b/packetbeat/_meta/fields.common.yml index d25bac0b732..3deafbfd909 100644 --- a/packetbeat/_meta/fields.common.yml +++ b/packetbeat/_meta/fields.common.yml @@ -97,185 +97,22 @@ description: > These fields contain data about the flow itself. fields: - - name: "start_time" - type: date - required: true - format: YYYY-MM-DDTHH:MM:SS.milliZ - example: 2015-01-24T14:06:05.071Z - description: > - The time, the first packet for the flow has been seen. - - - name: "last_time" - type: date - required: true - format: YYYY-MM-DDTHH:MM:SS.milliZ - example: 2015-01-24T14:06:05.071Z - description: > - The time, the most recent processed packet for the flow has been seen. - - - name: final + - name: flow.final + type: boolean description: > Indicates if event is last event in flow. If final is false, the event reports an intermediate flow state only. - - name: flow_id - description: > - Internal flow id based on connection meta data and address. - - - name: vlan + - name: flow.id description: > - Innermost VLAN address used in network packets. - - - name: outer_vlan - description: > - Second innermost VLAN address used in network packets. - - - - name: source - type: group - description: > - Properties of the source host - fields: - - name: ip_location - type: geo_point - example: "40.715, -74.011" - description: > - The GeoIP location of the `ip_source` IP address. The field is a string - containing the latitude and longitude separated by a comma. + Internal flow ID based on connection meta data and address. - - name: outer_ip - description: > - Second innermost IPv4 source address as indicated by first packet seen - for the current flow. - - - name: outer_ip_location - type: geo_point - example: "40.715, -74.011" - description: > - The GeoIP location of the `outer_ip_source` IP address. The field is a - string containing the latitude and longitude separated by a comma. - - - name: ipv6 - description: > - Innermost IPv6 source address as indicated by first packet seen for the - current flow. - - - name: ipv6_location - type: geo_point - example: "60.715, -76.011" - description: > - The GeoIP location of the `ipv6_source` IP address. The field is a string - containing the latitude and longitude separated by a comma. - - - name: outer_ipv6 - description: > - Second innermost IPv6 source address as indicated by first packet seen - for the current flow. - - - name: outer_ipv6_location - type: geo_point - example: "60.715, -76.011" - description: > - The GeoIP location of the `outer_ipv6_source` IP address. The field is a - string containing the latitude and longitude separated by a comma. - - - name: stats - type: group - description: > - Object with source to destination flow measurements. - fields: - - name: net_packets_total - type: long - description: > - Total number of packets - - - name: net_bytes_total - type: long - description: > - Total number of bytes - - - name: dest - type: group - description: > - Properties of the destination host - fields: - - name: mac - description: > - Destination MAC address as indicated by first packet seen for the current flow. - - - name: ip - description: > - Innermost IPv4 destination address as indicated by first packet seen for the - current flow. - - - name: ip_location - type: geo_point - example: "40.715, -74.011" - description: > - The GeoIP location of the `ip_dest` IP address. The field is a string - containing the latitude and longitude separated by a comma. - - - name: outer_ip - description: > - Second innermost IPv4 destination address as indicated by first packet - seen for the current flow. - - - name: outer_ip_location - type: geo_point - example: "40.715, -74.011" - description: > - The GeoIP location of the `outer_ip_dest` IP address. The field is a - string containing the latitude and longitude separated by a comma. - - - name: ipv6 - description: > - Innermost IPv6 destination address as indicated by first packet seen for the - current flow. - - - name: ipv6_location - type: geo_point - example: "60.715, -76.011" - description: > - The GeoIP location of the `ipv6_dest` IP address. The field is a string - containing the latitude and longitude separated by a comma. - - - name: outer_ipv6 - description: > - Second innermost IPv6 destination address as indicated by first packet - seen for the current flow. - - - name: outer_ipv6_location - type: geo_point - example: "60.715, -76.011" - description: > - The GeoIP location of the `outer_ipv6_dest` IP address. The field is a - string containing the latitude and longitude separated by a comma. - - - name: port - description: > - Destination port number as indicated by first packet seen for the current flow. - - - name: stats - type: group - description: > - Object with destination to source flow measurements. - fields: - - name: net_packets_total - type: long - description: > - Total number of packets - - - name: net_bytes_total - type: long - description: > - Total number of bytes - - name: icmp_id - description: > - ICMP id used in ICMP based flow. - - - name: connection_id + - name: flow.vlan + type: long description: > - optional TCP connection id + VLAN identifier from the 802.1q frame. In case of a multi-tagged frame + this field will be an array with the outer tag's VLAN identifier listed + first. - key: trans_event title: "Transaction Event" diff --git a/packetbeat/_meta/sample_outputs/flow.json b/packetbeat/_meta/sample_outputs/flow.json new file mode 100644 index 00000000000..a7cb0bf8379 --- /dev/null +++ b/packetbeat/_meta/sample_outputs/flow.json @@ -0,0 +1,34 @@ +{ + "@timestamp": "2018-11-15T14:41:24.000Z", + "destination": { + "bytes": 460, + "ip": "198.51.100.2", + "mac": "06:05:04:03:02:01", + "packets": 2, + "port": 80 + }, + "event": { + "duration": 3000000000, + "end": "2018-11-15T14:41:24.000Z", + "start": "2018-11-15T14:41:21.000Z", + "type": "flow" + }, + "flow": { + "final": true, + "id": "FQQA/wz/Dv//////Fv8BAQEBAgMEBQYGBQQDAgGrAMsAcQPGM2QC9ZdQAA", + "vlan": 171 + }, + "network": { + "bytes": 470, + "packets": 3, + "transport": "tcp", + "type": "ipv4" + }, + "source": { + "bytes": 10, + "ip": "203.0.113.3", + "mac": "01:02:03:04:05:06", + "packets": 1, + "port": 38901 + } +} \ No newline at end of file diff --git a/packetbeat/decoder/decoder.go b/packetbeat/decoder/decoder.go index 5a76d4ae4e3..d5b1d28848a 100644 --- a/packetbeat/decoder/decoder.go +++ b/packetbeat/decoder/decoder.go @@ -67,8 +67,8 @@ type Decoder struct { } const ( - netPacketsTotalCounter = "net_packets_total" - netBytesTotalCounter = "net_bytes_total" + netPacketsTotalCounter = "packets" + netBytesTotalCounter = "bytes" ) // New creates and initializes a new packet decoder. diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index fa9bd4ff270..f8a93649d21 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -2258,7 +2258,7 @@ ECS fields. [float] == agent fields -The agent fields contain the data about the agent/client/shipper that created the event. +The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken. @@ -2274,24 +2274,29 @@ Version of the agent. -- -*`agent.type`*:: +*`agent.name`*:: + -- type: keyword -example: filebeat +example: foo Name of the agent. +This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. +If no name is given, the name is often left empty. -- -*`agent.hostname`*:: +*`agent.type`*:: + -- type: keyword -Hostname of the agent. +example: filebeat + +Type of the agent. +The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine. -- @@ -2362,7 +2367,7 @@ List of keywords used to tag each event. -- type: object -example: {'key2': 'value2', 'key1': 'value1'} +example: {'application': 'foo-bar', 'env': 'production'} Key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. @@ -2382,6 +2387,183 @@ For log events the message field contains the log message. In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message. +-- + +[float] +== client fields + +A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjuction with server fields. Client fields are generally not populated for packet-level events. + + + +*`client.ip`*:: ++ +-- +type: ip + +IP address of the client. +Can be one or multiple IPv4 or IPv6 addresses. + + +-- + +*`client.port`*:: ++ +-- +type: long + +Port of the client. + + +-- + +*`client.mac`*:: ++ +-- +type: keyword + +MAC address of the client. + + +-- + +*`client.domain`*:: ++ +-- +type: keyword + +Client domain. + + +-- + +*`client.bytes`*:: ++ +-- +type: long + +example: 184 + +Bytes sent from the client to the server. + + +-- + +*`client.packets`*:: ++ +-- +type: long + +example: 12 + +Packets sent from the client to the server. + + +-- + +[float] +== geo fields + +Geo fields can carry data about a specific location related to an event or geo information derived from an IP field. + + + +*`client.geo.location`*:: ++ +-- +type: geo_point + +example: { "lon": -73.614830, "lat": 45.505918 } + +Longitude and latitude. + + +-- + +*`client.geo.continent_name`*:: ++ +-- +type: keyword + +example: North America + +Name of the continent. + + +-- + +*`client.geo.country_name`*:: ++ +-- +type: keyword + +example: Canada + +Country name. + + +-- + +*`client.geo.region_name`*:: ++ +-- +type: keyword + +example: Quebec + +Region name. + + +-- + +*`client.geo.city_name`*:: ++ +-- +type: keyword + +example: Montreal + +City name. + + +-- + +*`client.geo.country_iso_code`*:: ++ +-- +type: keyword + +example: CA + +Country ISO code. + + +-- + +*`client.geo.region_iso_code`*:: ++ +-- +type: keyword + +example: CA-QC + +Region ISO code. + + +-- + +*`client.geo.name`*:: ++ +-- +type: keyword + +example: boston-dc + +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. + + -- [float] @@ -2546,7 +2728,7 @@ Image labels. [float] == destination fields -Destination fields describe details about the destination of a packet/event. +Destination fields describe details about the destination of a packet/event. Destination fields are usually populated in conjunction with source fields. @@ -2591,161 +2773,154 @@ Destination domain. -- -[float] -== geo fields - -Geolocation for destination. - - -*`destination.geo.continent_name`*:: +*`destination.bytes`*:: + -- -type: keyword +type: long -Name of the continent. +example: 184 + +Bytes sent from the destination to the source. -- -*`destination.geo.country_iso_code`*:: +*`destination.packets`*:: + -- -type: keyword +type: long -Country ISO code. +example: 12 + +Packets sent from the destination to the source. -- +[float] +== geo fields + +Geo fields can carry data about a specific location related to an event or geo information derived from an IP field. + + + *`destination.geo.location`*:: + -- type: geo_point +example: { "lon": -73.614830, "lat": 45.505918 } + Longitude and latitude. -- -*`destination.geo.region_name`*:: +*`destination.geo.continent_name`*:: + -- type: keyword -Region name. +example: North America + +Name of the continent. -- -*`destination.geo.city_name`*:: +*`destination.geo.country_name`*:: + -- type: keyword -City name. +example: Canada + +Country name. -- -*`destination.geo.region_iso_code`*:: +*`destination.geo.region_name`*:: + -- type: keyword -Region ISO code. - - --- - -[float] -== device fields +example: Quebec -Device fields are used to provide additional information about the device that is the source of the information. This could be a firewall, network device, etc. +Region name. +-- -*`device.mac`*:: +*`destination.geo.city_name`*:: + -- type: keyword -MAC address of the device +example: Montreal - --- - -*`device.ip`*:: -+ --- -type: ip - -IP address of the device. +City name. -- -*`device.hostname`*:: +*`destination.geo.country_iso_code`*:: + -- type: keyword -Hostname of the device. - - --- - -*`device.vendor`*:: -+ --- -type: text +example: CA -Device vendor information. +Country ISO code. -- -*`device.version`*:: +*`destination.geo.region_iso_code`*:: + -- type: keyword -Device version. +example: CA-QC + +Region ISO code. -- -*`device.serial_number`*:: +*`destination.geo.name`*:: + -- type: keyword -Device serial number. +example: boston-dc +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. --- -*`device.timezone.offset.sec`*:: -+ -- -type: long -example: -5400 +[float] +== ecs fields -Timezone offset of the host in seconds. -Number of seconds relative to UTC. If the offset is -01:30 the value will be -5400. +Meta-information specific to ECS. --- -*`device.type`*:: +*`ecs.version`*:: + -- type: keyword -example: firewall +example: 1.0.0-beta1 + +required: True -The type of the device the data is coming from. -There is no predefined list of device types. Some examples are `endpoint`, `firewall`, `ids`, `ips`, `proxy`. +ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. +When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. +The current version is 1.0.0-beta1 . -- @@ -2790,7 +2965,7 @@ Error code describing the error. [float] == event fields -The event fields are used for context information about the data itself. +The event fields are used for context information about the log or metric event itself. A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. @@ -2804,6 +2979,19 @@ example: 8a4f500d Unique ID to describe the event. +-- + +*`event.kind`*:: ++ +-- +type: keyword + +example: state + +The kind of the event. +This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. + + -- *`event.category`*:: @@ -2811,36 +2999,47 @@ Unique ID to describe the event. -- type: keyword -example: metrics +example: user-management Event category. -This can be a user defined category. +This contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. -- -*`event.type`*:: +*`event.action`*:: + -- type: keyword -example: nginx-stats-metrics +example: user-password-change -A type given to this kind of event which can be used for grouping. -This is normally defined by the user. +The action captured by the event. +This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. -- -*`event.action`*:: +*`event.outcome`*:: + -- type: keyword -example: reject +example: success + +The outcome of the event. +If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. + + +-- + +*`event.type`*:: ++ +-- +type: keyword -The action captured by the event. The type of action will vary from system to system but is likely to include actions by security services, such as blocking or quarantining; as well as more generic actions such as login events, file i/o or proxy forwarding events. -The value is normally defined by the user. +Reserved for future usage. +Please avoid using this field for user data. -- @@ -2910,29 +3109,24 @@ Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log i -- -*`event.version`*:: +*`event.duration`*:: + -- -type: keyword - -example: 0.1.0 - -required: True +type: long -The version field contains the version an event for ECS adheres to. -This field should be provided as part of each event to make it possible to detect to which ECS version an event belongs. -event.version is a required field and must exist in all events. It describes which ECS version the event adheres to. -The current version is 0.1.0. +Duration of the event in nanoseconds. +If event.start and event.end are known this value should be the difference between the end and start time. -- -*`event.duration`*:: +*`event.timezone`*:: + -- -type: long +type: keyword -Duration of the event in nanoseconds. +This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. +Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). -- @@ -2949,12 +3143,32 @@ In case the two timestamps are identical, @timestamp should be used. -- -*`event.risk_score`*:: +*`event.start`*:: + -- -type: float +type: date -Risk score or priority of the event (e.g. security solutions). Use your system's original value here. +event.start contains the date when the event started or when the activity was first observed. + + +-- + +*`event.end`*:: ++ +-- +type: date + +event.end contains the date when the event ended or when the activity was last observed. + + +-- + +*`event.risk_score`*:: ++ +-- +type: float + +Risk score or priority of the event (e.g. security solutions). Use your system's original value here. -- @@ -2973,45 +3187,25 @@ This is mainly useful if you use more than one system that assigns risk scores, [float] == file fields -File fields provide details about each file. +A file is defined as a set of information that has been created on, or has existed on a filesystem. File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. *`file.path`*:: + -- -type: text - -Path to the file. - -*`file.path.raw`*:: -+ --- type: keyword -Path to the file. This is a non-analyzed field that is useful for aggregations. - - --- +Path to the file. -- *`file.target_path`*:: + -- -type: text - -Target path for symlinks. - -*`file.target_path.raw`*:: -+ --- type: keyword -Path to the file. This is a non-analyzed field that is useful for aggregations. - - --- +Target path for symlinks. -- @@ -3131,193 +3325,284 @@ Last time file metadata changed. -- [float] -== geo fields +== group fields -Geo fields can carry data about a specific location related to an event or geo information for an IP field. +The group fields are meant to represent groups that are relevant to the event. -*`geo.continent_name`*:: +*`group.id`*:: + -- type: keyword -Name of the continent. +Unique identifier for the group on the system/platform. -- -*`geo.country_iso_code`*:: +*`group.name`*:: + -- type: keyword -Country ISO code. +Name of the group. -- -*`geo.location`*:: +[float] +== host fields + +A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or on which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. + + + +*`host.hostname`*:: + -- -type: geo_point +type: keyword -Longitude and latitude. +Hostname of the host. +It normally contains what the `hostname` command returns on the host machine. -- -*`geo.region_name`*:: +*`host.name`*:: + -- type: keyword -Region name. +Name of the host. +It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. -- -*`geo.city_name`*:: +*`host.id`*:: + -- type: keyword -City name. +Unique host id. +As hostname is not always unique, use values that are meaningful in your environment. +Example: The current usage of `beat.name`. + + +-- + +*`host.ip`*:: ++ +-- +type: ip + +Host ip address. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +Host mac address. + + +-- + +*`host.type`*:: ++ +-- +type: keyword + +Type of host. +For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. + + +-- + +*`host.architecture`*:: ++ +-- +type: keyword + +example: x86_64 + +Operating system architecture. -- [float] -== host fields +== os fields -Host fields provide information related to a host. A host can be a physical machine, a virtual machine, or a Docker container. -Normally the host information is related to the machine on which the event was generated/collected, but they can be used differently if needed. +The OS fields contain information about the operating system. -*`host.timezone.offset.sec`*:: +*`host.os.platform`*:: + -- -type: long +type: keyword -example: -5400 +example: darwin -Timezone offset of the host in seconds. -Number of seconds relative to UTC. If the offset is -01:30 the value will be -5400. +Operating system platform (such centos, ubuntu, windows). -- -*`host.name`*:: +*`host.os.name`*:: + -- type: keyword -host.name is the hostname of the host. -It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. +example: Mac OS X + +Operating system name. -- -*`host.id`*:: +*`host.os.family`*:: + -- type: keyword -Unique host id. -As hostname is not always unique, use values that are meaningful in your environment. -Example: The current usage of `beat.name`. +example: debian + +OS family (such as redhat, debian, freebsd, windows). -- -*`host.ip`*:: +*`host.os.version`*:: + -- -type: ip +type: keyword -Host ip address. +example: 10.12.6-rc2 + +Operating system version as a raw string. -- -*`host.mac`*:: +*`host.os.kernel`*:: + -- type: keyword -Host mac address. +example: 4.4.0-112-generic + +Operating system kernel version as a raw string. -- -*`host.type`*:: +[float] +== geo fields + +Geo fields can carry data about a specific location related to an event or geo information derived from an IP field. + + + +*`host.geo.location`*:: ++ +-- +type: geo_point + +example: { "lon": -73.614830, "lat": 45.505918 } + +Longitude and latitude. + + +-- + +*`host.geo.continent_name`*:: + -- type: keyword -Type of host. -For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. +example: North America + +Name of the continent. -- -*`host.os.platform`*:: +*`host.geo.country_name`*:: + -- type: keyword -example: darwin +example: Canada -Operating system platform (centos, ubuntu, windows, etc.) +Country name. -- -*`host.os.name`*:: +*`host.geo.region_name`*:: + -- type: keyword -example: Mac OS X +example: Quebec -Operating system name. +Region name. -- -*`host.os.family`*:: +*`host.geo.city_name`*:: + -- type: keyword -example: debian +example: Montreal -OS family (redhat, debian, freebsd, windows, etc.) +City name. -- -*`host.os.version`*:: +*`host.geo.country_iso_code`*:: + -- type: keyword -example: 10.12.6 +example: CA -Operating system version. +Country ISO code. -- -*`host.architecture`*:: +*`host.geo.region_iso_code`*:: + -- type: keyword -example: x86_64 +example: CA-QC -Operating system architecture. +Region ISO code. + + +-- + +*`host.geo.name`*:: ++ +-- +type: keyword + +example: boston-dc + +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -- @@ -3325,7 +3610,7 @@ Operating system architecture. [float] == http fields -Fields related to HTTP requests and responses. +Fields related to HTTP activity. @@ -3339,6 +3624,18 @@ example: GET, POST, PUT Http request method. +-- + +*`http.request.referrer`*:: ++ +-- +type: keyword + +example: https://blog.example.com/ + +Referrer for this HTTP request. + + -- *`http.response.status_code`*:: @@ -3356,7 +3653,7 @@ Http response status code. *`http.response.body`*:: + -- -type: text +type: keyword example: Hello world @@ -3417,1155 +3714,1528 @@ Field is not indexed. [float] == network fields -Fields related to network data. +The network is defined as the communication path over which a host or network event happens. The network.* fields should be populated with details about the network activity associated with an event. *`network.name`*:: + -- -type: text +type: keyword example: Guest Wifi Name given by operators to sections of their network. -*`network.name.raw`*:: +-- + +*`network.type`*:: + -- type: keyword -Name given by operators to sections of their network. +example: IPv4 +In the OSI Model this would be the Network Layer. IPv4, IPv6, IPSec, PIM, etc --- -- -*`network.protocol`*:: +*`network.iana_number`*:: + -- type: keyword -example: http +example: 6 -Network protocol name. +IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. -- -*`network.direction`*:: +*`network.transport`*:: + -- type: keyword -example: inbound +example: TCP -Direction of the network traffic. -Recommended values are: - * inbound - * outbound - * unknown +Same as network.iana_number, but instead using the Keyword name of the transport layer (UDP, TCP, IPv6-ICMP, etc.) -- -*`network.forwarded_ip`*:: +*`network.application`*:: + -- -type: ip +type: keyword -example: 192.1.1.2 +example: AIM -Host IP address when the source IP address is the proxy. +A name given to an application. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. -- -*`network.inbound.bytes`*:: +*`network.protocol`*:: + -- -type: long +type: keyword -example: 184 +example: http -Network inbound bytes. +L7 Network protocol name. ex. http, lumberjack, transport protocol -- -*`network.inbound.packets`*:: +*`network.direction`*:: + -- -type: long +type: keyword -example: 12 +example: inbound + +Direction of the network traffic. +Recommended values are: + * inbound + * outbound + * internal + * external + * unknown -Network inbound packets. +When mapping events from a host-based monitoring context, populate this field from the host's point of view. +When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. -- -*`network.outbound.bytes`*:: +*`network.forwarded_ip`*:: + -- -type: long +type: ip -example: 184 +example: 192.1.1.2 -Network outbound bytes. +Host IP address when the source IP address is the proxy. -- -*`network.outbound.packets`*:: +*`network.community_id`*:: + -- -type: long +type: keyword -example: 12 +example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= -Network outbound packets. +A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. +Learn more at https://github.com/corelight/community-id-spec. -- -*`network.total.bytes`*:: +*`network.bytes`*:: + -- type: long example: 368 -Network total bytes. The sum of inbound.bytes + outbound.bytes. +Total bytes transferred in both directions. +If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. -- -*`network.total.packets`*:: +*`network.packets`*:: + -- type: long example: 24 -Network outbound packets. The sum of inbound.packets + outbound.packets +Total packets transferred in both directions. +If `source.packets` and `destination.packets` are known, `network.packets` is their sum. -- [float] -== organization fields +== observer fields -The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations. +An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. -*`organization.name`*:: +*`observer.mac`*:: + -- -type: text +type: keyword -Organization name. +MAC address of the observer -- -*`organization.id`*:: +*`observer.ip`*:: + -- -type: keyword +type: ip -Unique identifier for the organization. +IP address of the observer. -- -[float] -== os fields +*`observer.hostname`*:: ++ +-- +type: keyword -The OS fields contain information about the operating system. These fields are often used inside other prefixes, such as `host.os.*` or `user_agent.os.*`. +Hostname of the observer. +-- -*`os.platform`*:: +*`observer.vendor`*:: + -- type: keyword -example: darwin - -Operating system platform (such centos, ubuntu, windows). - - --- - -*`os.name`*:: -+ --- -type: keyword - -example: Mac OS X - -Operating system name. +observer vendor information. -- -*`os.family`*:: +*`observer.version`*:: + -- type: keyword -example: debian - -OS family (such as redhat, debian, freebsd, windows). +Observer version. -- -*`os.version`*:: +*`observer.serial_number`*:: + -- type: keyword -example: 10.12.6-rc2 - -Operating system version as a raw string. +Observer serial number. -- -*`os.kernel`*:: +*`observer.type`*:: + -- type: keyword -example: 4.4.0-112-generic +example: firewall -Operating system kernel version as a raw string. +The type of the observer the data is coming from. +There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. -- [float] -== process fields +== os fields -These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. +The OS fields contain information about the operating system. -*`process.args`*:: +*`observer.os.platform`*:: + -- type: keyword -example: ['-l', 'user', '10.0.0.16'] +example: darwin -Process arguments. -May be filtered to protect sensitive information. +Operating system platform (such centos, ubuntu, windows). -- -*`process.name`*:: +*`observer.os.name`*:: + -- type: keyword -example: ssh +example: Mac OS X -Process name. -Sometimes called program name or similar. +Operating system name. -- -*`process.pid`*:: +*`observer.os.family`*:: + -- -type: long +type: keyword -Process id. +example: debian + +OS family (such as redhat, debian, freebsd, windows). -- -*`process.ppid`*:: +*`observer.os.version`*:: + -- -type: long +type: keyword -Process parent id. +example: 10.12.6-rc2 + +Operating system version as a raw string. -- -*`process.title`*:: +*`observer.os.kernel`*:: + -- type: keyword -Process title. -The proctitle, often the same as process name. +example: 4.4.0-112-generic + +Operating system kernel version as a raw string. -- [float] -== service fields +== geo fields -The service fields describe the service for or from which the data was collected. These fields help you find and correlate logs for a specific service and version. +Geo fields can carry data about a specific location related to an event or geo information derived from an IP field. -*`service.id`*:: +*`observer.geo.location`*:: + -- -type: keyword +type: geo_point -example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 +example: { "lon": -73.614830, "lat": 45.505918 } -Unique identifier of the running service. -This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. -Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. +Longitude and latitude. -- -*`service.name`*:: +*`observer.geo.continent_name`*:: + -- type: keyword -example: elasticsearch +example: North America -Name of the service data is collected from. -The name can be used to group and correlate logs and metrics from one service. -Example: If logs or metrics are collected from Redis, `service.name` would be `redis`. +Name of the continent. -- -*`service.type`*:: +*`observer.geo.country_name`*:: + -- type: keyword -Service type. +example: Canada + +Country name. -- -*`service.state`*:: +*`observer.geo.region_name`*:: + -- type: keyword -Current state of the service. +example: Quebec + +Region name. -- -*`service.version`*:: +*`observer.geo.city_name`*:: + -- type: keyword -example: 3.2.4 +example: Montreal -Version of the service the data was collected from. -This allows to look at a data set only for a specific version of a service. +City name. -- -*`service.ephemeral_id`*:: +*`observer.geo.country_iso_code`*:: + -- type: keyword -example: 8a4f500f +example: CA -Ephemeral identifier of this service (if one exists). -This id normally changes across restarts, but `service.id` does not. +Country ISO code. -- -[float] -== source fields +*`observer.geo.region_iso_code`*:: ++ +-- +type: keyword -Source fields describe details about the source of the event. +example: CA-QC +Region ISO code. -*`source.ip`*:: +-- + +*`observer.geo.name`*:: + -- -type: ip +type: keyword -IP address of the source. -Can be one or multiple IPv4 or IPv6 addresses. +example: boston-dc +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. --- -*`source.port`*:: -+ -- -type: long -Port of the source. +[float] +== organization fields +The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations. --- -*`source.mac`*:: + +*`organization.name`*:: + -- type: keyword -MAC address of the source. +Organization name. -- -*`source.domain`*:: +*`organization.id`*:: + -- type: keyword -Source domain. +Unique identifier for the organization. -- [float] -== geo fields +== os fields -Geolocation for source. +The OS fields contain information about the operating system. -*`source.geo.continent_name`*:: + +*`os.platform`*:: + -- type: keyword -Name of the continent. +example: darwin + +Operating system platform (such centos, ubuntu, windows). -- -*`source.geo.country_iso_code`*:: +*`os.name`*:: + -- type: keyword -Country ISO code. - - --- - -*`source.geo.location`*:: -+ --- -type: geo_point +example: Mac OS X -Longitude and latitude. +Operating system name. -- -*`source.geo.region_name`*:: +*`os.family`*:: + -- type: keyword -Region name. +example: debian + +OS family (such as redhat, debian, freebsd, windows). -- -*`source.geo.city_name`*:: +*`os.version`*:: + -- type: keyword -City name. +example: 10.12.6-rc2 + +Operating system version as a raw string. -- -*`source.geo.region_iso_code`*:: +*`os.kernel`*:: + -- type: keyword -Region ISO code. +example: 4.4.0-112-generic + +Operating system kernel version as a raw string. -- [float] -== url fields +== process fields -URL fields provide a complete URL, with scheme, host, and path. The URL object can be reused in other prefixes, such as `host.url.*` for example. Keep the structure consistent whenever you use URL fields. +These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. -*`url.original`*:: +*`process.pid`*:: + -- -type: keyword - -example: https://elastic.co:443/search?q=elasticsearch#top +type: long -Full url. The field is stored as keyword. -`url.href` is a [multi field](https://www.elastic.co/guide/en/ elasticsearch/reference/6.2/ multi-fields.html#_multi_fields_with_multiple_analyzers). The data is stored as keyword `url.href` and test `url.href.analyzed`. These fields enable you to run a query against part of the url still works splitting up the URL at ingest time. -`href` is an analyzed field so the parsed information can be accessed through `href.analyzed` in queries. +Process id. -- -*`url.scheme`*:: +*`process.name`*:: + -- type: keyword -example: https +example: ssh -Scheme of the request, such as "https". -Note: The `:` is not part of the scheme. +Process name. +Sometimes called program name or similar. -- -*`url.hostname`*:: +*`process.ppid`*:: + -- -type: keyword - -example: elastic.co +type: long -Hostname of the request, such as "elastic.co". -In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `hostname` field. +Process parent id. -- -*`url.port`*:: +*`process.args`*:: + -- -type: integer +type: keyword -example: 443 +example: ['ssh', '-l', 'user', '10.0.0.16'] -Port of the request, such as 443. +Process arguments. +May be filtered to protect sensitive information. -- -*`url.path`*:: +*`process.executable`*:: + -- -type: text +type: keyword -Path of the request, such as "/search". +example: /usr/bin/ssh + +Absolute path to the process executable. -*`url.path.raw`*:: +-- + +*`process.title`*:: + -- type: keyword -URL path. A non-analyzed field that is useful for aggregations. - +Process title. +The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. --- -- -*`url.query`*:: +*`process.thread.id`*:: + -- -type: text +type: long -The query field describes the query string of the request, such as "q=elasticsearch". -The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. +example: 4242 + +Thread ID. -*`url.query.raw`*:: +-- + +*`process.start`*:: + -- -type: keyword +type: date -URL query part. A non-analyzed field that is useful for aggregations. +example: 2016-05-23T08:05:34.853Z +The time the process started. --- -- -*`url.fragment`*:: +*`process.working_directory`*:: + -- type: keyword -Portion of the url after the `#`, such as "top". -The `#` is not part of the fragment. +example: /home/alice +The working directory of the process. --- -*`url.username`*:: -+ -- -type: keyword -Username of the request. +[float] +== related fields +This field set is meant to facilitate pivoting around a piece of data. Some pieces of information can be seen in many places in ECS. To facilitate searching for them, append values to their corresponding field in `related.`. A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. --- -*`url.password`*:: + +*`related.ip`*:: + -- -type: keyword +type: ip -Password of the request. +All of the IPs seen on your event. -- [float] -== user fields +== server fields -The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. +A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. -*`user.id`*:: +*`server.ip`*:: + -- -type: keyword +type: ip -One or multiple unique identifiers of the user. +IP address of the server. +Can be one or multiple IPv4 or IPv6 addresses. -- -*`user.name`*:: +*`server.port`*:: + -- -type: keyword +type: long -Name of the user. -The field is a keyword, and will not be tokenized. +Port of the server. -- -*`user.email`*:: +*`server.mac`*:: + -- type: keyword -User email address. +MAC address of the server. -- -*`user.hash`*:: +*`server.domain`*:: + -- type: keyword -Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +Server domain. -- -[float] -== user_agent fields +*`server.bytes`*:: ++ +-- +type: long -The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string. +example: 184 +Bytes sent from the server to the client. -*`user_agent.original`*:: -+ -- -type: text -Unparsed version of the user_agent. +*`server.packets`*:: ++ +-- +type: long + +example: 12 + +Packets sent from the server to the client. -- -*`user_agent.device`*:: +[float] +== geo fields + +Geo fields can carry data about a specific location related to an event or geo information derived from an IP field. + + + +*`server.geo.location`*:: ++ +-- +type: geo_point + +example: { "lon": -73.614830, "lat": 45.505918 } + +Longitude and latitude. + + +-- + +*`server.geo.continent_name`*:: + -- type: keyword -Name of the physical device. +example: North America + +Name of the continent. -- -*`user_agent.version`*:: +*`server.geo.country_name`*:: + -- type: keyword -Version of the physical device. +example: Canada + +Country name. -- -*`user_agent.major`*:: +*`server.geo.region_name`*:: + -- -type: long +type: keyword -Major version of the user agent. +example: Quebec + +Region name. -- -*`user_agent.minor`*:: +*`server.geo.city_name`*:: + -- -type: long +type: keyword -Minor version of the user agent. +example: Montreal + +City name. -- -*`user_agent.patch`*:: +*`server.geo.country_iso_code`*:: + -- type: keyword -Patch version of the user agent. +example: CA + +Country ISO code. -- -*`user_agent.name`*:: +*`server.geo.region_iso_code`*:: + -- type: keyword -example: Chrome +example: CA-QC -Name of the user agent. +Region ISO code. -- -*`user_agent.os.name`*:: +*`server.geo.name`*:: + -- type: keyword -Name of the operating system. +example: boston-dc + +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -- -*`user_agent.os.version`*:: +[float] +== service fields + +The service fields describe the service for or from which the data was collected. These fields help you find and correlate logs for a specific service and version. + + + +*`service.id`*:: + -- type: keyword -Version of the operating system. +example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 + +Unique identifier of the running service. +This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. +Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. -- -*`user_agent.os.major`*:: +*`service.name`*:: + -- -type: long +type: keyword -Major version of the operating system. +example: elasticsearch-metrics + +Name of the service data is collected from. +The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. +Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. +In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified. -- -*`user_agent.os.minor`*:: +*`service.type`*:: ++ +-- +type: keyword + +example: elasticsearch + +The type of the service data is collected from. +The type can be used to group and correlate logs and metrics from one service type. +Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + + +-- + +*`service.state`*:: ++ +-- +type: keyword + +Current state of the service. + + +-- + +*`service.version`*:: ++ +-- +type: keyword + +example: 3.2.4 + +Version of the service the data was collected from. +This allows to look at a data set only for a specific version of a service. + + +-- + +*`service.ephemeral_id`*:: ++ +-- +type: keyword + +example: 8a4f500f + +Ephemeral identifier of this service (if one exists). +This id normally changes across restarts, but `service.id` does not. + + +-- + +[float] +== source fields + +Source fields describe details about the source of a packet/event. Source fields are usually populated in conjunction with destination fields. + + + +*`source.ip`*:: ++ +-- +type: ip + +IP address of the source. +Can be one or multiple IPv4 or IPv6 addresses. + + +-- + +*`source.port`*:: + -- type: long -Minor version of the operating system. +Port of the source. -- -*`beat.name`*:: +*`source.mac`*:: + -- -type: alias +type: keyword + +MAC address of the source. -alias to: agent.type -- -*`beat.hostname`*:: +*`source.domain`*:: + -- -type: alias +type: keyword + +Source domain. -alias to: agent.hostname -- -[[exported-fields-flows_event]] -== Flow Event fields +*`source.bytes`*:: ++ +-- +type: long -These fields contain data about the flow itself. +example: 184 +Bytes sent from the source to the destination. -*`start_time`*:: +-- + +*`source.packets`*:: + -- -type: date +type: long -example: 2015-01-24 14:06:05.071000 +example: 12 -format: YYYY-MM-DDTHH:MM:SS.milliZ +Packets sent from the source to the destination. -required: True -The time, the first packet for the flow has been seen. +-- + +[float] +== geo fields + +Geo fields can carry data about a specific location related to an event or geo information derived from an IP field. + +*`source.geo.location`*:: ++ -- +type: geo_point + +example: { "lon": -73.614830, "lat": 45.505918 } + +Longitude and latitude. + -*`last_time`*:: +-- + +*`source.geo.continent_name`*:: + -- -type: date +type: keyword -example: 2015-01-24 14:06:05.071000 +example: North America -format: YYYY-MM-DDTHH:MM:SS.milliZ +Name of the continent. -required: True -The time, the most recent processed packet for the flow has been seen. +-- + +*`source.geo.country_name`*:: ++ +-- +type: keyword + +example: Canada + +Country name. -- -*`final`*:: +*`source.geo.region_name`*:: + -- -Indicates if event is last event in flow. If final is false, the event reports an intermediate flow state only. +type: keyword + +example: Quebec + +Region name. -- -*`flow_id`*:: +*`source.geo.city_name`*:: + -- -Internal flow id based on connection meta data and address. +type: keyword + +example: Montreal + +City name. -- -*`vlan`*:: +*`source.geo.country_iso_code`*:: + -- -Innermost VLAN address used in network packets. +type: keyword + +example: CA + +Country ISO code. -- -*`outer_vlan`*:: +*`source.geo.region_iso_code`*:: + -- -Second innermost VLAN address used in network packets. +type: keyword + +example: CA-QC + +Region ISO code. + + +-- + +*`source.geo.name`*:: ++ +-- +type: keyword + +example: boston-dc + +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -- [float] -== source fields +== url fields -Properties of the source host +URL fields provide a complete URL, with scheme, host, and path. -*`source.ip_location`*:: +*`url.original`*:: + -- -type: geo_point +type: keyword -example: 40.715, -74.011 +example: https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch -The GeoIP location of the `ip_source` IP address. The field is a string containing the latitude and longitude separated by a comma. + +Unmodified original url as seen in the event source. +Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. +This field is meant to represent the URL as it was observed, complete or not. -- -*`source.outer_ip`*:: +*`url.full`*:: + -- -Second innermost IPv4 source address as indicated by first packet seen for the current flow. +type: keyword + +example: https://www.elastic.co:443/search?q=elasticsearch#top + +If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. -- -*`source.outer_ip_location`*:: +*`url.scheme`*:: + -- -type: geo_point +type: keyword -example: 40.715, -74.011 +example: https -The GeoIP location of the `outer_ip_source` IP address. The field is a string containing the latitude and longitude separated by a comma. +Scheme of the request, such as "https". +Note: The `:` is not part of the scheme. -- -*`source.ipv6`*:: +*`url.domain`*:: + -- -Innermost IPv6 source address as indicated by first packet seen for the current flow. +type: keyword + +example: www.elastic.co + +Domain of the request, such as "www.elastic.co". +In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. -- -*`source.ipv6_location`*:: +*`url.port`*:: + -- -type: geo_point +type: integer -example: 60.715, -76.011 +example: 443 -The GeoIP location of the `ipv6_source` IP address. The field is a string containing the latitude and longitude separated by a comma. +Port of the request, such as 443. -- -*`source.outer_ipv6`*:: +*`url.path`*:: + -- -Second innermost IPv6 source address as indicated by first packet seen for the current flow. +type: keyword + +Path of the request, such as "/search". -- -*`source.outer_ipv6_location`*:: +*`url.query`*:: + -- -type: geo_point +type: keyword + +The query field describes the query string of the request, such as "q=elasticsearch". +The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. + + +-- + +*`url.fragment`*:: ++ +-- +type: keyword + +Portion of the url after the `#`, such as "top". +The `#` is not part of the fragment. + -example: 60.715, -76.011 +-- + +*`url.username`*:: ++ +-- +type: keyword + +Username of the request. + + +-- + +*`url.password`*:: ++ +-- +type: keyword -The GeoIP location of the `outer_ipv6_source` IP address. The field is a string containing the latitude and longitude separated by a comma. +Password of the request. -- [float] -== stats fields +== user fields -Object with source to destination flow measurements. +The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. -*`source.stats.net_packets_total`*:: +*`user.id`*:: + -- -type: long +type: keyword -Total number of packets +One or multiple unique identifiers of the user. -- -*`source.stats.net_bytes_total`*:: +*`user.name`*:: + -- -type: long +type: keyword + +example: albert -Total number of bytes +Short name or login of the user. + + +-- + +*`user.full_name`*:: ++ +-- +type: keyword + +example: Albert Einstein + +User's full name, if available. + + +-- + +*`user.email`*:: ++ +-- +type: keyword + +User email address. + + +-- + +*`user.hash`*:: ++ +-- +type: keyword + +Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. + + +-- + +*`user.group`*:: ++ +-- +type: keyword + +Group the user is a part of. This field can contain a list of groups, if necessary. -- [float] -== dest fields +== user_agent fields -Properties of the destination host +The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string. -*`dest.mac`*:: +*`user_agent.original`*:: + -- -Destination MAC address as indicated by first packet seen for the current flow. +type: keyword + +example: Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 + +Unparsed version of the user_agent. +Field is not indexed. + -- -*`dest.ip`*:: +*`user_agent.name`*:: + -- -Innermost IPv4 destination address as indicated by first packet seen for the current flow. +type: keyword + +example: Safari + +Name of the user agent. -- -*`dest.ip_location`*:: +*`user_agent.version`*:: + -- -type: geo_point +type: keyword -example: 40.715, -74.011 +example: 12.0 -The GeoIP location of the `ip_dest` IP address. The field is a string containing the latitude and longitude separated by a comma. +Version of the user agent. -- -*`dest.outer_ip`*:: +*`user_agent.device.name`*:: + -- -Second innermost IPv4 destination address as indicated by first packet seen for the current flow. +type: keyword + +example: iPhone + +Name of the device. -- -*`dest.outer_ip_location`*:: +[float] +== os fields + +The OS fields contain information about the operating system. + + + +*`user_agent.os.platform`*:: + -- -type: geo_point +type: keyword -example: 40.715, -74.011 +example: darwin -The GeoIP location of the `outer_ip_dest` IP address. The field is a string containing the latitude and longitude separated by a comma. +Operating system platform (such centos, ubuntu, windows). -- -*`dest.ipv6`*:: +*`user_agent.os.name`*:: + -- -Innermost IPv6 destination address as indicated by first packet seen for the current flow. +type: keyword + +example: Mac OS X + +Operating system name. -- -*`dest.ipv6_location`*:: +*`user_agent.os.family`*:: + -- -type: geo_point +type: keyword -example: 60.715, -76.011 +example: debian -The GeoIP location of the `ipv6_dest` IP address. The field is a string containing the latitude and longitude separated by a comma. +OS family (such as redhat, debian, freebsd, windows). -- -*`dest.outer_ipv6`*:: +*`user_agent.os.version`*:: + -- -Second innermost IPv6 destination address as indicated by first packet seen for the current flow. +type: keyword + +example: 10.12.6-rc2 + +Operating system version as a raw string. -- -*`dest.outer_ipv6_location`*:: +*`user_agent.os.kernel`*:: + -- -type: geo_point +type: keyword -example: 60.715, -76.011 +example: 4.4.0-112-generic -The GeoIP location of the `outer_ipv6_dest` IP address. The field is a string containing the latitude and longitude separated by a comma. +Operating system kernel version as a raw string. -- -*`dest.port`*:: +*`beat.name`*:: + -- -Destination port number as indicated by first packet seen for the current flow. +type: alias +alias to: agent.type -- -[float] -== stats fields +*`beat.hostname`*:: ++ +-- +type: alias -Object with destination to source flow measurements. +alias to: agent.hostname +-- +*`agent.hostname`*:: ++ +-- +type: keyword + +Hostname of the agent. + + +-- -*`dest.stats.net_packets_total`*:: +*`user_agent.patch`*:: + -- -type: long +type: keyword -Total number of packets +Patch version of the user agent. -- -*`dest.stats.net_bytes_total`*:: +*`user_agent.minor`*:: + -- -type: long +type: keyword -Total number of bytes +Minor version of the user agent. -- -*`icmp_id`*:: +*`user_agent.major`*:: + -- -ICMP id used in ICMP based flow. +type: keyword + +Major version of the user agent. -- -*`connection_id`*:: +*`user_agent.device`*:: + -- -optional TCP connection id +type: keyword + +Name of the physical device. -- -[[exported-fields-host-processor]] -== Host fields +*`user_agent.os.major`*:: ++ +-- +type: long -Info collected for the host machine. +Major version of the operating system. + + +-- + +*`user_agent.os.minor`*:: ++ +-- +type: long +Minor version of the operating system. +-- -*`host.os.kernel`*:: +*`url.hostname`*:: + -- type: keyword -The operating system's kernel version. +Hostname of the request, such as "elastic.co". -- +[[exported-fields-flows_event]] +== Flow Event fields + +These fields contain data about the flow itself. + + + +*`flow.final`*:: ++ +-- +type: boolean + +Indicates if event is last event in flow. If final is false, the event reports an intermediate flow state only. + + +-- + +*`flow.id`*:: ++ +-- +Internal flow ID based on connection meta data and address. + + +-- + +*`flow.vlan`*:: ++ +-- +type: long + +VLAN identifier from the 802.1q frame. In case of a multi-tagged frame this field will be an array with the outer tag's VLAN identifier listed first. + + +-- + +[[exported-fields-host-processor]] +== Host fields + +Info collected for the host machine. + + [[exported-fields-http]] == HTTP fields diff --git a/packetbeat/docs/packetbeat-options.asciidoc b/packetbeat/docs/packetbeat-options.asciidoc index 52741d4837c..9cdd06255ef 100644 --- a/packetbeat/docs/packetbeat-options.asciidoc +++ b/packetbeat/docs/packetbeat-options.asciidoc @@ -284,45 +284,51 @@ Here’s an example of a flow information sent by Packetbeat. See ["source","json",subs="attributes"] -------------------------------------------------------------------------------- { - "@timestamp": "2017-05-03T19:42:40.003Z", - "beat": { + "@timestamp": "2018-11-15T14:41:24.000Z", + "agent": { "hostname": "host.example.com", "name": "host.example.com", "version": "{stack-version}" }, - "connection_id": "AQAAAAAAAAA=", - "dest": { - "ip": "192.0.2.0", - "mac": "fe:ff:20:00:01:00", - "port": 80, - "stats": { - "net_bytes_total": 19236, - "net_packets_total": 16 - } + "destination": { + "bytes": 460, + "ip": "198.51.100.2", + "mac": "06:05:04:03:02:01", + "packets": 2, + "port": 80 }, - "final": false, <1> - "flow_id": "EQwA////DP//////FBgBAAEAAAEAAAD+/yAAAQCR/qDtQdDk3ywNUAABAAAAAAAAAA", - "last_time": "2017-05-03T19:42:24.151Z", - "source": { - "ip": "203.0.113.0", - "mac": "00:00:01:00:00:00", - "port": 3372, - "stats": { - "net_bytes_total": 1243, - "net_packets_total": 14 - } + "event": { + "duration": 3000000000, + "end": "2018-11-15T14:41:24.000Z", + "start": "2018-11-15T14:41:21.000Z", + "type": "flow" + }, + "flow": { + "final": true, <1> + "id": "FQQA/wz/Dv//////Fv8BAQEBAgMEBQYGBQQDAgGrAMsAcQPGM2QC9ZdQAA", + "vlan": 171 + }, + "network": { + "bytes": 470, + "packets": 3, + "transport": "tcp", + "type": "ipv4" }, - "start_time": "2017-05-03T19:42:24.151Z", - "transport": "tcp", - "type": "flow" + "source": { + "bytes": 10, + "ip": "203.0.113.3", + "mac": "01:02:03:04:05:06", + "packets": 1, + "port": 38901 + } } -------------------------------------------------------------------------------- -<1> Packetbeat sets the `final` flag to `false` to indicate that the event +<1> Packetbeat sets the `flow.final` flag to `false` to indicate that the event contains an intermediate report about a flow that it's tracking. When the flow -completes, Packetbeat sends one last event with `final` set to `true`. If you -want to aggregate sums of traffic, you need to filter on `final:true`, or use -some other technique, so that you get only the latest update from each flow. +completes, Packetbeat sends one last event with `flow.final` set to `true`. If +you want to aggregate sums of traffic, you need to filter on `final:true`, or +use some other technique, so that you get only the latest update from each flow. You can disable intermediate reports by setting `period: -1s`. [float] diff --git a/packetbeat/flows/flows_test.go b/packetbeat/flows/flows_test.go index b76e8c59344..06c903bd2a4 100644 --- a/packetbeat/flows/flows_test.go +++ b/packetbeat/flows/flows_test.go @@ -131,7 +131,8 @@ func TestFlowsCounting(t *testing.T) { t.Logf("event: %v", event) source := event["source"].(common.MapStr) - dest := event["dest"].(common.MapStr) + dest := event["destination"].(common.MapStr) + network := event["network"].(common.MapStr) // validate generated event assert.Equal(t, net.HardwareAddr(mac1).String(), source["mac"]) @@ -140,9 +141,9 @@ func TestFlowsCounting(t *testing.T) { assert.Equal(t, net.IP(ip2).String(), dest["ip"]) assert.Equal(t, uint16(256), source["port"]) assert.Equal(t, uint16(512), dest["port"]) - assert.Equal(t, "tcp", event["transport"]) + assert.Equal(t, "tcp", network["transport"]) - stat := source["stats"].(map[string]interface{}) + stat := source assert.Equal(t, int64(-1), stat["int1"]) assert.Equal(t, nil, stat["int2"]) assert.Equal(t, uint64(1), stat["uint1"]) @@ -150,7 +151,7 @@ func TestFlowsCounting(t *testing.T) { assert.Equal(t, 3.14, stat["float1"]) assert.Equal(t, nil, stat["float2"]) - stat = dest["stats"].(map[string]interface{}) + stat = dest assert.Equal(t, nil, stat["int1"]) assert.Equal(t, int64(-1), stat["int2"]) assert.Equal(t, nil, stat["uint1"]) diff --git a/packetbeat/flows/worker.go b/packetbeat/flows/worker.go index 8ce9bdbc901..2cc97c24d08 100644 --- a/packetbeat/flows/worker.go +++ b/packetbeat/flows/worker.go @@ -18,7 +18,6 @@ package flows import ( - "encoding/base64" "encoding/binary" "errors" "net" @@ -206,14 +205,22 @@ func createEvent( intNames, uintNames, floatNames []string, ) beat.Event { timestamp := ts + + event := common.MapStr{ + "start": common.Time(f.createTS), + "end": common.Time(f.ts), + "duration": f.ts.Sub(f.createTS), + "type": "flow", + } + flow := common.MapStr{ + "id": common.NetString(f.id.Serialize()), + "final": isOver, + } fields := common.MapStr{ - "start_time": common.Time(f.createTS), - "last_time": common.Time(f.ts), - "type": "flow", - "flow_id": common.NetString(f.id.Serialize()), - "final": isOver, + "event": event, + "flow": flow, } - + network := common.MapStr{} source := common.MapStr{} dest := common.MapStr{} tuple := common.IPPortTuple{} @@ -227,59 +234,65 @@ func createEvent( // add vlan if vlan := f.id.OutterVLan(); vlan != nil { - fields["outer_vlan"] = binary.LittleEndian.Uint16(vlan) + vlanID := uint64(binary.LittleEndian.Uint16(vlan)) + putOrAppendUint64(flow, "vlan", vlanID) } if vlan := f.id.VLan(); vlan != nil { - fields["vlan"] = binary.LittleEndian.Uint16(vlan) + vlanID := uint64(binary.LittleEndian.Uint16(vlan)) + putOrAppendUint64(flow, "vlan", vlanID) } // add icmp if icmp := f.id.ICMPv4(); icmp != nil { - fields["icmp_id"] = binary.LittleEndian.Uint16(icmp) + network["transport"] = "icmp" } else if icmp := f.id.ICMPv6(); icmp != nil { - fields["icmp_id"] = binary.LittleEndian.Uint16(icmp) + network["transport"] = "ipv6-icmp" } // ipv4 layer meta data if src, dst, ok := f.id.OutterIPv4Addr(); ok { srcIP, dstIP := net.IP(src), net.IP(dst) - source["outer_ip"] = srcIP.String() - dest["outer_ip"] = dstIP.String() + source["ip"] = srcIP.String() + dest["ip"] = dstIP.String() tuple.SrcIP = srcIP tuple.DstIP = dstIP tuple.IPLength = 4 + network["type"] = "ipv4" } if src, dst, ok := f.id.IPv4Addr(); ok { srcIP, dstIP := net.IP(src), net.IP(dst) - source["ip"] = srcIP.String() - dest["ip"] = dstIP.String() + putOrAppendString(source, "ip", srcIP.String()) + putOrAppendString(dest, "ip", dstIP.String()) // Save IPs for process matching if an outer layer was not present if tuple.IPLength == 0 { tuple.SrcIP = srcIP tuple.DstIP = dstIP tuple.IPLength = 4 } + network["type"] = "ipv4" } // ipv6 layer meta data if src, dst, ok := f.id.OutterIPv6Addr(); ok { srcIP, dstIP := net.IP(src), net.IP(dst) - source["outer_ipv6"] = srcIP.String() - dest["outer_ipv6"] = dstIP.String() + putOrAppendString(source, "ip", srcIP.String()) + putOrAppendString(dest, "ip", dstIP.String()) tuple.SrcIP = srcIP tuple.DstIP = dstIP tuple.IPLength = 6 + network["type"] = "ipv6" } if src, dst, ok := f.id.IPv6Addr(); ok { srcIP, dstIP := net.IP(src), net.IP(dst) - source["ipv6"] = net.IP(src).String() - dest["ipv6"] = net.IP(dst).String() + putOrAppendString(source, "ip", srcIP.String()) + putOrAppendString(dest, "ip", dstIP.String()) // Save IPs for process matching if an outer layer was not present if tuple.IPLength == 0 { tuple.SrcIP = srcIP tuple.DstIP = dstIP tuple.IPLength = 6 } + network["type"] = "ipv6" } // udp layer meta data @@ -287,7 +300,7 @@ func createEvent( tuple.SrcPort = binary.LittleEndian.Uint16(src) tuple.DstPort = binary.LittleEndian.Uint16(dst) source["port"], dest["port"] = tuple.SrcPort, tuple.DstPort - fields["transport"] = "udp" + network["transport"] = "udp" proto = applayer.TransportUDP } @@ -296,23 +309,45 @@ func createEvent( tuple.SrcPort = binary.LittleEndian.Uint16(src) tuple.DstPort = binary.LittleEndian.Uint16(dst) source["port"], dest["port"] = tuple.SrcPort, tuple.DstPort - fields["transport"] = "tcp" + network["transport"] = "tcp" proto = applayer.TransportTCP } - if id := f.id.ConnectionID(); id != nil { - fields["connection_id"] = base64.StdEncoding.EncodeToString(id) - } - + var totalBytes, totalPackets uint64 if f.stats[0] != nil { - source["stats"] = encodeStats(f.stats[0], intNames, uintNames, floatNames) + // Source stats. + stats := encodeStats(f.stats[0], intNames, uintNames, floatNames) + for k, v := range stats { + source[k] = v + } + + if v, found := stats["bytes"]; found { + totalBytes += v.(uint64) + } + if v, found := stats["packets"]; found { + totalPackets += v.(uint64) + } } if f.stats[1] != nil { - dest["stats"] = encodeStats(f.stats[1], intNames, uintNames, floatNames) + // Destination stats. + stats := encodeStats(f.stats[1], intNames, uintNames, floatNames) + for k, v := range stats { + dest[k] = v + } + + if v, found := stats["bytes"]; found { + totalBytes += v.(uint64) + } + if v, found := stats["packets"]; found { + totalPackets += v.(uint64) + } } + network["bytes"] = totalBytes + network["packets"] = totalPackets + fields["network"] = network fields["source"] = source - fields["dest"] = dest + fields["destination"] = dest // Set process information if it's available if tuple.IPLength != 0 && tuple.SrcPort != 0 { @@ -376,3 +411,43 @@ func encodeStats( return report } + +func putOrAppendString(m common.MapStr, key, value string) { + old, found := m[key] + if !found { + m[key] = value + return + } + + if old != nil { + switch v := old.(type) { + case string: + m[key] = []string{v, value} + case []string: + m[key] = append(v, value) + } + } +} + +func putOrAppendUint64(m common.MapStr, key string, value uint64) { + old, found := m[key] + if !found { + m[key] = value + return + } + + if old != nil { + switch v := old.(type) { + case uint8: + m[key] = []uint64{uint64(v), value} + case uint16: + m[key] = []uint64{uint64(v), value} + case uint32: + m[key] = []uint64{uint64(v), value} + case uint64: + m[key] = []uint64{uint64(v), value} + case []uint64: + m[key] = append(v, value) + } + } +} diff --git a/packetbeat/flows/worker_test.go b/packetbeat/flows/worker_test.go new file mode 100644 index 00000000000..912c170c37c --- /dev/null +++ b/packetbeat/flows/worker_test.go @@ -0,0 +1,123 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package flows + +import ( + "encoding/json" + "flag" + "io/ioutil" + "testing" + "time" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/mapval" + "github.com/elastic/beats/libbeat/logp" +) + +var ( + // Use `go test -data` to update sample event files. + dataFlag = flag.Bool("data", false, "Write updated data.json files") +) + +func TestCreateEvent(t *testing.T) { + logp.TestingSetup() + + // Build biflow event. + start := time.Unix(1542292881, 0) + end := start.Add(3 * time.Second) + vlan := uint16(171) + mac1 := []byte{1, 2, 3, 4, 5, 6} + mac2 := []byte{6, 5, 4, 3, 2, 1} + ip1 := []byte{203, 0, 113, 3} + ip2 := []byte{198, 51, 100, 2} + port1 := uint16(38901) + port2 := uint16(80) + + id := newFlowID() + id.AddEth(mac1, mac2) + id.AddVLan(vlan) + id.AddIPv4(ip1, ip2) + id.AddTCP(port1, port2) + + bif := &biFlow{ + id: id.rawFlowID, + killed: 1, + createTS: start, + ts: end, + dir: flowDirForward, + } + bif.stats[0] = &flowStats{uintFlags: []uint8{1, 1}, uints: []uint64{10, 1}} + bif.stats[1] = &flowStats{uintFlags: []uint8{1, 1}, uints: []uint64{460, 2}} + event := createEvent(time.Now(), bif, true, nil, []string{"bytes", "packets"}, nil) + + // Validate the contents of the event. + validate := mapval.MustCompile(mapval.Map{ + "source": mapval.Map{ + "mac": "01:02:03:04:05:06", + "ip": "203.0.113.3", + "port": port1, + "bytes": uint64(10), + "packets": uint64(1), + }, + "destination": mapval.Map{ + "mac": "06:05:04:03:02:01", + "ip": "198.51.100.2", + "port": port2, + "bytes": uint64(460), + "packets": uint64(2), + }, + "flow": mapval.Map{ + "id": mapval.KeyPresent, + "final": true, + "vlan": mapval.KeyPresent, + }, + "network": mapval.Map{ + "bytes": uint64(470), + "packets": uint64(3), + "type": "ipv4", + "transport": "tcp", + }, + "event": mapval.Map{ + "start": mapval.KeyPresent, + "end": mapval.KeyPresent, + "duration": mapval.KeyPresent, + "type": "flow", + }, + }) + + result := validate(event.Fields) + if errs := result.Errors(); len(errs) > 0 { + for _, err := range errs { + t.Error(err) + } + t.FailNow() + } + + // Write the event to disk if -data is used. + if *dataFlag { + event.Fields.Put("@timestamp", common.Time(end)) + output, err := json.MarshalIndent(&event.Fields, "", " ") + if err != nil { + t.Fatal(err) + } + + if err := ioutil.WriteFile("../_meta/sample_outputs/flow.json", output, 0644); err != nil { + t.Fatal(err) + } + } +} diff --git a/packetbeat/tests/system/packetbeat.py b/packetbeat/tests/system/packetbeat.py index c614d9cabf5..d1cc555c2b8 100644 --- a/packetbeat/tests/system/packetbeat.py +++ b/packetbeat/tests/system/packetbeat.py @@ -11,7 +11,8 @@ TRANS_REQUIRED_FIELDS = ["@timestamp", "type", "status", "agent.type", "agent.hostname", "agent.version"] -FLOWS_REQUIRED_FIELDS = ["@timestamp", "type", +FLOWS_REQUIRED_FIELDS = ["@timestamp", "event.type", "event.start", + "event.end", "event.duration", "flow.id", "agent.type", "agent.hostname", "agent.version"] @@ -113,7 +114,9 @@ def read_output(self, with open(os.path.join(self.working_dir, output_file), "r") as f: for line in f: document = self.flatten_object(json.loads(line), self.dict_fields) - if not types or document["type"] in types: + if not types or \ + ("type" in document and document["type"] in types) or \ + ("event.type" in document and document["event.type"] in types): jsons.append(document) self.all_have_fields(jsons, required_fields or TRANS_REQUIRED_FIELDS) self.all_fields_are_expected(jsons, self.expected_fields) diff --git a/packetbeat/tests/system/pcaps/802.1q-q-in-q-icmp.pcap b/packetbeat/tests/system/pcaps/802.1q-q-in-q-icmp.pcap new file mode 100644 index 00000000000..98f14f645a7 Binary files /dev/null and b/packetbeat/tests/system/pcaps/802.1q-q-in-q-icmp.pcap differ diff --git a/packetbeat/tests/system/test_0060_flows.py b/packetbeat/tests/system/test_0060_flows.py index e6d23780f53..2fae71c4754 100644 --- a/packetbeat/tests/system/test_0060_flows.py +++ b/packetbeat/tests/system/test_0060_flows.py @@ -34,22 +34,22 @@ def test_mysql_flow(self): pprint(objs) assert len(objs) == 1 check_fields(objs[0], { - 'final': True, + 'flow.final': True, 'source.mac': '0a:00:27:00:00:00', - 'dest.mac': '08:00:27:76:d7:41', - 'dest.ip': '192.168.33.14', + 'destination.mac': '08:00:27:76:d7:41', + 'destination.ip': '192.168.33.14', 'source.ip': '192.168.33.1', - 'transport': 'tcp', + 'network.transport': 'tcp', 'source.port': 60137, - 'dest.port': 3306, - 'source.stats.net_packets_total': 22, - 'source.stats.net_bytes_total': 1480, - 'dest.stats.net_packets_total': 10, - 'dest.stats.net_bytes_total': 181133, + 'destination.port': 3306, + 'source.packets': 22, + 'source.bytes': 1480, + 'destination.packets': 10, + 'destination.bytes': 181133, }) - start_ts = parse_timestamp(objs[0]['start_time']) - last_ts = parse_timestamp(objs[0]['last_time']) + start_ts = parse_timestamp(objs[0]['event.start']) + last_ts = parse_timestamp(objs[0]['event.end']) assert last_ts > start_ts def test_memcache_udp_flow(self): @@ -68,16 +68,16 @@ def test_memcache_udp_flow(self): pprint(objs) assert len(objs) == 1 check_fields(objs[0], { - 'final': True, + 'flow.final': True, 'source.mac': 'ac:bc:32:77:41:0b', - 'dest.mac': '08:00:27:dd:3b:28', + 'destination.mac': '08:00:27:dd:3b:28', 'source.ip': '192.168.188.37', - 'dest.ip': '192.168.188.38', - 'transport': 'udp', + 'destination.ip': '192.168.188.38', + 'network.transport': 'udp', 'source.port': 63888, - 'dest.port': 11211, - 'source.stats.net_packets_total': 3, - 'source.stats.net_bytes_total': 280, + 'destination.port': 11211, + 'source.packets': 3, + 'source.bytes': 280, }) def test_icmp4_ping(self): @@ -96,17 +96,16 @@ def test_icmp4_ping(self): pprint(objs) assert len(objs) == 1 check_fields(objs[0], { - 'final': True, + 'flow.final': True, 'source.mac': '00:00:00:00:00:01', - 'dest.mac': '00:00:00:00:00:02', - 'vlan': 10, + 'destination.mac': '00:00:00:00:00:02', + 'flow.vlan': 10, 'source.ip': '10.0.0.1', - 'dest.ip': '10.0.0.2', - 'icmp_id': 5, - 'source.stats.net_bytes_total': 50, - 'source.stats.net_packets_total': 1, - 'dest.stats.net_bytes_total': 50, - 'dest.stats.net_packets_total': 1, + 'destination.ip': '10.0.0.2', + 'source.bytes': 50, + 'source.packets': 1, + 'destination.bytes': 50, + 'destination.packets': 1, }) def test_icmp6_ping(self): @@ -125,15 +124,44 @@ def test_icmp6_ping(self): pprint(objs) assert len(objs) == 1 check_fields(objs[0], { - 'final': True, + 'flow.final': True, + 'flow.vlan': 10, 'source.mac': '00:00:00:00:00:01', - 'dest.mac': '00:00:00:00:00:02', - 'vlan': 10, - 'source.ipv6': '::1', - 'dest.ipv6': '::2', - 'icmp_id': 5, - 'source.stats.net_bytes_total': 70, - 'source.stats.net_packets_total': 1, - 'dest.stats.net_bytes_total': 70, - 'dest.stats.net_packets_total': 1, + 'source.ip': '::1', + 'source.bytes': 70, + 'source.packets': 1, + 'destination.mac': '00:00:00:00:00:02', + 'destination.ip': '::2', + 'destination.bytes': 70, + 'destination.packets': 1, + 'network.bytes': 140, + 'network.packets': 2, + }) + + def test_q_in_q_flow(self): + self.render_config_template( + flows=True, + shutdown_timeout="1s", + ) + self.run_packetbeat( + pcap="802.1q-q-in-q-icmp.pcap", + debug_selectors=["*"]) + + objs = self.read_output( + types=["flow"], + required_fields=FLOWS_REQUIRED_FIELDS) + + pprint(objs) + assert len(objs) == 1 + check_fields(objs[0], { + 'flow.final': True, + 'flow.vlan': [101, 600], + 'source.ip': '192.168.1.1', + 'source.bytes': 82, + 'source.packets': 1, + 'source.mac': '08:00:27:3d:25:4e', + 'destination.mac': '1c:af:f7:70:ed:7c', + 'destination.ip': '192.168.1.2', + 'network.bytes': 82, + 'network.packets': 1, })