From d9e23f9b81d6055f5cd8185ba02d551e28a973a4 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 10 Jan 2019 00:58:28 -0500 Subject: [PATCH 1/3] Update HTTP protocol to use ECS fields Here's a summary of what fields changed. Part of #7968 Changed - bytes_in -> source.bytes - bytes_out -> destination.bytes - http.request.body -> http.request.body.content - http.response.body -> http.response.body.content - http.response.code -> http.response.status_code - http.response.phrase -> http.response.status_phrase - method -> http.request.method (lowercased) - notes -> error.message - params -> url.query - path -> url.path - real_ip -> network.forwarded_ip - responsetime -> event.duration (unit are now nanoseconds) - transport -> network.transport Added - event.dataset = http - event.end - event.start - http.request.referrer (always added if Referer header is present) - http.version - network.bytes - network.community_id - network.protocol = http - network.type - source.domain (added if Host header is present and not an IP address) - url.domain - set with the Host header value - url.full (synthesized from data in the request/response) - url.port (when port is != 80) - user_agent.original - (always added if User-Agent header is present) Unchanged Packetbeat Fields - query = {{ http.request.method }} {{ url.path }} - request - text representation of the entire request - response - text representation of the entire response - status - type = http (we might remove this since we have event.dataset) The HTTP dashboard was updated too. --- CHANGELOG.next.asciidoc | 1 + dev-tools/ecs-migration.yml | 35 + .../kibana/6/dashboard/Packetbeat-http.json | 1483 ++++++----- .../6/dashboard/Packetbeat-overview.json | 2318 +++++++++-------- packetbeat/docs/fields.asciidoc | 30 +- packetbeat/protos/http/_meta/fields.yml | 15 +- packetbeat/protos/http/event.go | 107 + packetbeat/protos/http/event_test.go | 58 + packetbeat/protos/http/fields.go | 2 +- packetbeat/protos/http/http.go | 139 +- packetbeat/protos/http/http_parser.go | 23 +- packetbeat/protos/http/http_test.go | 18 +- .../tests/system/test_0003_http_simple.py | 36 +- packetbeat/tests/system/test_0006_wsgi.py | 12 +- packetbeat/tests/system/test_0008_realip.py | 4 +- .../test_0010_http_10_connection_close.py | 2 +- .../tests/system/test_0019_hide_params.py | 12 +- .../tests/system/test_0023_http_params.py | 8 +- .../tests/system/test_0024_http_query.py | 3 +- packetbeat/tests/system/test_0029_http_gap.py | 5 +- .../tests/system/test_0060_processors.py | 32 +- .../tests/system/test_0063_http_body.py | 2 +- .../tests/system/test_0065_unmatched_http.py | 8 +- 23 files changed, 2441 insertions(+), 1912 deletions(-) create mode 100644 packetbeat/protos/http/event.go create mode 100644 packetbeat/protos/http/event_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 953592f4a49..fc79bc2b8ff 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -59,6 +59,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Removed trailing dot from domain names reported by the DNS protocol. {pull}9941[9941] - Changed TLS protocol fields to align with ECS. {pull}9980[9980] - Changed ICMP protocol fields to align with ECS. {pull}10062[10062] +- Changed HTTP protocol fields to align with ECS. {pull}9976[9976] *Winlogbeat* diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 9af95e8368b..45dccc24665 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -805,6 +805,7 @@ alias: false comment: No alias mapping as field did not always exist + # Packetbeat ## Shared @@ -868,6 +869,40 @@ to: destination.packets alias: true +## HTTP +- from: http.request.body + to: http.request.body.content + alias: true + +- from: http.response.body + to: http.response.body.content + alias: true + +- from: http.response.code + to: http.response.status_code + alias: true + +- from: http.response.phrase + to: http.response.status_phrase + alias: true + +- from: http.request.params + to: url.query + alias: true + +- from: method + to: http.request.method + alias: false + comment: method is used by other protocols. + +- from: path + to: url.path + alias: false + +- from: real_ip + to: network.forwarded_ip + alias: false + # Heartbeat diff --git a/packetbeat/_meta/kibana/6/dashboard/Packetbeat-http.json b/packetbeat/_meta/kibana/6/dashboard/Packetbeat-http.json index 7b3eea0a11e..5817f81ae2d 100644 --- a/packetbeat/_meta/kibana/6/dashboard/Packetbeat-http.json +++ b/packetbeat/_meta/kibana/6/dashboard/Packetbeat-http.json @@ -1,606 +1,899 @@ { - "objects": [ - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] + "objects": [ + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchId": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", + "title": "Web Transactions [Packetbeat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "id": "2", + "params": { + "customInterval": "2h", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "mode": "stacked", + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "times": [], + "yAxis": {} + }, + "title": "Web transactions", + "type": "histogram" + } + }, + "id": "Web-transactions", + "type": "visualization", + "updated_at": "2019-01-09T19:09:57.656Z", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "packetbeat-*", + "key": "type", + "negate": false, + "params": { + "query": "http", + "type": "phrase" + }, + "type": "phrase", + "value": "http" + }, + "query": { + "match": { + "network.protocol": { + "query": "http", + "type": "phrase" } - }, - "savedSearchId": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", - "title": "Web transactions", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - } - ], - "listeners": {}, - "params": { - "addLegend": false, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "mode": "stacked", - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "times": [], - "yAxis": {} - }, - "title": "Web transactions", - "type": "histogram" + } } - }, - "id": "Web-transactions", - "type": "visualization", - "version": 3 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "packetbeat-*", - "key": "type", - "negate": false, - "value": "http" - }, - "query": { - "match": { - "type": { - "query": "http", - "type": "phrase" - } - } - } - } - ], - "highlight": { - "fields": { - "*": {} - }, - "post_tags": [ - "@/kibana-highlighted-field@" - ], - "pre_tags": [ - "@kibana-highlighted-field@" - ] - }, - "index": "packetbeat-*", - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "http.response.code: [300 TO *]" - } - } - } - }, - "title": "HTTP error codes", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": { - "field": "type" - }, - "schema": "metric", - "type": "cardinality" - }, - { - "id": "2", - "params": { - "field": "http.response.code", - "order": "desc", - "orderBy": "1", - "size": 5 - }, - "schema": "segment", - "type": "terms" - } - ], - "listeners": {}, - "params": { - "addLegend": false, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "mode": "stacked", - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "times": [], - "yAxis": {} - }, - "title": "HTTP error codes", - "type": "histogram" + } + ], + "highlight": { + "fields": { + "*": {} + }, + "post_tags": [ + "@/kibana-highlighted-field@" + ], + "pre_tags": [ + "@kibana-highlighted-field@" + ] + }, + "index": "packetbeat-*", + "query": { + "language": "lucene", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "http.response.status_code: [300 TO *]" } - }, - "id": "HTTP-error-codes", - "type": "visualization", - "version": 1 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "packetbeat-*", - "key": "type", - "negate": false, - "value": "http" - }, - "query": { - "match": { - "type": { - "query": "http", - "type": "phrase" - } - } - } - } - ], - "highlight": { - "fields": { - "*": {} - }, - "post_tags": [ - "@/kibana-highlighted-field@" - ], - "pre_tags": [ - "@kibana-highlighted-field@" - ] - }, - "index": "packetbeat-*", - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "!http.response.code: [200 TO 299]" - } - } + } + } + } + }, + "title": "HTTP error codes [Packetbeat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "field": "type" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "HTTP Status Code", + "field": "http.response.status_code", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": false, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "legendPosition": "right", + "mode": "stacked", + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Unique count of type" + }, + "mode": "stacked", + "show": "true", + "type": "histogram", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "shareYAxis": true, + "times": [], + "type": "histogram", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "defaultYExtents": false, + "mode": "normal", + "setYExtents": false, + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ], + "yAxis": {} + }, + "title": "HTTP error codes [Packetbeat]", + "type": "histogram" + } + }, + "id": "HTTP-error-codes", + "type": "visualization", + "updated_at": "2019-01-09T19:09:53.510Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "packetbeat-*", + "key": "network.protocol", + "negate": false, + "params": { + "query": "http", + "type": "phrase" + }, + "type": "phrase", + "value": "http" + }, + "query": { + "match": { + "network.protocol": { + "query": "http", + "type": "phrase" } - }, - "title": "HTTP error codes evolution", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": { - "field": "type" - }, - "schema": "metric", - "type": "cardinality" - }, - { - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - }, - { - "id": "3", - "params": { - "field": "http.response.code", - "order": "desc", - "orderBy": "1", - "size": 5 - }, - "schema": "group", - "type": "terms" - } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "drawLinesBetweenPoints": true, - "interpolate": "linear", - "radiusRatio": 9, - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "showCircles": true, - "smoothLines": false, - "times": [], - "yAxis": {} - }, - "title": "HTTP error codes evolution", - "type": "line" + } } - }, - "id": "HTTP-error-codes-evolution", - "type": "visualization", - "version": 1 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "lucene", - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "*" - } - } - } - } - }, - "title": "Navigation", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [], - "params": { - "fontSize": "10", - "markdown": "### Packetbeat:\n\n[Overview](#/dashboard/Packetbeat-Dashboard)\n\n[Flows](#/dashboard/Packetbeat-Flows)\n\n[Web transactions](#/dashboard/Packetbeat-HTTP)\n\n[MySQL performance](#/dashboard/Packetbeat-MySQL-performance)\n\n[PostgreSQL performance](#/dashboard/Packetbeat-PgSQL-performance)\n\n[MongoDB performance](#/dashboard/Packetbeat-MongoDB-performance)\n\n[Thrift-RPC performance](#/dashboard/Packetbeat-Thrift-performance)\n\n[NFS transactions](#/dashboard/Packetbeat-NFS)\n\n[Cassandra performance](#/dashboard/Packetbeat-Cassandra)" - }, - "title": "Navigation", - "type": "markdown" + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "packetbeat-*", + "key": "http.response.status_code", + "negate": true, + "params": { + "gte": 200, + "lt": 299 + }, + "type": "range", + "value": "200 to 299" + }, + "range": { + "http.response.status_code": { + "gte": 200, + "lte": 299 + } } - }, - "id": "Navigation", - "type": "visualization", - "version": 10 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", - "title": "Total number of HTTP transactions", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - } - ], - "listeners": {}, - "params": { - "fontSize": "37", - "handleNoResults": true - }, - "title": "Total number of HTTP transactions", - "type": "metric" + } + ], + "highlight": { + "fields": { + "*": {} + }, + "post_tags": [ + "@/kibana-highlighted-field@" + ], + "pre_tags": [ + "@kibana-highlighted-field@" + ] + }, + "index": "packetbeat-*", + "query": { + "language": "lucene", + "query": "" + } + } + }, + "title": "HTTP error codes evolution [Packetbeat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customInterval": "2h", + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "time_zone": "America/New_York", + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "HTTP Status Code", + "field": "http.response.status_code", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "mode": "normal", + "radiusRatio": 9, + "show": "true", + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "shareYAxis": true, + "showCircles": true, + "smoothLines": false, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "defaultYExtents": false, + "mode": "normal", + "setYExtents": false, + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ], + "yAxis": {} + }, + "title": "HTTP error codes evolution [Packetbeat]", + "type": "line" + } + }, + "id": "HTTP-error-codes-evolution", + "type": "visualization", + "updated_at": "2019-01-09T19:09:53.510Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "lucene", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "*" } - }, - "id": "Total-number-of-HTTP-transactions", - "type": "visualization", - "version": 2 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", - "title": "HTTP codes for the top queries", - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "id": "2", - "params": { - "field": "query", - "order": "desc", - "orderBy": "1", - "row": false, - "size": 5 - }, - "schema": "split", - "type": "terms" - }, - { - "id": "3", - "params": { - "field": "http.response.code", - "order": "desc", - "orderBy": "1", - "size": 10 - }, - "schema": "segment", - "type": "terms" - } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTooltip": true, - "isDonut": false, - "shareYAxis": true - }, - "type": "pie" + } + } + } + }, + "title": "Navigation", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "fontSize": "10", + "markdown": "### Packetbeat:\n\n[Overview](#/dashboard/Packetbeat-Dashboard)\n\n[DNS Overview](#/dashboard/65120940-1454-11e9-9de0-f98d1808db8e) | [Tunneling](#/dashboard/DNS-Unique-Domains)\n\n[DHCPv4 Overview](#/dashboard/a7b35890-8baa-11e8-9676-ef67484126fb)\n\n[Flows](#/dashboard/Packetbeat-Flows)\n\n[Web transactions](#/dashboard/Packetbeat-HTTP)\n\n[MySQL performance](#/dashboard/Packetbeat-MySQL-performance)\n\n[PostgreSQL performance](#/dashboard/Packetbeat-PgSQL-performance)\n\n[MongoDB performance](#/dashboard/Packetbeat-MongoDB-performance)\n\n[Thrift-RPC performance](#/dashboard/Packetbeat-Thrift-performance)\n\n[NFS transactions](#/dashboard/Packetbeat-NFS)\n\n[Cassandra performance](#/dashboard/Packetbeat-Cassandra)", + "openLinksInNewTab": false + }, + "title": "Navigation", + "type": "markdown" + } + }, + "id": "Navigation", + "type": "visualization", + "updated_at": "2019-01-09T19:10:00.747Z", + "version": 10 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "lucene", + "query": "" + } + } + }, + "savedSearchId": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", + "title": "Total number of HTTP transactions [Packetbeat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + } + ], + "params": { + "addLegend": false, + "addTooltip": true, + "fontSize": "37", + "handleNoResults": true, + "metric": { + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 10000 } - }, - "id": "HTTP-codes-for-the-top-queries", - "type": "visualization", - "version": 2 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] + ], + "invertColors": false, + "labels": { + "show": true + }, + "metricColorMode": "None", + "percentageMode": false, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": 60, + "labelColor": false, + "subText": "" + }, + "useRanges": false + }, + "type": "metric" + }, + "title": "Total number of HTTP transactions [Packetbeat]", + "type": "metric" + } + }, + "id": "Total-number-of-HTTP-transactions", + "type": "visualization", + "updated_at": "2019-01-09T19:09:53.510Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "lucene", + "query": "" + } + } + }, + "savedSearchId": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", + "title": "HTTP status codes for the top queries [Packetbeat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "HTTP Query", + "field": "query", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "row": false, + "size": 5 + }, + "schema": "split", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "HTTP Status Code", + "field": "http.response.status_code", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": false, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "shareYAxis": true, + "type": "pie" + }, + "title": "HTTP status codes for the top queries [Packetbeat]", + "type": "pie" + } + }, + "id": "HTTP-codes-for-the-top-queries", + "type": "visualization", + "updated_at": "2019-01-09T19:09:53.510Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "lucene", + "query": "" + } + } + }, + "savedSearchId": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", + "title": "Top 10 HTTP requests [Packetbeat]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "url.full", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "bucket", + "type": "terms" + } + ], + "params": { + "perPage": 10, + "showMetricsAtAllLevels": false, + "showPartialRows": false, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 10 HTTP requests [Packetbeat]", + "type": "table" + } + }, + "id": "Top-10-HTTP-requests", + "type": "visualization", + "updated_at": "2019-01-09T19:09:53.510Z", + "version": 1 + }, + { + "attributes": { + "columns": [ + "_source" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "packetbeat-*", + "key": "network.protocol", + "negate": false, + "params": { + "query": "http", + "type": "phrase" + }, + "type": "phrase", + "value": "http" + }, + "query": { + "match": { + "network.protocol": { + "query": "http", + "type": "phrase" } - }, - "savedSearchId": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", - "title": "Top 10 HTTP requests", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "id": "2", - "params": { - "field": "query", - "order": "desc", - "orderBy": "1", - "size": 10 - }, - "schema": "bucket", - "type": "terms" - } - ], - "listeners": {}, - "params": { - "perPage": 10, - "showMeticsAtAllLevels": false, - "showPartialRows": false - }, - "title": "Top 10 HTTP requests", - "type": "table" + } } - }, - "id": "Top-10-HTTP-requests", - "type": "visualization", - "version": 2 - }, - { - "attributes": { - "columns": [ - "_source" - ], - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "packetbeat-*", - "key": "type", - "negate": false, - "params": { - "query": "http", - "type": "phrase" - }, - "type": "phrase", - "value": "http" - }, - "query": { - "match": { - "type": { - "query": "http", - "type": "phrase" - } - } - } - } - ], - "highlightAll": true, - "index": "packetbeat-*", - "query": { - "language": "lucene", - "query": "*" - }, - "version": true - } - }, - "sort": [ - "@timestamp", - "desc" - ], - "title": "Web Transactions [Packetbeat]", - "version": 1 - }, - "id": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", - "type": "search", - "version": 2 - }, - { - "attributes": { - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "highlightAll": true, - "query": { - "language": "lucene", - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "*" - } - } - }, - "version": true - } - }, - "optionsJSON": { - "darkTheme": false - }, - "panelsJSON": [ - { - "col": 4, - "id": "Web-transactions", - "panelIndex": 1, - "row": 1, - "size_x": 9, - "size_y": 4, - "type": "visualization" - }, - { - "col": 1, - "id": "HTTP-error-codes", - "panelIndex": 2, - "row": 8, - "size_x": 6, - "size_y": 3, - "type": "visualization" - }, - { - "col": 7, - "id": "HTTP-error-codes-evolution", - "panelIndex": 3, - "row": 8, - "size_x": 6, - "size_y": 3, - "type": "visualization" - }, - { - "col": 1, - "id": "Navigation", - "panelIndex": 4, - "row": 1, - "size_x": 3, - "size_y": 4, - "type": "visualization" - }, - { - "col": 1, - "id": "Total-number-of-HTTP-transactions", - "panelIndex": 5, - "row": 5, - "size_x": 3, - "size_y": 3, - "type": "visualization" - }, - { - "col": 4, - "id": "HTTP-codes-for-the-top-queries", - "panelIndex": 6, - "row": 5, - "size_x": 9, - "size_y": 3, - "type": "visualization" - }, - { - "col": 1, - "id": "Top-10-HTTP-requests", - "panelIndex": 7, - "row": 11, - "size_x": 12, - "size_y": 5, - "type": "visualization" - } - ], - "timeRestore": false, - "title": "[Packetbeat] HTTP", - "uiStateJSON": { - "P-5": { - "vis": { - "defaultColors": { - "0 - 100": "rgb(0,104,55)" - } - } - }, - "P-7": { - "vis": { - "params": { - "sort": { - "columnIndex": null, - "direction": null - } - } - } - } - }, - "version": 1 - }, - "id": "Packetbeat-HTTP", - "type": "dashboard", - "version": 2 - } - ], - "version": "6.0.0-beta1-SNAPSHOT" -} \ No newline at end of file + } + ], + "highlightAll": true, + "index": "packetbeat-*", + "query": { + "language": "lucene", + "query": "*" + }, + "version": true + } + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "Web Transactions [Packetbeat]", + "version": 1 + }, + "id": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", + "type": "search", + "updated_at": "2019-01-09T19:11:58.092Z", + "version": 3 + }, + { + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "lucene", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "*" + } + } + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "gridData": { + "h": 20, + "i": "1", + "w": 36, + "x": 12, + "y": 0 + }, + "id": "Web-transactions", + "panelIndex": "1", + "type": "visualization", + "version": "7.0.0-SNAPSHOT" + }, + { + "gridData": { + "h": 15, + "i": "2", + "w": 24, + "x": 0, + "y": 35 + }, + "id": "HTTP-error-codes", + "panelIndex": "2", + "type": "visualization", + "version": "7.0.0-SNAPSHOT" + }, + { + "gridData": { + "h": 15, + "i": "3", + "w": 24, + "x": 24, + "y": 35 + }, + "id": "HTTP-error-codes-evolution", + "panelIndex": "3", + "type": "visualization", + "version": "7.0.0-SNAPSHOT" + }, + { + "gridData": { + "h": 20, + "i": "4", + "w": 12, + "x": 0, + "y": 0 + }, + "id": "Navigation", + "panelIndex": "4", + "type": "visualization", + "version": "7.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "gridData": { + "h": 15, + "i": "5", + "w": 12, + "x": 0, + "y": 20 + }, + "id": "Total-number-of-HTTP-transactions", + "panelIndex": "5", + "type": "visualization", + "version": "7.0.0-SNAPSHOT" + }, + { + "gridData": { + "h": 15, + "i": "6", + "w": 36, + "x": 12, + "y": 20 + }, + "id": "HTTP-codes-for-the-top-queries", + "panelIndex": "6", + "type": "visualization", + "version": "7.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 25, + "i": "7", + "w": 48, + "x": 0, + "y": 50 + }, + "id": "Top-10-HTTP-requests", + "panelIndex": "7", + "type": "visualization", + "version": "7.0.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "[Packetbeat] HTTP", + "version": 1 + }, + "id": "Packetbeat-HTTP", + "type": "dashboard", + "updated_at": "2019-01-09T19:09:53.510Z", + "version": 1 + } + ], + "version": "7.0.0-SNAPSHOT" +} diff --git a/packetbeat/_meta/kibana/6/dashboard/Packetbeat-overview.json b/packetbeat/_meta/kibana/6/dashboard/Packetbeat-overview.json index 62d32910f85..7222716425e 100644 --- a/packetbeat/_meta/kibana/6/dashboard/Packetbeat-overview.json +++ b/packetbeat/_meta/kibana/6/dashboard/Packetbeat-overview.json @@ -1,1164 +1,1182 @@ { - "objects": [ - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", - "title": "Web transactions", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - } - ], - "listeners": {}, - "params": { - "addLegend": false, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "mode": "stacked", - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "times": [], - "yAxis": {} - }, - "title": "Web transactions", - "type": "histogram" - } - }, - "id": "Web-transactions", - "type": "visualization", - "version": 3 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "800e2a00-88cb-11e7-ad9c-db80de0bf8d3", - "title": "DB transactions", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - }, - { - "id": "3", - "params": { - "field": "type", - "order": "desc", - "orderBy": "1", - "size": 5 - }, - "schema": "group", - "type": "terms" - } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "mode": "stacked", - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "times": [], - "yAxis": {} - }, - "title": "DB transactions", - "type": "histogram" - } - }, - "id": "DB-transactions", - "type": "visualization", - "version": 2 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "3dd366e0-88cc-11e7-ad9c-db80de0bf8d3", - "title": "Cache transactions", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - } - ], - "listeners": {}, - "params": { - "addLegend": false, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "mode": "stacked", - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "times": [], - "yAxis": {} - }, - "title": "Cache transactions", - "type": "histogram" + "objects": [ + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchId": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", + "title": "Web transactions", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "id": "2", + "params": { + "customInterval": "2h", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "mode": "stacked", + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "times": [], + "yAxis": {} + }, + "title": "Web transactions", + "type": "histogram" + } + }, + "id": "Web-transactions", + "type": "visualization", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchId": "800e2a00-88cb-11e7-ad9c-db80de0bf8d3", + "title": "DB transactions", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "id": "2", + "params": { + "customInterval": "2h", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "id": "3", + "params": { + "field": "type", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "mode": "stacked", + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "times": [], + "yAxis": {} + }, + "title": "DB transactions", + "type": "histogram" + } + }, + "id": "DB-transactions", + "type": "visualization", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchId": "3dd366e0-88cc-11e7-ad9c-db80de0bf8d3", + "title": "Cache transactions", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "id": "2", + "params": { + "customInterval": "2h", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "mode": "stacked", + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "times": [], + "yAxis": {} + }, + "title": "Cache transactions", + "type": "histogram" + } + }, + "id": "Cache-transactions", + "type": "visualization", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchId": "d3089370-88cc-11e7-ad9c-db80de0bf8d3", + "title": "RPC transactions", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "id": "1", + "params": { + "field": "type" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "id": "2", + "params": { + "customInterval": "2h", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "mode": "stacked", + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "times": [], + "yAxis": {} + }, + "title": "RPC transactions", + "type": "histogram" + } + }, + "id": "RPC-transactions", + "type": "visualization", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchId": "Packetbeat-Search", + "title": "Response times percentiles", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "field": "responsetime", + "percents": [ + 75, + 95, + 99 + ] + }, + "schema": "metric", + "type": "percentiles" + }, + { + "enabled": true, + "id": "2", + "params": { + "customInterval": "2h", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "legendPosition": "right", + "radiusRatio": 9, + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "showCircles": true, + "smoothLines": true, + "times": [], + "yAxis": {} + }, + "title": "Response times percentiles", + "type": "line" + } + }, + "id": "Response-times-percentiles", + "type": "visualization", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchId": "Transactions-errors", + "title": "Errors count over time", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "id": "2", + "params": { + "customInterval": "30s", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "custom", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "id": "3", + "params": { + "field": "type", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "mode": "stacked", + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "times": [], + "yAxis": {} + }, + "title": "New Visualization", + "type": "histogram" + } + }, + "id": "Errors-count-over-time", + "type": "visualization", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchId": "Packetbeat-Search", + "title": "Errors vs successful transactions", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "id": "2", + "params": { + "customInterval": "2h", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "id": "3", + "params": { + "field": "status", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "mode": "percentage", + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "times": [], + "yAxis": {} + }, + "title": "Errors vs successful transactions", + "type": "histogram" + } + }, + "id": "Errors-vs-successful-transactions", + "type": "visualization", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchId": "Packetbeat-Search", + "title": "Latency histogram", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "extended_bounds": {}, + "field": "responsetime", + "interval": 10 + }, + "schema": "segment", + "type": "histogram" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "interpolate": "linear", + "legendPosition": "right", + "mode": "stacked", + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "smoothLines": true, + "times": [], + "yAxis": {} + }, + "title": "Latency histogram", + "type": "area" + } + }, + "id": "Latency-histogram", + "type": "visualization", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "index": "packetbeat-*", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "*" + } + } + } + }, + "title": "Client locations", + "uiStateJSON": { + "mapCenter": [ + 0, + -0.17578125 + ] + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "autoPrecision": true, + "field": "client.geo.location" + }, + "schema": "segment", + "type": "geohash_grid" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "heatBlur": 15, + "heatMaxZoom": 16, + "heatMinOpacity": 0.1, + "heatNormalizeData": true, + "heatRadius": 25, + "isDesaturated": true, + "mapCenter": [ + 15, + 5 + ], + "mapType": "Scaled Circle Markers", + "mapZoom": 2, + "wms": { + "enabled": false, + "options": { + "attribution": "Maps provided by USGS", + "format": "image/png", + "layers": "0", + "styles": "", + "transparent": true, + "version": "1.3.0" + }, + "url": "https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer" + } + }, + "title": "Client locations", + "type": "tile_map" + } + }, + "id": "Client-locations", + "type": "visualization", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchId": "Packetbeat-Search", + "title": "Response times repartition", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "id": "2", + "params": { + "customInterval": "2h", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "id": "3", + "params": { + "extended_bounds": {}, + "field": "responsetime", + "interval": 10 + }, + "schema": "group", + "type": "histogram" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "mode": "stacked", + "scale": "linear", + "setYExtents": false, + "shareYAxis": true, + "times": [], + "yAxis": {} + }, + "title": "Response times repartition", + "type": "histogram" + } + }, + "id": "Response-times-repartition", + "type": "visualization", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "lucene", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "*" } - }, - "id": "Cache-transactions", - "type": "visualization", - "version": 2 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] + } + } + } + }, + "title": "Navigation", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "fontSize": "10", + "markdown": "### Packetbeat:\n\n[Overview](#/dashboard/Packetbeat-Dashboard)\n\n[Flows](#/dashboard/Packetbeat-Flows)\n\n[Web transactions](#/dashboard/Packetbeat-HTTP)\n\n[MySQL performance](#/dashboard/Packetbeat-MySQL-performance)\n\n[PostgreSQL performance](#/dashboard/Packetbeat-PgSQL-performance)\n\n[MongoDB performance](#/dashboard/Packetbeat-MongoDB-performance)\n\n[Thrift-RPC performance](#/dashboard/Packetbeat-Thrift-performance)\n\n[NFS transactions](#/dashboard/Packetbeat-NFS)\n\n[Cassandra performance](#/dashboard/Packetbeat-Cassandra)" + }, + "title": "Navigation", + "type": "markdown" + } + }, + "id": "Navigation", + "type": "visualization", + "updated_at": "2019-01-09T20:47:55.346Z", + "version": 10 + }, + { + "attributes": { + "columns": [ + "_source" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "packetbeat-*", + "key": "network.protocol", + "negate": false, + "params": { + "query": "http", + "type": "phrase" + }, + "type": "phrase", + "value": "http" + }, + "query": { + "match": { + "network.protocol": { + "query": "http", + "type": "phrase" } - }, - "savedSearchId": "d3089370-88cc-11e7-ad9c-db80de0bf8d3", - "title": "RPC transactions", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": { - "field": "type" - }, - "schema": "metric", - "type": "cardinality" - }, - { - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - } - ], - "listeners": {}, - "params": { - "addLegend": false, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "mode": "stacked", - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "times": [], - "yAxis": {} - }, - "title": "RPC transactions", - "type": "histogram" + } } - }, - "id": "RPC-transactions", - "type": "visualization", - "version": 2 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "Packetbeat-Search", - "title": "Response times percentiles", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "field": "responsetime", - "percents": [ - 75, - 95, - 99 - ] - }, - "schema": "metric", - "type": "percentiles" - }, - { - "enabled": true, - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" + } + ], + "highlightAll": true, + "index": "packetbeat-*", + "query": { + "language": "lucene", + "query": "*" + }, + "version": true + } + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "Web Transactions [Packetbeat]", + "version": 1 + }, + "id": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", + "type": "search", + "updated_at": "2019-01-09T20:52:28.979Z", + "version": 3 + }, + { + "attributes": { + "columns": [ + "_source" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "packetbeat-*", + "key": "type", + "negate": false, + "type": "phrases", + "value": "mysql, postgresql" + }, + "query": { + "bool": { + "minimum_should_match": 1, + "should": [ + { + "match_phrase": { + "type": "mysql" } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "drawLinesBetweenPoints": true, - "interpolate": "linear", - "legendPosition": "right", - "radiusRatio": 9, - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "showCircles": true, - "smoothLines": true, - "times": [], - "yAxis": {} - }, - "title": "Response times percentiles", - "type": "line" - } - }, - "id": "Response-times-percentiles", - "type": "visualization", - "version": 1 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "Transactions-errors", - "title": "Errors count over time", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "id": "2", - "params": { - "customInterval": "30s", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "custom", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - }, - { - "id": "3", - "params": { - "field": "type", - "order": "desc", - "orderBy": "1", - "size": 5 - }, - "schema": "group", - "type": "terms" + }, + { + "match_phrase": { + "type": "postgresql" } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "mode": "stacked", - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "times": [], - "yAxis": {} - }, - "title": "New Visualization", - "type": "histogram" - } - }, - "id": "Errors-count-over-time", - "type": "visualization", - "version": 1 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "Packetbeat-Search", - "title": "Errors vs successful transactions", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - }, - { - "id": "3", - "params": { - "field": "status", - "order": "desc", - "orderBy": "1", - "size": 5 - }, - "schema": "group", - "type": "terms" - } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "mode": "percentage", - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "times": [], - "yAxis": {} - }, - "title": "Errors vs successful transactions", - "type": "histogram" + } + ] + } } - }, - "id": "Errors-vs-successful-transactions", - "type": "visualization", - "version": 1 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "Packetbeat-Search", - "title": "Latency histogram", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "enabled": true, - "id": "2", - "params": { - "extended_bounds": {}, - "field": "responsetime", - "interval": 10 - }, - "schema": "segment", - "type": "histogram" + } + ], + "highlightAll": true, + "index": "packetbeat-*", + "query": { + "language": "lucene", + "query": "" + }, + "version": true + } + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "DB transactions", + "version": 1 + }, + "id": "800e2a00-88cb-11e7-ad9c-db80de0bf8d3", + "type": "search", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "columns": [ + "_source" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "packetbeat-*", + "key": "type", + "negate": false, + "type": "phrases", + "value": "redis, memcache" + }, + "query": { + "bool": { + "minimum_should_match": 1, + "should": [ + { + "match_phrase": { + "type": "redis" } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "interpolate": "linear", - "legendPosition": "right", - "mode": "stacked", - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "smoothLines": true, - "times": [], - "yAxis": {} - }, - "title": "Latency histogram", - "type": "area" - } - }, - "id": "Latency-histogram", - "type": "visualization", - "version": 1 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "index": "packetbeat-*", - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "*" - } + }, + { + "match_phrase": { + "type": "memcache" } - } - }, - "title": "Client locations", - "uiStateJSON": { - "mapCenter": [ - 0, - -0.17578125 + } ] - }, - "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "enabled": true, - "id": "2", - "params": { - "autoPrecision": true, - "field": "client_geoip.location" - }, - "schema": "segment", - "type": "geohash_grid" - } - ], - "listeners": {}, - "params": { - "addTooltip": true, - "heatBlur": 15, - "heatMaxZoom": 16, - "heatMinOpacity": 0.1, - "heatNormalizeData": true, - "heatRadius": 25, - "isDesaturated": true, - "mapCenter": [ - 15, - 5 - ], - "mapType": "Scaled Circle Markers", - "mapZoom": 2, - "wms": { - "enabled": false, - "options": { - "attribution": "Maps provided by USGS", - "format": "image/png", - "layers": "0", - "styles": "", - "transparent": true, - "version": "1.3.0" - }, - "url": "https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer" - } - }, - "title": "Client locations", - "type": "tile_map" + } } - }, - "id": "Client-locations", - "type": "visualization", - "version": 1 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] + } + ], + "highlightAll": true, + "index": "packetbeat-*", + "query": { + "language": "lucene", + "query": "" + }, + "version": true + } + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "Cache Transactions [Packetbeat]", + "version": 1 + }, + "id": "3dd366e0-88cc-11e7-ad9c-db80de0bf8d3", + "type": "search", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "columns": [ + "_source" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "packetbeat-*", + "key": "type", + "negate": false, + "params": { + "query": "thrift", + "type": "phrase" + }, + "type": "phrase", + "value": "thrift" + }, + "query": { + "match": { + "type": { + "query": "thrift", + "type": "phrase" } - }, - "savedSearchId": "Packetbeat-Search", - "title": "Response times repartition", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - }, - { - "id": "3", - "params": { - "extended_bounds": {}, - "field": "responsetime", - "interval": 10 - }, - "schema": "group", - "type": "histogram" - } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "mode": "stacked", - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "times": [], - "yAxis": {} - }, - "title": "Response times repartition", - "type": "histogram" + } } - }, - "id": "Response-times-repartition", - "type": "visualization", - "version": 1 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "lucene", - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "*" - } - } - } - } - }, - "title": "Navigation", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [], - "params": { - "fontSize": "10", - "markdown": "### Packetbeat:\n\n[Overview](#/dashboard/Packetbeat-Dashboard)\n\n[Flows](#/dashboard/Packetbeat-Flows)\n\n[Web transactions](#/dashboard/Packetbeat-HTTP)\n\n[MySQL performance](#/dashboard/Packetbeat-MySQL-performance)\n\n[PostgreSQL performance](#/dashboard/Packetbeat-PgSQL-performance)\n\n[MongoDB performance](#/dashboard/Packetbeat-MongoDB-performance)\n\n[Thrift-RPC performance](#/dashboard/Packetbeat-Thrift-performance)\n\n[NFS transactions](#/dashboard/Packetbeat-NFS)\n\n[Cassandra performance](#/dashboard/Packetbeat-Cassandra)" - }, - "title": "Navigation", - "type": "markdown" + } + ], + "highlightAll": true, + "index": "packetbeat-*", + "query": { + "language": "lucene", + "query": "" + }, + "version": true + } + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "RPC transactions", + "version": 1 + }, + "id": "d3089370-88cc-11e7-ad9c-db80de0bf8d3", + "type": "search", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "columns": [ + "client_ip", + "client_port", + "ip", + "port", + "type", + "query", + "method", + "responsetime", + "status" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlight": { + "fields": { + "*": {} + }, + "fragment_size": 2147483647, + "post_tags": [ + "@/kibana-highlighted-field@" + ], + "pre_tags": [ + "@kibana-highlighted-field@" + ], + "require_field_match": false + }, + "index": "packetbeat-*", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "NOT type: flow" + } + } + } + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "Packetbeat Search", + "version": 1 + }, + "id": "Packetbeat-Search", + "type": "search", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "columns": [ + "client_ip", + "client_port", + "ip", + "port", + "type", + "query", + "method", + "responsetime", + "status" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlight": { + "fields": { + "*": {} + }, + "fragment_size": 2147483647, + "post_tags": [ + "@/kibana-highlighted-field@" + ], + "pre_tags": [ + "@kibana-highlighted-field@" + ], + "require_field_match": false + }, + "index": "packetbeat-*", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "!type: flow AND !status: OK" + } + } + } + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "Transactions errors", + "version": 1 + }, + "id": "Transactions-errors", + "type": "search", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + }, + { + "attributes": { + "description": "Packetbeat overview dashboard", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "lucene", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "*" } - }, - "id": "Navigation", - "type": "visualization", - "version": 10 - }, - { - "attributes": { - "columns": [ - "_source" - ], - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "packetbeat-*", - "key": "type", - "negate": false, - "params": { - "query": "http", - "type": "phrase" - }, - "type": "phrase", - "value": "http" - }, - "query": { - "match": { - "type": { - "query": "http", - "type": "phrase" - } - } - } - } - ], - "highlightAll": true, - "index": "packetbeat-*", - "query": { - "language": "lucene", - "query": "*" - }, - "version": true - } - }, - "sort": [ - "@timestamp", - "desc" - ], - "title": "Web Transactions [Packetbeat]", - "version": 1 - }, - "id": "71908f00-88ca-11e7-ad9c-db80de0bf8d3", - "type": "search", - "version": 2 - }, - { - "attributes": { - "columns": [ - "_source" - ], - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "packetbeat-*", - "key": "type", - "negate": false, - "type": "phrases", - "value": "mysql, postgresql" - }, - "query": { - "bool": { - "minimum_should_match": 1, - "should": [ - { - "match_phrase": { - "type": "mysql" - } - }, - { - "match_phrase": { - "type": "postgresql" - } - } - ] - } - } - } - ], - "highlightAll": true, - "index": "packetbeat-*", - "query": { - "language": "lucene", - "query": "" - }, - "version": true - } - }, - "sort": [ - "@timestamp", - "desc" - ], - "title": "DB transactions", - "version": 1 - }, - "id": "800e2a00-88cb-11e7-ad9c-db80de0bf8d3", - "type": "search", - "version": 1 - }, - { - "attributes": { - "columns": [ - "_source" - ], - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "packetbeat-*", - "key": "type", - "negate": false, - "type": "phrases", - "value": "redis, memcache" - }, - "query": { - "bool": { - "minimum_should_match": 1, - "should": [ - { - "match_phrase": { - "type": "redis" - } - }, - { - "match_phrase": { - "type": "memcache" - } - } - ] - } - } - } - ], - "highlightAll": true, - "index": "packetbeat-*", - "query": { - "language": "lucene", - "query": "" - }, - "version": true - } - }, - "sort": [ - "@timestamp", - "desc" - ], - "title": "Cache Transactions [Packetbeat]", - "version": 1 - }, - "id": "3dd366e0-88cc-11e7-ad9c-db80de0bf8d3", - "type": "search", - "version": 1 - }, - { - "attributes": { - "columns": [ - "_source" - ], - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "packetbeat-*", - "key": "type", - "negate": false, - "params": { - "query": "thrift", - "type": "phrase" - }, - "type": "phrase", - "value": "thrift" - }, - "query": { - "match": { - "type": { - "query": "thrift", - "type": "phrase" - } - } - } - } - ], - "highlightAll": true, - "index": "packetbeat-*", - "query": { - "language": "lucene", - "query": "" - }, - "version": true - } - }, - "sort": [ - "@timestamp", - "desc" - ], - "title": "RPC transactions", - "version": 1 - }, - "id": "d3089370-88cc-11e7-ad9c-db80de0bf8d3", - "type": "search", - "version": 1 - }, - { - "attributes": { - "columns": [ - "client_ip", - "client_port", - "ip", - "port", - "type", - "query", - "method", - "responsetime", - "status" - ], - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "highlight": { - "fields": { - "*": {} - }, - "fragment_size": 2147483647, - "post_tags": [ - "@/kibana-highlighted-field@" - ], - "pre_tags": [ - "@kibana-highlighted-field@" - ], - "require_field_match": false - }, - "index": "packetbeat-*", - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "NOT type: flow" - } - } - } - }, - "sort": [ - "@timestamp", - "desc" - ], - "title": "Packetbeat Search", - "version": 1 - }, - "id": "Packetbeat-Search", - "type": "search", - "version": 1 - }, - { - "attributes": { - "columns": [ - "client_ip", - "client_port", - "ip", - "port", - "type", - "query", - "method", - "responsetime", - "status" - ], - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "highlight": { - "fields": { - "*": {} - }, - "fragment_size": 2147483647, - "post_tags": [ - "@/kibana-highlighted-field@" - ], - "pre_tags": [ - "@kibana-highlighted-field@" - ], - "require_field_match": false - }, - "index": "packetbeat-*", - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "!type: flow AND !status: OK" - } - } - } - }, - "sort": [ - "@timestamp", - "desc" - ], - "title": "Transactions errors", - "version": 1 - }, - "id": "Transactions-errors", - "type": "search", - "version": 1 - }, - { - "attributes": { - "description": "Packetbeat overview dashboard", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "highlightAll": true, - "query": { - "language": "lucene", - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "*" - } - } - }, - "version": true - } - }, - "optionsJSON": { - "darkTheme": false - }, - "panelsJSON": [ - { - "col": 1, - "id": "Web-transactions", - "panelIndex": 1, - "row": 5, - "size_x": 3, - "size_y": 2, - "type": "visualization" - }, - { - "col": 4, - "id": "DB-transactions", - "panelIndex": 2, - "row": 5, - "size_x": 3, - "size_y": 2, - "type": "visualization" - }, - { - "col": 7, - "id": "Cache-transactions", - "panelIndex": 3, - "row": 5, - "size_x": 3, - "size_y": 2, - "type": "visualization" - }, - { - "col": 10, - "id": "RPC-transactions", - "panelIndex": 4, - "row": 5, - "size_x": 3, - "size_y": 2, - "type": "visualization" - }, - { - "col": 1, - "id": "Response-times-percentiles", - "panelIndex": 5, - "row": 10, - "size_x": 6, - "size_y": 3, - "type": "visualization" - }, - { - "col": 1, - "id": "Errors-count-over-time", - "panelIndex": 6, - "row": 13, - "size_x": 6, - "size_y": 3, - "type": "visualization" - }, - { - "col": 7, - "id": "Errors-vs-successful-transactions", - "panelIndex": 7, - "row": 10, - "size_x": 6, - "size_y": 3, - "type": "visualization" - }, - { - "col": 7, - "id": "Latency-histogram", - "panelIndex": 8, - "row": 13, - "size_x": 6, - "size_y": 3, - "type": "visualization" - }, - { - "col": 4, - "id": "Client-locations", - "panelIndex": 9, - "row": 1, - "size_x": 9, - "size_y": 4, - "type": "visualization" - }, - { - "col": 1, - "id": "Response-times-repartition", - "panelIndex": 10, - "row": 7, - "size_x": 12, - "size_y": 3, - "type": "visualization" - }, - { - "col": 1, - "id": "Navigation", - "panelIndex": 11, - "row": 1, - "size_x": 3, - "size_y": 4, - "type": "visualization" - } - ], - "timeRestore": false, - "title": "[Packetbeat] Overview", - "uiStateJSON": { - "P-9": { - "mapBounds": { - "bottom_right": { - "lat": -54.97761367069625, - "lon": 153.984375 - }, - "top_left": { - "lat": 54.97761367069628, - "lon": -153.984375 - } - }, - "mapCollar": { - "bottom_right": { - "lat": -90, - "lon": 180 - }, - "top_left": { - "lat": 90, - "lon": -180 - }, - "zoom": 2 - }, - "mapZoom": 2 - } - }, - "version": 1 - }, - "id": "Packetbeat-Dashboard", - "type": "dashboard", - "version": 2 - } - ], - "version": "6.0.0-beta1-SNAPSHOT" -} \ No newline at end of file + } + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "col": 1, + "id": "Web-transactions", + "panelIndex": 1, + "row": 5, + "size_x": 3, + "size_y": 2, + "type": "visualization" + }, + { + "col": 4, + "id": "DB-transactions", + "panelIndex": 2, + "row": 5, + "size_x": 3, + "size_y": 2, + "type": "visualization" + }, + { + "col": 7, + "id": "Cache-transactions", + "panelIndex": 3, + "row": 5, + "size_x": 3, + "size_y": 2, + "type": "visualization" + }, + { + "col": 10, + "id": "RPC-transactions", + "panelIndex": 4, + "row": 5, + "size_x": 3, + "size_y": 2, + "type": "visualization" + }, + { + "col": 1, + "id": "Response-times-percentiles", + "panelIndex": 5, + "row": 10, + "size_x": 6, + "size_y": 3, + "type": "visualization" + }, + { + "col": 1, + "id": "Errors-count-over-time", + "panelIndex": 6, + "row": 13, + "size_x": 6, + "size_y": 3, + "type": "visualization" + }, + { + "col": 7, + "id": "Errors-vs-successful-transactions", + "panelIndex": 7, + "row": 10, + "size_x": 6, + "size_y": 3, + "type": "visualization" + }, + { + "col": 7, + "id": "Latency-histogram", + "panelIndex": 8, + "row": 13, + "size_x": 6, + "size_y": 3, + "type": "visualization" + }, + { + "col": 4, + "id": "Client-locations", + "panelIndex": 9, + "row": 1, + "size_x": 9, + "size_y": 4, + "type": "visualization" + }, + { + "col": 1, + "id": "Response-times-repartition", + "panelIndex": 10, + "row": 7, + "size_x": 12, + "size_y": 3, + "type": "visualization" + }, + { + "col": 1, + "id": "Navigation", + "panelIndex": 11, + "row": 1, + "size_x": 3, + "size_y": 4, + "type": "visualization" + } + ], + "timeRestore": false, + "title": "[Packetbeat] Overview", + "uiStateJSON": { + "P-9": { + "mapBounds": { + "bottom_right": { + "lat": -54.97761367069625, + "lon": 153.984375 + }, + "top_left": { + "lat": 54.97761367069628, + "lon": -153.984375 + } + }, + "mapCollar": { + "bottom_right": { + "lat": -90, + "lon": 180 + }, + "top_left": { + "lat": 90, + "lon": -180 + }, + "zoom": 2 + }, + "mapZoom": 2 + } + }, + "version": 1 + }, + "id": "Packetbeat-Dashboard", + "type": "dashboard", + "updated_at": "2019-01-09T20:47:52.257Z", + "version": 1 + } + ], + "version": "7.0.0-SNAPSHOT" +} diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 8b689ccc21f..567d8f22438 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -5411,14 +5411,6 @@ Information about the HTTP request and response. HTTP request -*`http.request.params`*:: -+ --- -The query parameters or form values. The query parameters are available in the Request-URI and the form values are set in the HTTP body when the content-type is set to `x-www-form-urlencoded`. - - --- - *`http.request.headers`*:: + -- @@ -5427,15 +5419,6 @@ type: object A map containing the captured header fields from the request. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas. --- - -*`http.request.body`*:: -+ --- -type: text - -The body of the HTTP request. - -- [float] @@ -5444,19 +5427,10 @@ The body of the HTTP request. HTTP response -*`http.response.code`*:: -+ --- -example: 404 - -The HTTP status code. - --- - -*`http.response.phrase`*:: +*`http.response.status_phrase`*:: + -- -example: Not found. +example: Not Found The HTTP status phrase. diff --git a/packetbeat/protos/http/_meta/fields.yml b/packetbeat/protos/http/_meta/fields.yml index ac79fe2b09b..47e42c6c5a9 100644 --- a/packetbeat/protos/http/_meta/fields.yml +++ b/packetbeat/protos/http/_meta/fields.yml @@ -10,10 +10,6 @@ description: HTTP request type: group fields: - - name: params - description: > - The query parameters or form values. The query parameters are available in the Request-URI - and the form values are set in the HTTP body when the content-type is set to `x-www-form-urlencoded`. - name: headers type: object object_type: keyword @@ -22,21 +18,14 @@ Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas. - - name: body - type: text - description: The body of the HTTP request. - name: response description: HTTP response type: group fields: - - name: code - description: The HTTP status code. - example: 404 - - - name: phrase + - name: status_phrase description: The HTTP status phrase. - example: Not found. + example: Not Found - name: headers type: object diff --git a/packetbeat/protos/http/event.go b/packetbeat/protos/http/event.go new file mode 100644 index 00000000000..4cc508b8c86 --- /dev/null +++ b/packetbeat/protos/http/event.go @@ -0,0 +1,107 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package http + +import ( + "net" + "net/url" + "strconv" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/ecs/code/go/ecs" +) + +// ProtocolFields contains HTTP fields. This contains all the HTTP fields from +// ECS. The ecs.Http type is not used because we customize some of the data +// data types to reduce memory allocations (common.NetString instead of string). +type ProtocolFields struct { + // Http request method. + // The field value must be normalized to lowercase for querying. See + // "Lowercase Capitalization" in the "Implementing ECS" section. + RequestMethod common.NetString `ecs:"request.method"` + + // The full http request body. + RequestBodyContent common.NetString `ecs:"request.body.content"` + + // Referrer for this HTTP request. + RequestReferrer common.NetString `ecs:"request.referrer"` + + // Http response status code. + ResponseStatusCode int64 `ecs:"response.status_code"` + + // The full http response body. + ResponseBodyContent common.NetString `ecs:"response.body.content"` + + // Http version. + Version string `ecs:"version"` + + // Total size in bytes of the request (body and headers). + RequestBytes int64 `ecs:"request.bytes"` + + // Size in bytes of the request body. + RequestBodyBytes int64 `ecs:"request.body.bytes"` + + // Total size in bytes of the response (body and headers). + ResponseBytes int64 `ecs:"response.bytes"` + + // Size in bytes of the response body. + ResponseBodyBytes int64 `ecs:"response.body.bytes"` + + // HTTP request headers. + RequestHeaders common.MapStr `packetbeat:"request.headers"` + + // HTTP response headers. + ResponseHeaders common.MapStr `packetbeat:"response.headers"` + + // HTTP response status phrase. + ResponseStatusPhrase common.NetString `packetbeat:"response.status_phrase"` +} + +// netURL returns a new ecs.Url object with data from the HTTP request. +func newURL(host string, port int32, path, query string) *ecs.Url { + u := &ecs.Url{ + Scheme: "http", + Domain: host, + Path: path, + Query: query, + } + if port != 80 { + u.Port = port + } + u.Full = synthesizeFullURL(u, port) + return u +} + +func synthesizeFullURL(u *ecs.Url, port int32) string { + if u.Domain == "" || port <= 0 { + return "" + } + + host := u.Domain + if port != 80 { + host = net.JoinHostPort(u.Domain, strconv.Itoa(int(u.Port))) + } + + urlBuilder := url.URL{ + Scheme: u.Scheme, + Host: host, + Path: u.Path, + RawQuery: u.Query, + } + return urlBuilder.String() +} diff --git a/packetbeat/protos/http/event_test.go b/packetbeat/protos/http/event_test.go new file mode 100644 index 00000000000..1cbcb003066 --- /dev/null +++ b/packetbeat/protos/http/event_test.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package http + +import ( + "reflect" + "testing" + + "github.com/elastic/ecs/code/go/ecs" +) + +// TestProtocolFieldsIsInSyncWithECS ensures that Packetbeat's clone of +// ecs.Http stays in sync. +func TestProtocolFieldsIsInSyncWithECS(t *testing.T) { + ecs := getFields(reflect.TypeOf(ecs.Http{})) + packetbeat := getFields(reflect.TypeOf(ProtocolFields{})) + + for name := range ecs { + _, found := packetbeat[name] + if !found { + t.Errorf("Packetbeat is missing field=%v that's defined in ECS HTTP", name) + } + delete(packetbeat, name) + } + + for name := range packetbeat { + t.Errorf("packetbeat has more HTTP fields than ECS: %v", name) + } +} + +func getFields(typ reflect.Type) map[string]reflect.Type { + fields := map[string]reflect.Type{} + for i := 0; i < typ.NumField(); i++ { + structField := typ.Field(i) + tag := structField.Tag.Get("ecs") + if tag == "" { + continue + } + + fields[tag] = structField.Type + } + return fields +} diff --git a/packetbeat/protos/http/fields.go b/packetbeat/protos/http/fields.go index 1ffab72986c..9a84ff1a3e5 100644 --- a/packetbeat/protos/http/fields.go +++ b/packetbeat/protos/http/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzUVMGOnDAMvfMV1p4L6mFPHCr11rlU1WqqHruGmCFdSNjEGYa/rxyY6QykWo32tJzAcd6zn/3I4YWmElrmIQNgzR2V8PBtv//xkAEo8rXTA2trSpBg7geqdaNroCMZhkZTp3yRwfJWZgAAORjs6YIqD08DlXBwNpwjN9g701jXo3wAVjYwcEuRERy9BvIMaBQ48oM1nooF45r0mni5c4knOknkbGtMcVzzDOiw9zdHK6ovq0OAfUvwGshN821ich6sAxEAjtgF8kU6CR0BHlF3WHUE2kSNnuY28p9Puw2VSCY5V9ARxBOfr0cpKqsmGFuaQ7U1TIZzkQO0j9ls4fmUj+OYC1YeXEemtorUc5EUpiVU5NbKzALb6g/VvDqag7/njBeaRuvUfbp+hR6HWDxqo81h7gUHDo7UUtAyTGic7eP5sgTFBu1Xq+v23Ia0vyCJILU1jT4EJ2MoYNdc0kbNbYT12NMGcilBBIpTGBx58dAyiZ68xwN9ko8JRt11UMmoZAGYFFTTBrG2fY8+PQGZaVJ+ptNa/BtlZfXiQthmY8IiS1httuQbXtsk3W822be3Co9snpGDj/nrwdIJ+0F+cY+fH7O0pVuH/i6e+cZ/mb5bhsYGo4o04cexyu2/99/zLq9s0MQ77/TKBrOaLl75GwAA//8kTPxR" + return "eJzUU7mO1EAQzf0VTxsz/gAHSCSITRDBSISrsrs8btZ90FVm8N+j9jGs7ZEQmmg76q7jvTpen/DKY4VONRaAWu25wtOX8/nbUwEYlibZqDb4Ctl4ksiNbW0D/sVe0VrujZQFlltVAMAJnhzfUPPRMXKFSwrDatlgP/s2JEf5AarDoNCOJ0Yk/jmwKMgbJJYYvHC5YLwlfUu85Nzsdzq5E3Os8R7HpkEmw0k2vhUn1D+40Z1rNr7MEa88XkMyu5BNpR93TuATHEU0wStZb/1lGlRDUYfEZiloqRltCm7yL72WB7TvnW26tQ1oWJFgJXO09jIkqnsu8dzewq5WuwlWyPEBcikhDwiUGDGxZKlYP+U4FqELf8iPEVfb96gZwpESKRvU4wGxCc6RlMWdNc9y+MeeD0H/v2hR0kFeYpdI9i1vKM+rbOcMzBn7wfNvcjH/tK9B8TkM3hTvXF/bf/n3PCSwA1oW3IMCO2DW401gfwIAAP//G5dzPQ==" } diff --git a/packetbeat/protos/http/http.go b/packetbeat/protos/http/http.go index 4e37e860c3a..a2082811dfb 100644 --- a/packetbeat/protos/http/http.go +++ b/packetbeat/protos/http/http.go @@ -20,6 +20,7 @@ package http import ( "bytes" "fmt" + "net" "net/url" "strings" "time" @@ -30,9 +31,10 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/monitoring" - + "github.com/elastic/beats/packetbeat/pb" "github.com/elastic/beats/packetbeat/procs" "github.com/elastic/beats/packetbeat/protos" + "github.com/elastic/ecs/code/go/ecs" ) var debugf = logp.MakeDebug("http") @@ -495,86 +497,104 @@ func (http *httpPlugin) newTransaction(requ, resp *message) beat.Event { } } - httpDetails := common.MapStr{} - fields := common.MapStr{ - "type": "http", - "status": status, - "http": httpDetails, + var ts time.Time + var src, dst *common.Endpoint + for _, m := range []*message{requ, resp} { + if m == nil { + continue + } + ts = m.ts + src, dst = m.getEndpoints() + break } - var timestamp time.Time + evt, pbf := pb.NewBeatEvent(ts) + pbf.SetSource(src) + pbf.SetDestination(dst) + pbf.Network.Transport = "tcp" + pbf.Network.Protocol = "http" + fields := evt.Fields + fields["type"] = pbf.Network.Protocol + fields["status"] = status + + var httpFields ProtocolFields + var notes []string if requ != nil { - // Body must be decoded before extractParameters http.decodeBody(requ) path, params, err := http.extractParameters(requ) if err != nil { logp.Warn("Fail to parse HTTP parameters: %v", err) } - httpDetails["request"] = common.MapStr{ - "params": params, - "headers": http.collectHeaders(requ), - } - fields["method"] = requ.method - fields["path"] = path - fields["query"] = fmt.Sprintf("%s %s", requ.method, path) - fields["bytes_in"] = requ.size - - fields["src"], fields["dst"] = requ.getEndpoints() - - http.setBody(httpDetails["request"].(common.MapStr), requ) - timestamp = requ.ts - - if len(requ.notes) > 0 { - fields["notes"] = requ.notes + host := string(requ.host) + pbf.Source.Bytes = int64(requ.size) + if net.ParseIP(host) == nil { + pbf.Destination.Domain = host } + pbf.Event.Start = requ.ts + pbf.Network.ForwardedIP = string(requ.realIP) + notes = append(notes, requ.notes...) - if len(requ.realIP) > 0 { - fields["real_ip"] = requ.realIP + // http + httpFields.Version = requ.version.String() + httpFields.RequestBytes = int64(requ.size) + httpFields.RequestBodyBytes = int64(requ.contentLength) + httpFields.RequestMethod = bytes.ToLower(requ.method) + httpFields.RequestReferrer = requ.referer + if requ.sendBody && len(requ.body) > 0 { + httpFields.RequestBodyBytes = int64(len(requ.body)) + httpFields.RequestBodyContent = common.NetString(requ.body) } + httpFields.RequestHeaders = http.collectHeaders(requ) + + // url + u := newURL(host, int32(pbf.Destination.Port), path, params) + pb.MarshalStruct(evt.Fields, "url", u) + + // user-agent + userAgent := ecs.UserAgent{Original: string(requ.userAgent)} + pb.MarshalStruct(evt.Fields, "user_agent", userAgent) + // packetbeat root fields if http.sendRequest { fields["request"] = string(http.makeRawMessage(requ)) } + fields["query"] = fmt.Sprintf("%s %s", requ.method, path) } if resp != nil { http.decodeBody(resp) - httpDetails["response"] = common.MapStr{ - "code": resp.statusCode, - "phrase": resp.statusPhrase, - "headers": http.collectHeaders(resp), + + pbf.Destination.Bytes = int64(resp.size) + pbf.Event.End = resp.ts + notes = append(notes, resp.notes...) + + // http + httpFields.ResponseStatusCode = int64(resp.statusCode) + httpFields.ResponseStatusPhrase = bytes.ToLower(resp.statusPhrase) + httpFields.ResponseBytes = int64(resp.size) + httpFields.ResponseBodyBytes = int64(resp.contentLength) + if resp.sendBody && len(resp.body) > 0 { + httpFields.ResponseBodyBytes = int64(len(resp.body)) + httpFields.ResponseBodyContent = common.NetString(resp.body) } - http.setBody(httpDetails["response"].(common.MapStr), resp) - fields["bytes_out"] = resp.size + httpFields.ResponseHeaders = http.collectHeaders(resp) + // packetbeat root fields if http.sendResponse { fields["response"] = string(http.makeRawMessage(resp)) } - - if len(resp.notes) > 0 { - if fields["notes"] != nil { - fields["notes"] = append(fields["notes"].([]string), resp.notes...) - } else { - fields["notes"] = resp.notes - } - } - if requ == nil { - timestamp = resp.ts - fields["src"], fields["dst"] = resp.getEndpoints() - } } - // resp_time in milliseconds - if requ != nil && resp != nil { - fields["responsetime"] = int32(resp.ts.Sub(requ.ts).Nanoseconds() / 1e6) + if len(notes) == 1 { + fields.Put("error.message", notes[0]) + } else if len(notes) > 1 { + fields.Put("error.message", notes) } - return beat.Event{ - Timestamp: timestamp, - Fields: fields, - } + pb.MarshalStruct(evt.Fields, "http", httpFields) + return evt } func (http *httpPlugin) makeRawMessage(m *message) string { @@ -595,7 +615,7 @@ func (http *httpPlugin) publishTransaction(event beat.Event) { http.results(event) } -func (http *httpPlugin) collectHeaders(m *message) interface{} { +func (http *httpPlugin) collectHeaders(m *message) common.MapStr { hdrs := map[string]interface{}{} hdrs["content-length"] = m.contentLength @@ -604,19 +624,18 @@ func (http *httpPlugin) collectHeaders(m *message) interface{} { } if http.parserConfig.sendHeaders { - cookie := "cookie" if !m.isRequest { cookie = "set-cookie" } for name, value := range m.headers { - if strings.ToLower(name) == "content-type" { - continue - } - if strings.ToLower(name) == "content-length" { + switch { + case bytes.Equal([]byte(name), nameContentLength), + bytes.Equal([]byte(name), nameContentType): continue } + if http.splitCookie && name == cookie { hdrs[name] = splitCookiesHeader(string(value)) } else { @@ -627,12 +646,6 @@ func (http *httpPlugin) collectHeaders(m *message) interface{} { return hdrs } -func (http *httpPlugin) setBody(result common.MapStr, m *message) { - if m.sendBody && len(m.body) > 0 { - result.Put("body.content", string(m.body)) - } -} - func (http *httpPlugin) decodeBody(m *message) { if m.saveBody && len(m.body) > 0 { if http.mustDecodeBody && len(m.encodings) > 0 { diff --git a/packetbeat/protos/http/http_parser.go b/packetbeat/protos/http/http_parser.go index e629039c828..6ec8a53b6b1 100644 --- a/packetbeat/protos/http/http_parser.go +++ b/packetbeat/protos/http/http_parser.go @@ -55,6 +55,9 @@ type message struct { // Http Headers contentLength int contentType common.NetString + host common.NetString + referer common.NetString + userAgent common.NetString encodings []string isChunked bool headers map[string]common.NetString @@ -80,6 +83,13 @@ type version struct { minor uint8 } +func (v version) String() string { + if v.major == 1 && v.minor == 1 { + return "1.1" + } + return fmt.Sprintf("%d.%d", v.major, v.minor) +} + type parser struct { config *parserConfig } @@ -107,6 +117,9 @@ var ( nameTransferEncoding = []byte("transfer-encoding") nameContentEncoding = []byte("content-encoding") nameConnection = []byte("connection") + nameHost = []byte("host") + nameReferer = []byte("referer") + nameUserAgent = []byte("user-agent") ) func newParser(config *parserConfig) *parser { @@ -379,7 +392,6 @@ func (parser *parser) parseHeader(m *message, data []byte) (bool, bool, int) { // transfer-encoding. m.encodings = append(m.encodings, encodings...) } - } else if bytes.Equal(headerName, nameContentEncoding) { encodings := parseCommaSeparatedList(headerVal) // Append at the beginning of m.encodings, as Content-Encoding @@ -387,11 +399,16 @@ func (parser *parser) parseHeader(m *message, data []byte) (bool, bool, int) { m.encodings = append(encodings, m.encodings...) } else if bytes.Equal(headerName, nameConnection) { m.connection = headerVal - } - if len(config.realIPHeader) > 0 && bytes.Equal(headerName, []byte(config.realIPHeader)) { + } else if len(config.realIPHeader) > 0 && bytes.Equal(headerName, []byte(config.realIPHeader)) { if ips := bytes.SplitN(headerVal, []byte{','}, 2); len(ips) > 0 { m.realIP = trim(ips[0]) } + } else if bytes.Equal(headerName, nameHost) { + m.host = headerVal + } else if bytes.Equal(headerName, nameReferer) { + m.referer = headerVal + } else if bytes.Equal(headerName, nameUserAgent) { + m.userAgent = headerVal } if config.sendHeaders { diff --git a/packetbeat/protos/http/http_test.go b/packetbeat/protos/http/http_test.go index 34aff57ccc3..5e1bd41231b 100644 --- a/packetbeat/protos/http/http_test.go +++ b/packetbeat/protos/http/http_test.go @@ -1205,7 +1205,7 @@ func TestHttpParser_includeBodyFor(t *testing.T) { if err != nil { t.Fatal(err) } - assert.Equal(t, "done.", contents) + assert.Equal(t, common.NetString("done."), contents) } func TestHttpParser_sendRequestResponse(t *testing.T) { @@ -1307,8 +1307,10 @@ func Test_gap_in_body_http1dot0_fin(t *testing.T) { http.ReceivedFin(tcptuple, 1, private) trans := expectTransaction(t, &store) - assert.NotNil(t, trans) - assert.Equal(t, trans["notes"], []string{"Packet loss while capturing the response"}) + if assert.NotNil(t, trans) { + notes, _ := trans.GetValue("error.message") + assert.Equal(t, notes, "Packet loss while capturing the response") + } } func TestHttp_configsSettingAll(t *testing.T) { @@ -1598,14 +1600,14 @@ func TestHTTP_Encodings(t *testing.T) { assert.NotNil(t, trans, msg) body, err := trans.GetValue("http.response.body.content") if err == nil { - assert.Equal(t, testData.expectedBody, body, msg) + assert.Equal(t, common.NetString(testData.expectedBody), body, msg) } else { if len(testData.expectedBody) == 0 && len(testData.note) > 0 { - note, err := trans.GetValue("notes") + note, err := trans.GetValue("error.message") if !assert.Nil(t, err, msg) { - t.Fatal(err) + return } - assert.Equal(t, []string{testData.note}, note) + assert.Equal(t, testData.note, note) } else { t.Fatal(err) } @@ -1618,7 +1620,7 @@ func TestHTTP_Decoding_disabled(t *testing.T) { "Host: server\r\n" + "\r\n" - deflateBody := string([]byte{0xcb, 0xc8, 0xcf, 0x49, 0xe4, 0x02, 0x00}) + deflateBody := common.NetString{0xcb, 0xc8, 0xcf, 0x49, 0xe4, 0x02, 0x00} var store eventStore http := httpModForTests(&store) diff --git a/packetbeat/tests/system/test_0003_http_simple.py b/packetbeat/tests/system/test_0003_http_simple.py index fda9842bf52..1f62d609a16 100644 --- a/packetbeat/tests/system/test_0003_http_simple.py +++ b/packetbeat/tests/system/test_0003_http_simple.py @@ -16,18 +16,38 @@ def test_http_sample(self): assert all([o["server.ip"] == "192.168.1.110" for o in objs]) assert all([o["server.port"] == 80 for o in objs]) + assert all(["network.bytes" in o for o in objs]) + assert all([o["network.type"] == "ipv4" for o in objs]) + assert all([o["network.transport"] == "tcp" for o in objs]) + assert all([o["network.protocol"] == "http" for o in objs]) + assert all(["network.community_id" in o for o in objs]) + + assert all(["event.start" in o for o in objs]) + assert all(["event.end" in o for o in objs]) + assert all(["event.duration" in o for o in objs]) + + assert all(["http.request.method" in o for o in objs]) + assert all(["http.request.bytes" in o for o in objs]) + assert all(["http.response.bytes" in o for o in objs]) + assert all(["http.response.status_code" in o for o in objs]) + assert all(["http.response.status_phrase" in o for o in objs]) + + assert all(["url.full" in o for o in objs]) + + assert all(["user_agent.original" in o for o in objs]) + assert objs[0]["status"] == "OK" assert objs[1]["status"] == "OK" assert objs[2]["status"] == "Error" - assert all(["bytes_in" in o for o in objs]) - assert all(["bytes_out" in o for o in objs]) + assert all(["client.bytes" in o for o in objs]) + assert all(["server.bytes" in o for o in objs]) - assert objs[0]["bytes_in"] == 364 - assert objs[0]["bytes_out"] == 1000 + assert objs[0]["client.bytes"] == 364 + assert objs[0]["server.bytes"] == 1000 - assert objs[1]["bytes_in"] == 471 - assert objs[1]["bytes_out"] == 234 + assert objs[1]["client.bytes"] == 471 + assert objs[1]["server.bytes"] == 234 - assert objs[2]["bytes_in"] == 289 - assert objs[2]["bytes_out"] == 396 + assert objs[2]["client.bytes"] == 289 + assert objs[2]["server.bytes"] == 396 diff --git a/packetbeat/tests/system/test_0006_wsgi.py b/packetbeat/tests/system/test_0006_wsgi.py index 85d7019c1fb..65539872e70 100644 --- a/packetbeat/tests/system/test_0006_wsgi.py +++ b/packetbeat/tests/system/test_0006_wsgi.py @@ -23,10 +23,10 @@ def test_long_answer(self): assert o["client.port"] == 46249 assert o["server.port"] == 8888 assert o["status"] == "OK" - assert o["method"] == "GET" - assert o["path"] == "/" - assert o["http.response.code"] == 200 - assert o["http.response.phrase"] == "OK" + assert o["http.request.method"] == "get" + assert o["url.path"] == "/" + assert o["http.response.status_code"] == 200 + assert o["http.response.status_phrase"] == "ok" assert "request" not in objs[0] assert "response" not in objs[0] @@ -47,8 +47,8 @@ def test_drum_interaction(self): if i != 13]) assert objs[13]["status"] == "Error" - assert objs[13]["path"] == "/comment/" - assert objs[13]["http.response.code"] == 500 + assert objs[13]["url.path"] == "/comment/" + assert objs[13]["http.response.status_code"] == 500 def test_send_options(self): """ diff --git a/packetbeat/tests/system/test_0008_realip.py b/packetbeat/tests/system/test_0008_realip.py index bd028b1faab..1b249beb43c 100644 --- a/packetbeat/tests/system/test_0008_realip.py +++ b/packetbeat/tests/system/test_0008_realip.py @@ -19,7 +19,7 @@ def test_x_forward_for(self): assert len(objs) == 1 o = objs[0] - assert o["real_ip"] == "89.247.39.104" + assert o["network.forwarded_ip"] == "89.247.39.104" def test_x_forwarded_for_multiple_ip(self): self.render_config_template( @@ -33,4 +33,4 @@ def test_x_forwarded_for_multiple_ip(self): assert len(objs) == 1 o = objs[0] - assert o["real_ip"] == "89.247.39.104" + assert o["network.forwarded_ip"] == "89.247.39.104" diff --git a/packetbeat/tests/system/test_0010_http_10_connection_close.py b/packetbeat/tests/system/test_0010_http_10_connection_close.py index c1d2747a6ef..e034958f0db 100644 --- a/packetbeat/tests/system/test_0010_http_10_connection_close.py +++ b/packetbeat/tests/system/test_0010_http_10_connection_close.py @@ -14,7 +14,7 @@ def test_http_sample(self): assert obj["status"] == "OK" assert obj["http.response.headers"]["content-length"] == 11422 - assert obj["http.response.code"] == 200 + assert obj["http.response.status_code"] == 200 assert obj["type"] == "http" assert obj["client.ip"] == "127.0.0.1" assert obj["client.port"] == 37885 diff --git a/packetbeat/tests/system/test_0019_hide_params.py b/packetbeat/tests/system/test_0019_hide_params.py index d1b4d556d48..99937724c7b 100644 --- a/packetbeat/tests/system/test_0019_hide_params.py +++ b/packetbeat/tests/system/test_0019_hide_params.py @@ -24,8 +24,8 @@ def test_http_hide_post(self): assert len(objs) == 1 o = objs[0] assert o["type"] == "http" - assert o["http.request.params"] == "pass=xxxxx&user=monica" - assert o["path"] == "/login" + assert o["url.query"] == "pass=xxxxx&user=monica" + assert o["url.path"] == "/login" for _, val in o.items(): if isinstance(val, six.string_types): assert "secret" not in val @@ -45,8 +45,8 @@ def test_http_hide_get(self): assert len(objs) == 1 o = objs[0] assert o["type"] == "http" - assert o["http.request.params"] == "pass=xxxxx&user=monica" - assert o["path"] == "/login" + assert o["url.query"] == "pass=xxxxx&user=monica" + assert o["url.path"] == "/login" for _, val in o.items(): if isinstance(val, six.string_types): assert "secret" not in val @@ -63,5 +63,5 @@ def test_http_hide_post_default(self): assert len(objs) == 1 o = objs[0] assert o["type"] == "http" - assert o["http.request.params"] == "pass=secret&user=monica" - assert o["path"] == "/login" + assert o["url.query"] == "pass=secret&user=monica" + assert o["url.path"] == "/login" diff --git a/packetbeat/tests/system/test_0023_http_params.py b/packetbeat/tests/system/test_0023_http_params.py index ba989e0285d..a7b723867fd 100644 --- a/packetbeat/tests/system/test_0023_http_params.py +++ b/packetbeat/tests/system/test_0023_http_params.py @@ -19,8 +19,8 @@ def test_http_post(self): assert len(objs) == 1 o = objs[0] assert o["type"] == "http" - assert len(o["http.request.params"]) > 0 - assert o["http.request.params"] == "address=anklamerstr.14b&telephon=8932784368&" +\ + assert len(o["url.query"]) > 0 + assert o["url.query"] == "address=anklamerstr.14b&telephon=8932784368&" +\ "user=monica" def test_http_get(self): @@ -36,5 +36,5 @@ def test_http_get(self): o = objs[0] print(o) assert o["type"] == "http" - assert len(o["http.request.params"]) > 0 - assert o["http.request.params"] == "input=packetbeat&src_ip=192.35.243.1" + assert len(o["url.query"]) > 0 + assert o["url.query"] == "input=packetbeat&src_ip=192.35.243.1" diff --git a/packetbeat/tests/system/test_0024_http_query.py b/packetbeat/tests/system/test_0024_http_query.py index c340fc09ef7..6439255a397 100644 --- a/packetbeat/tests/system/test_0024_http_query.py +++ b/packetbeat/tests/system/test_0024_http_query.py @@ -34,4 +34,5 @@ def test_http_get(self): o = objs[0] assert o["type"] == "http" assert o["query"] == "GET /dashboard/transactions" - assert o["http.request.params"] == "input=packetbeat&src_ip=192.35.243.1" + assert o["url.path"] == "/dashboard/transactions" + assert o["url.query"] == "input=packetbeat&src_ip=192.35.243.1" diff --git a/packetbeat/tests/system/test_0029_http_gap.py b/packetbeat/tests/system/test_0029_http_gap.py index 28a41c14e58..011587e3a43 100644 --- a/packetbeat/tests/system/test_0029_http_gap.py +++ b/packetbeat/tests/system/test_0029_http_gap.py @@ -22,6 +22,5 @@ def test_gap_in_large_file(self): o = objs[0] assert o["status"] == "OK" - print(o["notes"]) - assert len(o["notes"]) == 1 - assert o["notes"][0] == "Packet loss while capturing the response" + print(o["error.message"]) + assert o["error.message"] == "Packet loss while capturing the response" diff --git a/packetbeat/tests/system/test_0060_processors.py b/packetbeat/tests/system/test_0060_processors.py index fc7984013f5..a844526666a 100644 --- a/packetbeat/tests/system/test_0060_processors.py +++ b/packetbeat/tests/system/test_0060_processors.py @@ -35,7 +35,7 @@ def test_drop_fields_with_cond(self): processors=[{ "drop_fields": { "fields": ["http.request.headers", "http.response.headers"], - "when": "equals.http.response.code: 200", + "when": "equals.http.response.status_code: 200", }, }] ) @@ -51,7 +51,7 @@ def test_drop_fields_with_cond(self): assert "http.response.headers" not in objs[0] assert "status" in objs[0] - assert "http.response.code" in objs[0] + assert "http.response.status_code" in objs[0] assert "http.request.headers" in objs[1] assert "http.response.headers" in objs[1] @@ -67,7 +67,7 @@ def test_include_fields_with_cond(self): processors=[{ "include_fields": { "fields": ["http"], - "when": "equals.http.code: 200", + "when": "equals.http.response.status_code: 200", }, }] ) @@ -82,10 +82,12 @@ def test_include_fields_with_cond(self): assert "http.request.headers" in objs[0] assert "http.response.headers" in objs[0] - assert "response" in objs[0] - assert "request" in objs[0] + assert objs[0]["http.response.status_code"] == 200 + assert objs[1]["http.response.status_code"] == 304 + assert objs[2]["http.response.status_code"] == 404 - assert "http.response.code" in objs[0] + assert "response" not in objs[0] + assert "request" not in objs[0] assert "request" in objs[1] assert "response" in objs[1] @@ -101,7 +103,7 @@ def test_drop_fields_with_cond_range(self): processors=[{ "drop_fields": { "fields": ["request", "response"], - "when": "range.http.response.code.lt: 300", + "when": "range.http.response.status_code.lt: 300", }, }] ) @@ -118,7 +120,7 @@ def test_drop_fields_with_cond_range(self): assert "request" not in objs[0] assert "status" in objs[0] - assert "http.response.code" in objs[0] + assert "http.response.status_code" in objs[0] assert "request" in objs[1] assert "response" in objs[1] @@ -131,7 +133,7 @@ def test_drop_event_with_cond(self): self.render_config_template( processors=[{ "drop_event": { - "when": "range.http.response.code.lt: 300", + "when": "range.http.response.status_code.lt: 300", }, }] ) @@ -144,7 +146,7 @@ def test_drop_event_with_cond(self): assert len(objs) == 2 assert all([o["type"] == "http" for o in objs]) - assert all([o["http.response.code"] > 300 for o in objs]) + assert all([o["http.response.status_code"] > 300 for o in objs]) def test_drop_end_fields(self): @@ -306,7 +308,7 @@ def test_drop_and_include_fields(self): }, { "drop_fields": { "fields": ["http.request.headers", "http.response.headers"], - "when": "equals.http.response.code: 200", + "when": "equals.http.response.status_code: 200", }, }] ) @@ -336,7 +338,7 @@ def test_condition_and(self): "when": """ and: - equals.type: http - - equals.http.response.code: 200 + - equals.http.response.status_code: 200 """ }, }] @@ -361,8 +363,8 @@ def test_condition_or(self): "drop_event": { "when": """ or: - - equals.http.response.code: 404 - - equals.http.response.code: 200 + - equals.http.response.status_code: 404 + - equals.http.response.status_code: 200 """ }, }] @@ -384,7 +386,7 @@ def test_condition_not(self): http_send_all_headers=True, processors=[{ "drop_event": { - "when.not": "equals.http.response.code: 200", + "when.not": "equals.http.response.status_code: 200", }, }] ) diff --git a/packetbeat/tests/system/test_0063_http_body.py b/packetbeat/tests/system/test_0063_http_body.py index 543db7e396f..349624e32d9 100644 --- a/packetbeat/tests/system/test_0063_http_body.py +++ b/packetbeat/tests/system/test_0063_http_body.py @@ -84,7 +84,7 @@ def test_wrong_content_type(self): assert o["http.request.headers"]["content-type"] == "application/x-www-form-urlencoded; charset=UTF-8" assert o["http.response.headers"]["content-type"] == "application/json; charset=UTF-8" - assert o["http.request.params"] == "%7B+%22query%22%3A+%7B+%22match_all%22%3A+%7B%7D%7D%7D%0A=" + assert o["url.query"] == "%7B+%22query%22%3A+%7B+%22match_all%22%3A+%7B%7D%7D%7D%0A=" assert len(o["http.request.body.content"]) > 0 assert len(o["http.response.body.content"]) > 0 diff --git a/packetbeat/tests/system/test_0065_unmatched_http.py b/packetbeat/tests/system/test_0065_unmatched_http.py index 57a33d630d4..9007dbe5c10 100644 --- a/packetbeat/tests/system/test_0065_unmatched_http.py +++ b/packetbeat/tests/system/test_0065_unmatched_http.py @@ -29,12 +29,12 @@ def test_unmatched_response(self): check_event(objs[0], { "type": "http", "status": "Error", - "http.response.code": 404, - "notes": ["Unmatched response"]}) + "http.response.status_code": 404, + "error.message": "Unmatched response"}) check_event(objs[1], { "type": "http", - "http.response.code": 200, + "http.response.status_code": 200, "http.request.headers": {"content-length": 0}, "status": "OK"}) @@ -57,4 +57,4 @@ def test_unmatched_request(self): "type": "http", "status": "Error", "query": "GET /something", - "notes": ["Unmatched request"]}) + "error.message": "Unmatched request"}) From bd247cec1d3a057a7591a46c2f5edb7b3f57ee88 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Jan 2019 10:40:40 -0500 Subject: [PATCH 2/3] Add http migration aliases --- dev-tools/ecs-migration.yml | 4 ++-- packetbeat/docs/fields.asciidoc | 27 +++++++++++++++++++++++++ packetbeat/protos/http/_meta/fields.yml | 15 ++++++++++++++ packetbeat/protos/http/fields.go | 2 +- 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 45dccc24665..ad080fa2da8 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -872,11 +872,11 @@ ## HTTP - from: http.request.body to: http.request.body.content - alias: true + alias: false - from: http.response.body to: http.response.body.content - alias: true + alias: false - from: http.response.code to: http.response.status_code diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 567d8f22438..1224e10056e 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -5419,6 +5419,15 @@ type: object A map containing the captured header fields from the request. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas. +-- + +*`http.request.params`*:: ++ +-- +type: alias + +alias to: url.query + -- [float] @@ -5444,6 +5453,24 @@ type: object A map containing the captured header fields from the response. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas. +-- + +*`http.response.code`*:: ++ +-- +type: alias + +alias to: http.response.status_code + +-- + +*`http.response.phrase`*:: ++ +-- +type: alias + +alias to: http.response.status_phrase + -- [[exported-fields-icmp]] diff --git a/packetbeat/protos/http/_meta/fields.yml b/packetbeat/protos/http/_meta/fields.yml index 47e42c6c5a9..72422ce77b0 100644 --- a/packetbeat/protos/http/_meta/fields.yml +++ b/packetbeat/protos/http/_meta/fields.yml @@ -19,6 +19,11 @@ header name are present in the message, they will be separated by commas. + - name: params + type: alias + migration: true + path: url.query + - name: response description: HTTP response type: group @@ -35,3 +40,13 @@ Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas. + + - name: code + type: alias + migration: true + path: http.response.status_code + + - name: phrase + type: alias + migration: true + path: http.response.status_phrase diff --git a/packetbeat/protos/http/fields.go b/packetbeat/protos/http/fields.go index 9a84ff1a3e5..e1fe7a5fe5f 100644 --- a/packetbeat/protos/http/fields.go +++ b/packetbeat/protos/http/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzUU7mO1EAQzf0VTxsz/gAHSCSITRDBSISrsrs8btZ90FVm8N+j9jGs7ZEQmmg76q7jvTpen/DKY4VONRaAWu25wtOX8/nbUwEYlibZqDb4Ctl4ksiNbW0D/sVe0VrujZQFlltVAMAJnhzfUPPRMXKFSwrDatlgP/s2JEf5AarDoNCOJ0Yk/jmwKMgbJJYYvHC5YLwlfUu85Nzsdzq5E3Os8R7HpkEmw0k2vhUn1D+40Z1rNr7MEa88XkMyu5BNpR93TuATHEU0wStZb/1lGlRDUYfEZiloqRltCm7yL72WB7TvnW26tQ1oWJFgJXO09jIkqnsu8dzewq5WuwlWyPEBcikhDwiUGDGxZKlYP+U4FqELf8iPEVfb96gZwpESKRvU4wGxCc6RlMWdNc9y+MeeD0H/v2hR0kFeYpdI9i1vKM+rbOcMzBn7wfNvcjH/tK9B8TkM3hTvXF/bf/n3PCSwA1oW3IMCO2DW401gfwIAAP//G5dzPQ==" + return "eJzUVMuOlUAQ3fMVJ7N2+AAWJm6MszEubuJyUkAB7dCP6a72yt+b5nEdoKMxNy5kBfU451T1aR7xwlOFQcQVgCgZucLDp8vly0MBtBwar5woayqk4GNw3KhONeDvbASd4rENZYH1rSoA4BGGNN9Q0yOT4wq9t3GL7LCfTGe9pvQBqm0UyMAzIzy/Rg4CMi08B2dN4HLFeEv6lnjtucUzk2RqzhpzHLsBmVr2YZfbcGz9jRs5pJbg81LxwtPV+vZQslP6/pAEPkCTQ2ONkDLK9POiGnISPberoFUzOm/1nF9nLU9oXwfVDNsYELshQYXE0ak+eqpHLvHU3cquSoYZNpDmE+QqIS0I5BnOc0hWUWbu0RwC9fwufUy4qnFEzQjsyJNwi3o6ITZWawplkT2C1KfzJ0CjomNGq97Tslvx8ajekQwVoh/L18h+KjLGWgz4B2ediv7eWkFIYnh2g6dwlLmjvGwXZenA0nE8av5B2qW7/dkKPtpo2vw6/x9H7/8Ev567LH1CSxa/09InzHr6vaUb2x6F3Gfo9B8ubxtbnTWz5K9UznP/QMHK8zMAAP//lgPSsQ==" } From 983c752f4174a345eedcc290e6cf65567550ab18 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 15 Jan 2019 09:47:51 -0500 Subject: [PATCH 3/3] Remove duplicate from ecs-migration --- dev-tools/ecs-migration.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index ad080fa2da8..5d4ad1e4f5b 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -764,18 +764,6 @@ to: process.executable alias: true -# Packetbeat - -- from: http.request.body - to: http.request.body.content - alias6: false - alias: false - -- from: http.response.body - to: http.response.body.content - alias6: false - alias: false - # Metricbeat