From c3c54d8c361e2085150b56f939f0f60ed74a4dbc Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 16 Jan 2019 17:02:43 -0500 Subject: [PATCH 1/2] Update Redis protocol to use ECS fields Here's a summary of what fields changed. Part of #7968 Changed - bytes_in -> source.bytes - bytes_out -> destination.bytes - responsetime -> event.duration (unit are now nanoseconds) - redis.error -> error.message (alias added) Added - source - destination - event.dataset = redis - event.end - event.start - network.community_id - network.transport = tcp - network.protocol = redis - network.bytes - network.type Unchanged Packetbeat Fields - method - resource - path - query - status - type = redis (we might remove this since we have event.dataset) --- CHANGELOG.next.asciidoc | 1 + dev-tools/ecs-migration.yml | 4 ++ packetbeat/docs/fields.asciidoc | 4 ++ packetbeat/protos/redis/_meta/fields.yml | 3 + packetbeat/protos/redis/fields.go | 2 +- packetbeat/protos/redis/redis.go | 61 ++++++++----------- .../tests/system/test_0013_redis_basic.py | 15 ++--- 7 files changed, 47 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e55192ad93e..597bf5dd0f8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -60,6 +60,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Changed TLS protocol fields to align with ECS. {pull}9980[9980] - Changed ICMP protocol fields to align with ECS. {pull}10062[10062] - Changed AMQP protocol fields to align with ECS. {pull}10090[10090] +- Changed Redis protocol fields to align with ECS. {pull}10126[10126] - Changed HTTP protocol fields to align with ECS. {pull}9976[9976] *Winlogbeat* diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 5d4ad1e4f5b..54375f91214 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -891,6 +891,10 @@ to: network.forwarded_ip alias: false +## Redis +- from: redis.error + to: error.message + alias: true # Heartbeat diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 1224e10056e..07e9e09896d 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -6517,6 +6517,10 @@ The return value of the Redis command in a human readable format. *`redis.error`*:: + -- +type: alias + +alias to: error.message + If the Redis command has resulted in an error, this field contains the error message returned by the Redis server. diff --git a/packetbeat/protos/redis/_meta/fields.yml b/packetbeat/protos/redis/_meta/fields.yml index e0dbd852ea5..eea49252f52 100644 --- a/packetbeat/protos/redis/_meta/fields.yml +++ b/packetbeat/protos/redis/_meta/fields.yml @@ -11,6 +11,9 @@ The return value of the Redis command in a human readable format. - name: error + type: alias + path: error.message + migration: true description: > If the Redis command has resulted in an error, this field contains the error message returned by the Redis server. diff --git a/packetbeat/protos/redis/fields.go b/packetbeat/protos/redis/fields.go index 98feabd4a1d..db3ae0e0b6c 100644 --- a/packetbeat/protos/redis/fields.go +++ b/packetbeat/protos/redis/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJyMj71uwzAMhHc9xSFzkwfw0L1r0b1grHMs1JIMkg7gty/8kzZBM5STRPK7Ox7xxbmBMiYLgCcf2ODwvvwPAYi0VtPoqZYGrwEA1tnRRrapSy14ZXF0iUO0U8D+atbVI4pk/sov5fPIBhet07h37olHyictn1cZJv4Mn0a61UfPncJKoXbwnltktDVnKRGpQNBPWQqUEuU8EF3VLH4Kf0JQter/3N+emfViUNo0ODfnskm+wPtk2+1oa3FJxRb+QXJdRaaZXG6nMeI83zkZ9Uo9hfAdAAD//8E/imI=" + return "eJyMkL1u6zAMhXc9xUHmGz+Ah7t3LboXjE3HRPVjkHSAvH1hS20dNEM5SSQPz0ee8cH3HsqjWABcPHKP0+v2PwVgZBtUFpeSe/wPALDXzrbwIJMM4BtnxyQcR+sC2qvfW8/IlPhn/BZ+X7jHVcu6tMxR8ajyVfP7jeLK38WnSF/xNnNTYVehTPCZKzKGkhLlEZJBmNdEGco00iUypqKJvAu/IFi16MGk4lMUskN2IZ9bb5fYjK5H4iRXpcrr+tddXp6hz2RQtjU61z1yNf0Hn8XqJTGU7CTZNv3DyL0Vja8dikdc7gcnY72xdiF8BgAA//8NmJ+O" } diff --git a/packetbeat/protos/redis/redis.go b/packetbeat/protos/redis/redis.go index 4e51dbb2e52..34600ef9ea8 100644 --- a/packetbeat/protos/redis/redis.go +++ b/packetbeat/protos/redis/redis.go @@ -26,6 +26,7 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/monitoring" + "github.com/elastic/beats/packetbeat/pb" "github.com/elastic/beats/packetbeat/procs" "github.com/elastic/beats/packetbeat/protos" "github.com/elastic/beats/packetbeat/protos/applayer" @@ -275,44 +276,37 @@ func (redis *redisPlugin) correlate(conn *redisConnectionData) { } func (redis *redisPlugin) newTransaction(requ, resp *redisMessage) beat.Event { - error := common.OK_STATUS - if resp.isError { - error = common.ERROR_STATUS - } - - var returnValue map[string]common.NetString - if resp.isError { - returnValue = map[string]common.NetString{ - "error": resp.message, - } - } else { - returnValue = map[string]common.NetString{ - "return_value": resp.message, - } - } - source, destination := common.MakeEndpointPair(requ.tcpTuple.BaseTuple, requ.cmdlineTuple) src, dst := &source, &destination if requ.direction == tcp.TCPDirectionReverse { src, dst = dst, src } - // resp_time in milliseconds - responseTime := int32(resp.ts.Sub(requ.ts).Nanoseconds() / 1e6) - - fields := common.MapStr{ - "type": "redis", - "status": error, - "responsetime": responseTime, - "redis": returnValue, - "method": common.NetString(bytes.ToUpper(requ.method)), - "resource": requ.path, - "query": requ.message, - "bytes_in": uint64(requ.size), - "bytes_out": uint64(resp.size), - "src": src, - "dst": dst, + evt, pbf := pb.NewBeatEvent(requ.ts) + pbf.SetSource(src) + pbf.SetDestination(dst) + pbf.Source.Bytes = int64(requ.size) + pbf.Destination.Bytes = int64(resp.size) + pbf.Event.Dataset = "redis" + pbf.Event.Start = requ.ts + pbf.Event.End = resp.ts + pbf.Network.Transport = "tcp" + pbf.Network.Protocol = pbf.Event.Dataset + + fields := evt.Fields + fields["type"] = pbf.Event.Dataset + fields["method"] = common.NetString(bytes.ToUpper(requ.method)) + fields["resource"] = requ.path + fields["query"] = requ.message + + if resp.isError { + evt.PutValue("status", common.ERROR_STATUS) + evt.PutValue("error.message", resp.message) + } else { + evt.PutValue("status", common.OK_STATUS) + evt.PutValue("redis.return_value", resp.message) } + if redis.sendRequest { fields["request"] = requ.message } @@ -320,10 +314,7 @@ func (redis *redisPlugin) newTransaction(requ, resp *redisMessage) beat.Event { fields["response"] = resp.message } - return beat.Event{ - Timestamp: requ.ts, - Fields: fields, - } + return evt } func (redis *redisPlugin) GapInStream(tcptuple *common.TCPTuple, dir uint8, diff --git a/packetbeat/tests/system/test_0013_redis_basic.py b/packetbeat/tests/system/test_0013_redis_basic.py index c5392a27e67..f9953d13246 100644 --- a/packetbeat/tests/system/test_0013_redis_basic.py +++ b/packetbeat/tests/system/test_0013_redis_basic.py @@ -20,6 +20,7 @@ def test_redis_session(self): objs = self.read_output() assert all([o["type"] == "redis" for o in objs]) + assert all([o["event.dataset"] == "redis" for o in objs]) assert objs[0]["method"] == "SET" assert objs[0]["resource"] == "key3" @@ -35,7 +36,7 @@ def test_redis_session(self): assert objs[2]["status"] == "Error" assert objs[2]["method"] == "LLEN" - assert objs[2]["redis.error"] == "ERR Operation against a key " + \ + assert objs[2]["error.message"] == "ERR Operation against a key " + \ "holding the wrong kind of value" # the rest should be successful @@ -45,8 +46,8 @@ def test_redis_session(self): assert all([isinstance(o["resource"], six.string_types) for o in objs[3:]]) assert all([isinstance(o["query"], six.string_types) for o in objs[3:]]) - assert all(["bytes_in" in o for o in objs]) - assert all(["bytes_out" in o for o in objs]) + assert all(["source.bytes" in o for o in objs]) + assert all(["destination.bytes" in o for o in objs]) def test_byteout_bytein(self): """ @@ -60,7 +61,7 @@ def test_byteout_bytein(self): objs = self.read_output() assert all([o["type"] == "redis" for o in objs]) - assert all([isinstance(o["bytes_out"], int) for o in objs]) - assert all([isinstance(o["bytes_in"], int) for o in objs]) - assert all([o["bytes_out"] > 0 for o in objs]) - assert all([o["bytes_in"] > 0 for o in objs]) + assert all([isinstance(o["source.bytes"], int) for o in objs]) + assert all([isinstance(o["destination.bytes"], int) for o in objs]) + assert all([o["source.bytes"] > 0 for o in objs]) + assert all([o["destination.bytes"] > 0 for o in objs]) From c06b45e0b204cd96c07cb21cbd7d90e9d438b21c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 17 Jan 2019 15:52:12 -0500 Subject: [PATCH 2/2] Revert redis.error rename to error.message This made it hard to tell the difference between Packetbeat processing errors and protocol level errors. --- dev-tools/ecs-migration.yml | 4 ---- packetbeat/docs/fields.asciidoc | 4 ---- packetbeat/protos/redis/_meta/fields.yml | 3 --- packetbeat/protos/redis/fields.go | 2 +- packetbeat/protos/redis/redis.go | 2 +- packetbeat/tests/system/test_0013_redis_basic.py | 2 +- 6 files changed, 3 insertions(+), 14 deletions(-) diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 54375f91214..5d4ad1e4f5b 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -891,10 +891,6 @@ to: network.forwarded_ip alias: false -## Redis -- from: redis.error - to: error.message - alias: true # Heartbeat diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 07e9e09896d..1224e10056e 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -6517,10 +6517,6 @@ The return value of the Redis command in a human readable format. *`redis.error`*:: + -- -type: alias - -alias to: error.message - If the Redis command has resulted in an error, this field contains the error message returned by the Redis server. diff --git a/packetbeat/protos/redis/_meta/fields.yml b/packetbeat/protos/redis/_meta/fields.yml index eea49252f52..e0dbd852ea5 100644 --- a/packetbeat/protos/redis/_meta/fields.yml +++ b/packetbeat/protos/redis/_meta/fields.yml @@ -11,9 +11,6 @@ The return value of the Redis command in a human readable format. - name: error - type: alias - path: error.message - migration: true description: > If the Redis command has resulted in an error, this field contains the error message returned by the Redis server. diff --git a/packetbeat/protos/redis/fields.go b/packetbeat/protos/redis/fields.go index db3ae0e0b6c..98feabd4a1d 100644 --- a/packetbeat/protos/redis/fields.go +++ b/packetbeat/protos/redis/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJyMkL1u6zAMhXc9xUHmGz+Ah7t3LboXjE3HRPVjkHSAvH1hS20dNEM5SSQPz0ee8cH3HsqjWABcPHKP0+v2PwVgZBtUFpeSe/wPALDXzrbwIJMM4BtnxyQcR+sC2qvfW8/IlPhn/BZ+X7jHVcu6tMxR8ajyVfP7jeLK38WnSF/xNnNTYVehTPCZKzKGkhLlEZJBmNdEGco00iUypqKJvAu/IFi16MGk4lMUskN2IZ9bb5fYjK5H4iRXpcrr+tddXp6hz2RQtjU61z1yNf0Hn8XqJTGU7CTZNv3DyL0Vja8dikdc7gcnY72xdiF8BgAA//8NmJ+O" + return "eJyMj71uwzAMhHc9xSFzkwfw0L1r0b1grHMs1JIMkg7gty/8kzZBM5STRPK7Ox7xxbmBMiYLgCcf2ODwvvwPAYi0VtPoqZYGrwEA1tnRRrapSy14ZXF0iUO0U8D+atbVI4pk/sov5fPIBhet07h37olHyictn1cZJv4Mn0a61UfPncJKoXbwnltktDVnKRGpQNBPWQqUEuU8EF3VLH4Kf0JQter/3N+emfViUNo0ODfnskm+wPtk2+1oa3FJxRb+QXJdRaaZXG6nMeI83zkZ9Uo9hfAdAAD//8E/imI=" } diff --git a/packetbeat/protos/redis/redis.go b/packetbeat/protos/redis/redis.go index 34600ef9ea8..c043d3bc1c7 100644 --- a/packetbeat/protos/redis/redis.go +++ b/packetbeat/protos/redis/redis.go @@ -301,7 +301,7 @@ func (redis *redisPlugin) newTransaction(requ, resp *redisMessage) beat.Event { if resp.isError { evt.PutValue("status", common.ERROR_STATUS) - evt.PutValue("error.message", resp.message) + evt.PutValue("redis.error", resp.message) } else { evt.PutValue("status", common.OK_STATUS) evt.PutValue("redis.return_value", resp.message) diff --git a/packetbeat/tests/system/test_0013_redis_basic.py b/packetbeat/tests/system/test_0013_redis_basic.py index f9953d13246..0b56810bb4c 100644 --- a/packetbeat/tests/system/test_0013_redis_basic.py +++ b/packetbeat/tests/system/test_0013_redis_basic.py @@ -36,7 +36,7 @@ def test_redis_session(self): assert objs[2]["status"] == "Error" assert objs[2]["method"] == "LLEN" - assert objs[2]["error.message"] == "ERR Operation against a key " + \ + assert objs[2]["redis.error"] == "ERR Operation against a key " + \ "holding the wrong kind of value" # the rest should be successful