From 4e5165bbe763506c678b64e6d78db67a8996832a Mon Sep 17 00:00:00 2001 From: Daxin Wang Date: Thu, 22 Sep 2022 11:15:09 +0800 Subject: [PATCH 1/4] Add request/response payload of Redis to span data Signed-off-by: Daxin Wang --- .../network/protocol/dubbo/dubbo_request.go | 2 +- .../network/protocol/dubbo/dubbo_response.go | 2 +- .../network/protocol/http/http_parser_test.go | 8 +++---- .../network/protocol/http/http_request.go | 2 +- .../network/protocol/http/http_response.go | 2 +- .../analyzer/network/protocol/protocol.go | 2 +- .../network/protocol/redis/redis_request.go | 2 ++ .../network/protocol/redis/redis_response.go | 2 ++ .../exporter/tools/adapter/net_dict.go | 10 +++++---- collector/pkg/model/constlabels/protocols.go | 21 ++++++++----------- 10 files changed, 28 insertions(+), 25 deletions(-) diff --git a/collector/pkg/component/analyzer/network/protocol/dubbo/dubbo_request.go b/collector/pkg/component/analyzer/network/protocol/dubbo/dubbo_request.go index b3c92cc8b..e5fe33bae 100644 --- a/collector/pkg/component/analyzer/network/protocol/dubbo/dubbo_request.go +++ b/collector/pkg/component/analyzer/network/protocol/dubbo/dubbo_request.go @@ -19,7 +19,7 @@ func parseDubboRequest() protocol.ParsePkgFn { } message.AddStringAttribute(constlabels.ContentKey, contentKey) - message.AddStringAttribute(constlabels.DubboRequestPayload, getAsciiString(message.GetData(16, protocol.GetDubboPayLoadLength()))) + message.AddStringAttribute(constlabels.RequestPayload, getAsciiString(message.GetData(16, protocol.GetDubboPayLoadLength()))) return true, true } } diff --git a/collector/pkg/component/analyzer/network/protocol/dubbo/dubbo_response.go b/collector/pkg/component/analyzer/network/protocol/dubbo/dubbo_response.go index c5ea4aa4e..e914ff8b2 100644 --- a/collector/pkg/component/analyzer/network/protocol/dubbo/dubbo_response.go +++ b/collector/pkg/component/analyzer/network/protocol/dubbo/dubbo_response.go @@ -23,7 +23,7 @@ func parseDubboResponse() protocol.ParsePkgFn { message.AddBoolAttribute(constlabels.IsError, true) message.AddIntAttribute(constlabels.ErrorType, int64(constlabels.ProtocolError)) } - message.AddStringAttribute(constlabels.DubboResponsePayload, getAsciiString(message.GetData(16, protocol.GetDubboPayLoadLength()))) + message.AddStringAttribute(constlabels.ResponsePayload, getAsciiString(message.GetData(16, protocol.GetDubboPayLoadLength()))) return true, true } } diff --git a/collector/pkg/component/analyzer/network/protocol/http/http_parser_test.go b/collector/pkg/component/analyzer/network/protocol/http/http_parser_test.go index de06a6568..7bf663db3 100644 --- a/collector/pkg/component/analyzer/network/protocol/http/http_parser_test.go +++ b/collector/pkg/component/analyzer/network/protocol/http/http_parser_test.go @@ -101,10 +101,10 @@ func TestParseHttpRequest_GetPayLoad(t *testing.T) { message := protocol.NewRequestMessage([]byte(httpData)) NewHttpParser("").ParseRequest(message) - if !message.HasAttribute(constlabels.HttpRequestPayload) { + if !message.HasAttribute(constlabels.RequestPayload) { t.Errorf("Fail to parse HttpRequest()") } - if got := message.GetStringAttribute(constlabels.HttpRequestPayload); got != tt.want { + if got := message.GetStringAttribute(constlabels.RequestPayload); got != tt.want { t.Errorf("GetHttpPayload() = %v, want %v", got, tt.want) } }) @@ -130,10 +130,10 @@ func TestParseHttpResponse_GetPayLoad(t *testing.T) { message := protocol.NewResponseMessage([]byte(httpData), model.NewAttributeMap()) NewHttpParser("").ParseResponse(message) - if !message.HasAttribute(constlabels.HttpResponsePayload) { + if !message.HasAttribute(constlabels.ResponsePayload) { t.Errorf("Fail to parse HttpResponse()") } - if got := message.GetStringAttribute(constlabels.HttpResponsePayload); got != tt.want { + if got := message.GetStringAttribute(constlabels.ResponsePayload); got != tt.want { t.Errorf("GetHttpPayload() = %v, want %v", got, tt.want) } }) diff --git a/collector/pkg/component/analyzer/network/protocol/http/http_request.go b/collector/pkg/component/analyzer/network/protocol/http/http_request.go index ab69e98ea..563e66de7 100644 --- a/collector/pkg/component/analyzer/network/protocol/http/http_request.go +++ b/collector/pkg/component/analyzer/network/protocol/http/http_request.go @@ -54,7 +54,7 @@ func parseHttpRequest(urlClusteringMethod urlclustering.ClusteringMethod) protoc message.AddStringAttribute(constlabels.HttpMethod, string(method)) message.AddByteArrayUtf8Attribute(constlabels.HttpUrl, url) - message.AddByteArrayUtf8Attribute(constlabels.HttpRequestPayload, message.GetData(0, protocol.GetHttpPayLoadLength())) + message.AddByteArrayUtf8Attribute(constlabels.RequestPayload, message.GetData(0, protocol.GetHttpPayLoadLength())) contentKey := urlClusteringMethod.Clustering(string(url)) if len(contentKey) == 0 { diff --git a/collector/pkg/component/analyzer/network/protocol/http/http_response.go b/collector/pkg/component/analyzer/network/protocol/http/http_response.go index e7b834be2..94311bc00 100644 --- a/collector/pkg/component/analyzer/network/protocol/http/http_response.go +++ b/collector/pkg/component/analyzer/network/protocol/http/http_response.go @@ -57,7 +57,7 @@ func parseHttpResponse() protocol.ParsePkgFn { } message.AddIntAttribute(constlabels.HttpStatusCode, statusCodeI) - message.AddByteArrayUtf8Attribute(constlabels.HttpResponsePayload, message.GetData(0, protocol.GetHttpPayLoadLength())) + message.AddByteArrayUtf8Attribute(constlabels.ResponsePayload, message.GetData(0, protocol.GetHttpPayLoadLength())) if statusCodeI >= 400 { message.AddBoolAttribute(constlabels.IsError, true) message.AddIntAttribute(constlabels.ErrorType, int64(constlabels.ProtocolError)) diff --git a/collector/pkg/component/analyzer/network/protocol/protocol.go b/collector/pkg/component/analyzer/network/protocol/protocol.go index 426981a7c..d367f164a 100644 --- a/collector/pkg/component/analyzer/network/protocol/protocol.go +++ b/collector/pkg/component/analyzer/network/protocol/protocol.go @@ -20,7 +20,7 @@ func GetPayLoadLength(protocol string) int { if length, ok := payloadLength[protocol]; ok { return length } - return 80 + return 200 } func GetHttpPayLoadLength() int { diff --git a/collector/pkg/component/analyzer/network/protocol/redis/redis_request.go b/collector/pkg/component/analyzer/network/protocol/redis/redis_request.go index 79255ea50..699566e5a 100644 --- a/collector/pkg/component/analyzer/network/protocol/redis/redis_request.go +++ b/collector/pkg/component/analyzer/network/protocol/redis/redis_request.go @@ -2,6 +2,7 @@ package redis import ( "github.com/Kindling-project/kindling/collector/pkg/component/analyzer/network/protocol" + "github.com/Kindling-project/kindling/collector/pkg/model/constlabels" ) /* @@ -20,6 +21,7 @@ func fastfailRedisRequest() protocol.FastFailFn { func parseRedisRequest() protocol.ParsePkgFn { return func(message *protocol.PayloadMessage) (bool, bool) { + message.AddByteArrayUtf8Attribute(constlabels.RequestPayload, message.GetData(0, protocol.GetPayLoadLength(protocol.REDIS))) return true, false } } diff --git a/collector/pkg/component/analyzer/network/protocol/redis/redis_response.go b/collector/pkg/component/analyzer/network/protocol/redis/redis_response.go index 17eeccfd8..a3ba4b76e 100644 --- a/collector/pkg/component/analyzer/network/protocol/redis/redis_response.go +++ b/collector/pkg/component/analyzer/network/protocol/redis/redis_response.go @@ -2,6 +2,7 @@ package redis import ( "github.com/Kindling-project/kindling/collector/pkg/component/analyzer/network/protocol" + "github.com/Kindling-project/kindling/collector/pkg/model/constlabels" ) /* @@ -24,6 +25,7 @@ func fastfailResponse() protocol.FastFailFn { func parseResponse() protocol.ParsePkgFn { return func(message *protocol.PayloadMessage) (bool, bool) { + message.AddByteArrayUtf8Attribute(constlabels.ResponsePayload, message.GetData(0, protocol.GetPayLoadLength(protocol.REDIS))) return true, false } } diff --git a/collector/pkg/component/consumer/exporter/tools/adapter/net_dict.go b/collector/pkg/component/consumer/exporter/tools/adapter/net_dict.go index 829e03030..cc2ee5c5b 100644 --- a/collector/pkg/component/consumer/exporter/tools/adapter/net_dict.go +++ b/collector/pkg/component/consumer/exporter/tools/adapter/net_dict.go @@ -224,9 +224,9 @@ var spanProtocol = []extraLabelsParam{ {constlabels.SpanHttpStatusCode, constlabels.HttpStatusCode, Int64}, {constlabels.SpanHttpTraceId, constlabels.HttpApmTraceId, String}, {constlabels.SpanHttpTraceType, constlabels.HttpApmTraceType, String}, - {constlabels.SpanHttpRequestHeaders, constlabels.HttpRequestPayload, String}, + {constlabels.SpanHttpRequestHeaders, constlabels.RequestPayload, String}, {constlabels.SpanHttpRequestBody, constlabels.STR_EMPTY, StrEmpty}, - {constlabels.SpanHttpResponseHeaders, constlabels.HttpResponsePayload, String}, + {constlabels.SpanHttpResponseHeaders, constlabels.ResponsePayload, String}, {constlabels.SpanHttpResponseBody, constlabels.STR_EMPTY, StrEmpty}, }, extraLabelsKey{HTTP}}, {[]dictionary{ @@ -239,13 +239,15 @@ var spanProtocol = []extraLabelsParam{ {constlabels.SpanDnsRCode, constlabels.DnsRcode, FromInt64ToString}, }, extraLabelsKey{DNS}}, {[]dictionary{ - {constlabels.SpanDubboRequestBody, constlabels.DubboRequestPayload, String}, - {constlabels.SpanDubboResponseBody, constlabels.DubboResponsePayload, String}, + {constlabels.SpanDubboRequestBody, constlabels.RequestPayload, String}, + {constlabels.SpanDubboResponseBody, constlabels.ResponsePayload, String}, {constlabels.SpanDubboErrorCode, constlabels.DubboErrorCode, Int64}, }, extraLabelsKey{DUBBO}}, {[]dictionary{ {constlabels.SpanRedisCommand, constlabels.RedisCommand, String}, {constlabels.SpanRedisErrorMsg, constlabels.RedisErrMsg, String}, + {constlabels.RequestPayload, constlabels.RequestPayload, String}, + {constlabels.ResponsePayload, constlabels.ResponsePayload, String}, }, extraLabelsKey{REDIS}}, { []dictionary{}, extraLabelsKey{UNSUPPORTED}, diff --git a/collector/pkg/model/constlabels/protocols.go b/collector/pkg/model/constlabels/protocols.go index fe6b9869a..b20847cfe 100644 --- a/collector/pkg/model/constlabels/protocols.go +++ b/collector/pkg/model/constlabels/protocols.go @@ -1,15 +1,15 @@ package constlabels const ( - ContentKey = "content_key" + ContentKey = "content_key" + RequestPayload = "request_payload" + ResponsePayload = "response_payload" - HttpMethod = "http_method" - HttpUrl = "http_url" - HttpApmTraceType = "trace_type" - HttpApmTraceId = "trace_id" - HttpRequestPayload = "request_payload" - HttpResponsePayload = "response_payload" - HttpStatusCode = "http_status_code" + HttpMethod = "http_method" + HttpUrl = "http_url" + HttpApmTraceType = "trace_type" + HttpApmTraceId = "trace_id" + HttpStatusCode = "http_status_code" DnsId = "dns_id" DnsDomain = "dns_domain" @@ -27,10 +27,7 @@ const ( KafkaVersion = "kafka_version" KafkaCorrelationId = "kafka_id" KafkaTopic = "kafka_topic" - KafkaPartition = "kafka_partition" KafkaErrorCode = "kafka_error_code" - DubboRequestPayload = "request_payload" - DubboResponsePayload = "response_payload" - DubboErrorCode = "dubbo_error_code" + DubboErrorCode = "dubbo_error_code" ) From 6da2956ab2ccc1dd47ea192cbe4b3e6d387874b0 Mon Sep 17 00:00:00 2001 From: Daxin Wang Date: Thu, 22 Sep 2022 11:28:03 +0800 Subject: [PATCH 2/4] Add new testcases Signed-off-by: Daxin Wang --- .../network/protocol/testdata/redis/server-trace-get.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/collector/pkg/component/analyzer/network/protocol/testdata/redis/server-trace-get.yml b/collector/pkg/component/analyzer/network/protocol/testdata/redis/server-trace-get.yml index df19f8ce1..b6bef6b84 100644 --- a/collector/pkg/component/analyzer/network/protocol/testdata/redis/server-trace-get.yml +++ b/collector/pkg/component/analyzer/network/protocol/testdata/redis/server-trace-get.yml @@ -46,3 +46,5 @@ trace: redis_command: "get" is_error: false error_type: 0 + request_payload: "*2\r\n$3\r\nget\r\n$3\r\nkey\r\n" + response_payload: "$3\r\nabc\r\n" From 3190ee430222350ac7b20b1442073b3663d93d75 Mon Sep 17 00:00:00 2001 From: Daxin Wang Date: Thu, 22 Sep 2022 13:56:30 +0800 Subject: [PATCH 3/4] Rename fields name in Span data Signed-off-by: Daxin Wang --- .../component/consumer/exporter/tools/adapter/net_dict.go | 4 ++-- collector/pkg/model/constlabels/const.go | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/collector/pkg/component/consumer/exporter/tools/adapter/net_dict.go b/collector/pkg/component/consumer/exporter/tools/adapter/net_dict.go index cc2ee5c5b..ad3609b7c 100644 --- a/collector/pkg/component/consumer/exporter/tools/adapter/net_dict.go +++ b/collector/pkg/component/consumer/exporter/tools/adapter/net_dict.go @@ -246,8 +246,8 @@ var spanProtocol = []extraLabelsParam{ {[]dictionary{ {constlabels.SpanRedisCommand, constlabels.RedisCommand, String}, {constlabels.SpanRedisErrorMsg, constlabels.RedisErrMsg, String}, - {constlabels.RequestPayload, constlabels.RequestPayload, String}, - {constlabels.ResponsePayload, constlabels.ResponsePayload, String}, + {constlabels.SpanRedisRequestPayload, constlabels.RequestPayload, String}, + {constlabels.SpanRedisResponsePayload, constlabels.ResponsePayload, String}, }, extraLabelsKey{REDIS}}, { []dictionary{}, extraLabelsKey{UNSUPPORTED}, diff --git a/collector/pkg/model/constlabels/const.go b/collector/pkg/model/constlabels/const.go index f06abadec..f9963c326 100644 --- a/collector/pkg/model/constlabels/const.go +++ b/collector/pkg/model/constlabels/const.go @@ -103,8 +103,10 @@ const ( SpanDubboRequestBody = "dubbo.request_body" SpanDubboResponseBody = "dubbo.response_body" - SpanRedisCommand = "redis.command" - SpanRedisErrorMsg = "redis.error_msg" + SpanRedisCommand = "redis.command" + SpanRedisErrorMsg = "redis.error_msg" + SpanRedisRequestPayload = "redis.request_payload" + SpanRedisResponsePayload = "redis.request_payload" NetWorkAnalyzeMetricGroup = "netAnalyzeMetrics" ) From af92b26a3cda1d0ce80a19bb256dde52c560e891 Mon Sep 17 00:00:00 2001 From: Daxin Wang Date: Thu, 22 Sep 2022 14:07:15 +0800 Subject: [PATCH 4/4] update changelog Signed-off-by: Daxin Wang --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc325c3f6..0e34d2828 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,15 @@ 2. Records in this file are not identical to the title of their Pull Requests. A detailed description is necessary for understanding what changes are and why they are made. ## Unreleased +## Enhancements +- Add request and response payload of `Redis` protocol message to `Span` data. ([#325](https://github.com/CloudDectective-Harmonycloud/kindling/pull/325)) ## v0.4.1 - 2022-09-21 ### Enhancements - When processing Redis' Requests, add additional labels to describe the key information of the message. Check [Metrics Document](https://github.com/CloudDectective-Harmonycloud/kindling/blob/main/docs/prometheus_metrics.md) for more details. ([#321](https://github.com/CloudDectective-Harmonycloud/kindling/pull/321)) ### Bug fixes -- Fix the bug when the kernel does not support some kprobe, the probe crashes. +- Fix the bug when the kernel does not support some kprobe, the probe crashes. ([#320](https://github.com/CloudDectective-Harmonycloud/kindling/pull/320)) ## v0.4.0 - 2022-09-19 ### Enhancements