add latency switch

Signed-off-by: ii2day <ji.li@daocloud.io>
kdoctor-io · Jul 12, 2023 · 87e3cca · 87e3cca
1 parent b026a76
commit 87e3cca
Show file tree

Hide file tree

Showing 14 changed files with 132 additions and 69 deletions.
diff --git a/charts/crds/kdoctor.io_apphttphealthies.yaml b/charts/crds/kdoctor.io_apphttphealthies.yaml
@@ -93,6 +93,9 @@ spec:
                 type: object
               schedule:
                 properties:
+                  enableLatencyMetric:
+                    default: false
+                    type: boolean
                   roundNumber:
                     default: 1
                     format: int64

diff --git a/charts/crds/kdoctor.io_netdnses.yaml b/charts/crds/kdoctor.io_netdnses.yaml
@@ -104,6 +104,9 @@ spec:
                 type: object
               schedule:
                 properties:
+                  enableLatencyMetric:
+                    default: false
+                    type: boolean
                   roundNumber:
                     default: 1
                     format: int64

diff --git a/charts/crds/kdoctor.io_netreaches.yaml b/charts/crds/kdoctor.io_netreaches.yaml
@@ -93,6 +93,9 @@ spec:
                 type: object
               schedule:
                 properties:
+                  enableLatencyMetric:
+                    default: false
+                    type: boolean
                   roundNumber:
                     default: 1
                     format: int64

diff --git a/pkg/k8s/apis/kdoctor.io/v1beta1/common_types.go b/pkg/k8s/apis/kdoctor.io/v1beta1/common_types.go
@@ -17,6 +17,10 @@ type SchedulePlan struct {
 	// +kubebuilder:default=1
 	// +kubebuilder:validation:Minimum=-1
 	RoundNumber int64 `json:"roundNumber"`
+
+	// +kubebuilder:default=false
+	// +kubebuilder:validation:Optional
+	EnableLatencyMetric bool `json:"enableLatencyMetric,omitempty"`
 }
 
 type TaskStatus struct {

diff --git a/pkg/loadRequest/loadDns/dns.go b/pkg/loadRequest/loadDns/dns.go
@@ -34,6 +34,7 @@ type DnsRequestData struct {
 	PerRequestTimeoutInMs int
 	Qps                   int
 	DurationInSecond      int
+	EnableLatencyMetric   bool
 }
 
 func DnsRequest(logger *zap.Logger, reqData *DnsRequestData) (result *v1beta1.DNSMetrics, err error) {
@@ -52,12 +53,13 @@ func DnsRequest(logger *zap.Logger, reqData *DnsRequestData) (result *v1beta1.DN
 	duration := time.Duration(reqData.DurationInSecond) * time.Second
 
 	w := &Work{
-		Concurrency: config.AgentConfig.Configmap.NetdnsDefaultConcurrency,
-		QPS:         reqData.Qps,
-		Timeout:     reqData.PerRequestTimeoutInMs,
-		Msg:         new(dns.Msg).SetQuestion(reqData.TargetDomain, reqData.DnsType),
-		Protocol:    string(reqData.Protocol),
-		ServerAddr:  reqData.DnsServerAddr,
+		Concurrency:         config.AgentConfig.Configmap.NetdnsDefaultConcurrency,
+		QPS:                 reqData.Qps,
+		Timeout:             reqData.PerRequestTimeoutInMs,
+		Msg:                 new(dns.Msg).SetQuestion(reqData.TargetDomain, reqData.DnsType),
+		Protocol:            string(reqData.Protocol),
+		ServerAddr:          reqData.DnsServerAddr,
+		EnableLatencyMetric: reqData.EnableLatencyMetric,
 	}
 	w.Init()
 

diff --git a/pkg/loadRequest/loadDns/dns_reporter.go b/pkg/loadRequest/loadDns/dns_reporter.go
@@ -8,10 +8,9 @@ import (
 	"time"
 )
 
-// We report for max 1M results.
-const maxRes = 1000000
-
 type report struct {
+	enableLatencyMetric bool
+
 	avgTotal float64
 	average  float64
 	tps      float64
@@ -27,13 +26,14 @@ type report struct {
 	ReplyCode    map[string]int
 }
 
-func newReport(results chan *result) *report {
+func newReport(results chan *result, enableLatencyMetric bool) *report {
 	return &report{
-		results:   results,
-		done:      make(chan bool, 1),
-		errorDist: make(map[string]int),
-		lats:      make([]float32, 0, maxRes),
-		ReplyCode: make(map[string]int),
+		results:             results,
+		done:                make(chan bool, 1),
+		errorDist:           make(map[string]int),
+		lats:                make([]float32, 0),
+		ReplyCode:           make(map[string]int),
+		enableLatencyMetric: enableLatencyMetric,
 	}
 }
 
@@ -46,7 +46,7 @@ func runReporter(r *report) {
 			r.failedCount++
 		} else {
 			r.avgTotal += res.duration.Seconds()
-			if len(r.lats) < maxRes {
+			if r.enableLatencyMetric {
 				r.lats = append(r.lats, float32(res.duration.Milliseconds()))
 			}
 			rcodeStr := dns.RcodeToString[res.msg.Rcode]

diff --git a/pkg/loadRequest/loadDns/dns_requester.go b/pkg/loadRequest/loadDns/dns_requester.go
@@ -37,6 +37,8 @@ type Work struct {
 	// Qps is the rate limit in queries per second.
 	QPS int
 
+	EnableLatencyMetric bool
+
 	initOnce  sync.Once
 	results   chan *result
 	stopCh    chan struct{}
@@ -57,7 +59,7 @@ func (b *Work) Init() {
 func (b *Work) Run() {
 	b.Init()
 	b.startTime = metav1.Now()
-	b.report = newReport(b.results)
+	b.report = newReport(b.results, b.EnableLatencyMetric)
 	// Run the reporter first, it polls the result channel until it is closed.
 	go func() {
 		runReporter(b.report)
@@ -141,26 +143,28 @@ func (b *Work) runWorkers() {
 func (b *Work) AggregateMetric() *v1beta1.DNSMetrics {
 	latency := v1beta1.LatencyDistribution{}
 
-	t, _ := stats.Mean(b.report.lats)
-	latency.Mean = t
+	if b.EnableLatencyMetric {
+		t, _ := stats.Mean(b.report.lats)
+		latency.Mean = t
 
-	t, _ = stats.Max(b.report.lats)
-	latency.Max = t
+		t, _ = stats.Max(b.report.lats)
+		latency.Max = t
 
-	t, _ = stats.Min(b.report.lats)
-	latency.Min = t
+		t, _ = stats.Min(b.report.lats)
+		latency.Min = t
 
-	t, _ = stats.Percentile(b.report.lats, 50)
-	latency.P50 = t
+		t, _ = stats.Percentile(b.report.lats, 50)
+		latency.P50 = t
 
-	t, _ = stats.Percentile(b.report.lats, 90)
-	latency.P90 = t
+		t, _ = stats.Percentile(b.report.lats, 90)
+		latency.P90 = t
 
-	t, _ = stats.Percentile(b.report.lats, 95)
-	latency.P95 = t
+		t, _ = stats.Percentile(b.report.lats, 95)
+		latency.P95 = t
 
-	t, _ = stats.Percentile(b.report.lats, 99)
-	latency.P99 = t
+		t, _ = stats.Percentile(b.report.lats, 99)
+		latency.P99 = t
+	}
 
 	metric := &v1beta1.DNSMetrics{
 		StartTime:     b.startTime,

diff --git a/pkg/loadRequest/loadDns/dns_test.go b/pkg/loadRequest/loadDns/dns_test.go
@@ -46,6 +46,37 @@ var _ = Describe("test dns ", Label("dns"), func() {
 
 	})
 
+	It("test latency ", func() {
+
+		dnsServer := "223.5.5.5:53"
+		req := &loadDns.DnsRequestData{
+			Protocol:              loadDns.RequestMethodUdp,
+			DnsType:               dns.TypeA,
+			TargetDomain:          "www.baidu.com",
+			DnsServerAddr:         dnsServer,
+			PerRequestTimeoutInMs: 1000,
+			DurationInSecond:      1,
+			Qps:                   10,
+			EnableLatencyMetric:   true,
+		}
+
+		log := logger.NewStdoutLogger("debug", "test")
+		result, e := loadDns.DnsRequest(log, req)
+		Expect(e).NotTo(HaveOccurred(), "failed to execute , error=%v", e)
+		Expect(int(result.FailedCounts)).To(Equal(0))
+		Expect(len(result.ReplyCode)).To(Equal(1))
+		Expect(result.ReplyCode).Should(HaveKey(dns.RcodeToString[dns.RcodeSuccess]))
+
+		jsongByte, e := json.Marshal(result)
+		Expect(e).NotTo(HaveOccurred(), "failed to Marshal , error=%v", e)
+
+		var out bytes.Buffer
+		e = json.Indent(&out, jsongByte, "", "\t")
+		Expect(e).NotTo(HaveOccurred(), "failed to Indent , error=%v", e)
+		fmt.Printf("%s\n", out.String())
+
+	})
+
 	It("test tcp ", func() {
 
 		dnsServer := "223.5.5.5:53"

diff --git a/pkg/loadRequest/loadHttp/http.go b/pkg/loadRequest/loadHttp/http.go
@@ -42,6 +42,7 @@ type HttpRequestData struct {
 	DisableKeepAlives   bool
 	DisableCompression  bool
 	ExpectStatusCode    *int
+	EnableLatencyMetric bool
 }
 
 func HttpRequest(logger *zap.Logger, reqData *HttpRequestData) *v1beta1.HttpMetrics {
@@ -56,17 +57,18 @@ func HttpRequest(logger *zap.Logger, reqData *HttpRequestData) *v1beta1.HttpMetr
 	logger.Sugar().Infof("http request Concurrency=%d", config.AgentConfig.Configmap.NethttpDefaultConcurrency)
 
 	w := &Work{
-		Request:            req,
-		Concurrency:        config.AgentConfig.Configmap.NethttpDefaultConcurrency,
-		QPS:                reqData.Qps,
-		Timeout:            reqData.PerRequestTimeoutMS,
-		DisableCompression: reqData.DisableCompression,
-		DisableKeepAlives:  reqData.DisableKeepAlives,
-		Http2:              reqData.Http2,
-		Cert:               reqData.ClientCert,
-		CertPool:           reqData.CaCertPool,
-		ExpectStatusCode:   reqData.ExpectStatusCode,
-		RequestBody:        reqData.Body,
+		Request:             req,
+		Concurrency:         config.AgentConfig.Configmap.NethttpDefaultConcurrency,
+		QPS:                 reqData.Qps,
+		Timeout:             reqData.PerRequestTimeoutMS,
+		DisableCompression:  reqData.DisableCompression,
+		DisableKeepAlives:   reqData.DisableKeepAlives,
+		Http2:               reqData.Http2,
+		Cert:                reqData.ClientCert,
+		CertPool:            reqData.CaCertPool,
+		ExpectStatusCode:    reqData.ExpectStatusCode,
+		RequestBody:         reqData.Body,
+		EnableLatencyMetric: reqData.EnableLatencyMetric,
 	}
 	logger.Sugar().Infof("do http requests work=%v", w)
 	w.Init()

diff --git a/pkg/loadRequest/loadHttp/http_reporter.go b/pkg/loadRequest/loadHttp/http_reporter.go
@@ -24,11 +24,8 @@ import (
 	"time"
 )
 
-// We report for max 1M results.
-// todo (ii2day) configmap limit
-const maxRes = 1000000
-
 type report struct {
+	enableLatencyMetric bool
 	// transactions Per Second
 	tps float64
 
@@ -43,13 +40,14 @@ type report struct {
 	totalCount  int64
 }
 
-func newReport(results chan *result, n int) *report {
+func newReport(results chan *result, enableLatencyMetric bool) *report {
 	return &report{
-		results:     results,
-		done:        make(chan bool, 1),
-		errorDist:   make(map[string]int),
-		latencies:   make([]float32, 0, maxRes),
-		statusCodes: make(map[int]int),
+		results:             results,
+		done:                make(chan bool, 1),
+		errorDist:           make(map[string]int),
+		latencies:           make([]float32, 0),
+		statusCodes:         make(map[int]int),
+		enableLatencyMetric: enableLatencyMetric,
 	}
 }
 
@@ -61,7 +59,7 @@ func runReporter(r *report) {
 		if res.err != nil {
 			r.errorDist[res.err.Error()]++
 		} else {
-			if len(r.latencies) < maxRes {
+			if r.enableLatencyMetric {
 				r.latencies = append(r.latencies, float32(res.duration.Milliseconds()))
 			}
 			if res.contentLength > 0 {

diff --git a/pkg/loadRequest/loadHttp/http_requester.go b/pkg/loadRequest/loadHttp/http_requester.go
@@ -30,7 +30,6 @@ import (
 	"golang.org/x/net/http2"
 	"io"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"math"
 	"net/http"
 	"net/http/httptrace"
 	"net/url"
@@ -133,6 +132,10 @@ type Work struct {
 	// Optional.
 	ExpectStatusCode *int
 
+	// EnableLatencyMetric is collect latency metric . default false
+	// Optional.
+	EnableLatencyMetric bool
+
 	initOnce  sync.Once
 	results   chan *result
 	stopCh    chan struct{}
@@ -155,7 +158,7 @@ func (b *Work) Run() {
 	b.Init()
 	b.startTime = metav1.Now()
 	b.start = time.Since(b.startTime.Time)
-	b.report = newReport(b.results, math.MaxInt32)
+	b.report = newReport(b.results, b.EnableLatencyMetric)
 	// Run the reporter first, it polls the result channel until it is closed.
 	go func() {
 		runReporter(b.report)
@@ -319,26 +322,28 @@ func (b *Work) now() time.Duration { return time.Since(b.startTime.Time) }
 func (b *Work) AggregateMetric() *v1beta1.HttpMetrics {
 	latency := v1beta1.LatencyDistribution{}
 
-	t, _ := stats.Mean(b.report.latencies)
-	latency.Mean = t
+	if b.EnableLatencyMetric {
+		t, _ := stats.Mean(b.report.latencies)
+		latency.Mean = t
 
-	t, _ = stats.Max(b.report.latencies)
-	latency.Max = t
+		t, _ = stats.Max(b.report.latencies)
+		latency.Max = t
 
-	t, _ = stats.Min(b.report.latencies)
-	latency.Min = t
+		t, _ = stats.Min(b.report.latencies)
+		latency.Min = t
 
-	t, _ = stats.Percentile(b.report.latencies, 50)
-	latency.P50 = t
+		t, _ = stats.Percentile(b.report.latencies, 50)
+		latency.P50 = t
 
-	t, _ = stats.Percentile(b.report.latencies, 90)
-	latency.P90 = t
+		t, _ = stats.Percentile(b.report.latencies, 90)
+		latency.P90 = t
 
-	t, _ = stats.Percentile(b.report.latencies, 95)
-	latency.P95 = t
+		t, _ = stats.Percentile(b.report.latencies, 95)
+		latency.P95 = t
 
-	t, _ = stats.Percentile(b.report.latencies, 99)
-	latency.P99 = t
+		t, _ = stats.Percentile(b.report.latencies, 99)
+		latency.P99 = t
+	}
 
 	var errNum int64
 	for _, v := range b.report.errorDist {

diff --git a/pkg/pluginManager/apphttphealthy/agentExecuteTask.go b/pkg/pluginManager/apphttphealthy/agentExecuteTask.go
@@ -102,6 +102,7 @@ func (s *PluginAppHttpHealthy) AgentExecuteTask(logger *zap.Logger, ctx context.
 		RequestTimeSecond:   request.DurationInSecond,
 		Http2:               target.Http2,
 		ExpectStatusCode:    instance.Spec.SuccessCondition.StatusCode,
+		EnableLatencyMetric: instance.Spec.Schedule.EnableLatencyMetric,
 	}
 
 	// https cert