From cbe977f8b88c856075fab90ce7f516df93e96cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Senart?= Date: Sat, 11 Oct 2014 13:10:14 +0200 Subject: [PATCH 1/4] Improve percentiles computation with HDR histogram This commit also introduces new field in the Metrics.Latencies struct. * Min * P999 --- lib/metrics.go | 47 ++++++++++++++++++++++++++++------------------- lib/reporters.go | 6 +++--- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/lib/metrics.go b/lib/metrics.go index 56525bb6..fd6317cc 100644 --- a/lib/metrics.go +++ b/lib/metrics.go @@ -4,7 +4,7 @@ import ( "strconv" "time" - "github.com/bmizerany/perks/quantile" + hdr "github.com/codahale/hdrhistogram" ) // Metrics holds the stats computed out of a slice of Results @@ -12,10 +12,12 @@ import ( type Metrics struct { Latencies struct { Mean time.Duration `json:"mean"` - P50 time.Duration `json:"50th"` // P50 is the 50th percentile upper value - P95 time.Duration `json:"95th"` // P95 is the 95th percentile upper value - P99 time.Duration `json:"99th"` // P99 is the 99th percentile upper value + P50 time.Duration `json:"50th"` // P50 is the 50th percentile upper value + P95 time.Duration `json:"95th"` // P95 is the 95th percentile upper value + P99 time.Duration `json:"99th"` // P99 is the 99th percentile upper value + P999 time.Duration `json:"999th"` // P999 is the 99.9th percentile upper value Max time.Duration `json:"max"` + Min time.Duration `json:"min"` } `json:"latencies"` BytesIn struct { @@ -51,48 +53,55 @@ func NewMetrics(r Results) *Metrics { } var ( - errorSet = map[string]struct{}{} - quants = quantile.NewTargeted(0.50, 0.95, 0.99) - totalSuccess int - totalLatencies time.Duration - latest time.Time + errorSet = map[string]struct{}{} + success int + latest time.Time ) + m.Latencies.Min = r[0].Latency + m.Latencies.Max = m.Latencies.Min + for _, result := range r { - quants.Insert(float64(result.Latency)) m.StatusCodes[strconv.Itoa(int(result.Code))]++ - totalLatencies += result.Latency m.BytesOut.Total += result.BytesOut m.BytesIn.Total += result.BytesIn if result.Latency > m.Latencies.Max { m.Latencies.Max = result.Latency } + if result.Latency < m.Latencies.Min { + m.Latencies.Min = result.Latency + } if end := result.Timestamp.Add(result.Latency); end.After(latest) { latest = end } - if result.Code >= 200 && result.Code < 300 { - totalSuccess++ + if result.Code >= 200 && result.Code < 400 { + success++ } if result.Error != "" { errorSet[result.Error] = struct{}{} } } + hist := hdr.New(int64(m.Latencies.Min), int64(m.Latencies.Max), 5) + for _, result := range r { + hist.RecordValue(int64(result.Latency)) + } + m.Requests = uint64(len(r)) m.Duration = r[len(r)-1].Timestamp.Sub(r[0].Timestamp) m.Wait = latest.Sub(r[len(r)-1].Timestamp) - m.Latencies.Mean = time.Duration(float64(totalLatencies) / float64(m.Requests)) - m.Latencies.P50 = time.Duration(quants.Query(0.50)) - m.Latencies.P95 = time.Duration(quants.Query(0.95)) - m.Latencies.P99 = time.Duration(quants.Query(0.99)) + m.Latencies.Mean = time.Duration(hist.Mean()) + m.Latencies.P50 = time.Duration(hist.ValueAtQuantile(0.50)) + m.Latencies.P95 = time.Duration(hist.ValueAtQuantile(0.95)) + m.Latencies.P99 = time.Duration(hist.ValueAtQuantile(0.99)) + m.Latencies.P999 = time.Duration(hist.ValueAtQuantile(0.999)) m.BytesIn.Mean = float64(m.BytesIn.Total) / float64(m.Requests) m.BytesOut.Mean = float64(m.BytesOut.Total) / float64(m.Requests) - m.Success = float64(totalSuccess) / float64(m.Requests) + m.Success = float64(success) / float64(m.Requests) m.Errors = make([]string, 0, len(errorSet)) for err := range errorSet { m.Errors = append(m.Errors, err) } - return m } diff --git a/lib/reporters.go b/lib/reporters.go index 8d9db752..3aa6389e 100644 --- a/lib/reporters.go +++ b/lib/reporters.go @@ -26,11 +26,11 @@ var ReportText ReporterFunc = func(r Results) ([]byte, error) { m := NewMetrics(r) out := &bytes.Buffer{} - w := tabwriter.NewWriter(out, 0, 8, 2, '\t', tabwriter.StripEscape) + w := tabwriter.NewWriter(out, 0, 8, 2, ' ', tabwriter.StripEscape) fmt.Fprintf(w, "Requests\t[total]\t%d\n", m.Requests) fmt.Fprintf(w, "Duration\t[total, attack, wait]\t%s, %s, %s\n", m.Duration+m.Wait, m.Duration, m.Wait) - fmt.Fprintf(w, "Latencies\t[mean, 50, 95, 99, max]\t%s, %s, %s, %s, %s\n", - m.Latencies.Mean, m.Latencies.P50, m.Latencies.P95, m.Latencies.P99, m.Latencies.Max) + fmt.Fprintf(w, "Latencies\t[mean, min, 50, 95, 99, 999, max]\t%s, %s, %s, %s, %s, %s, %s\n", + m.Latencies.Mean, m.Latencies.Min, m.Latencies.P50, m.Latencies.P95, m.Latencies.P99, m.Latencies.P999, m.Latencies.Max) fmt.Fprintf(w, "Bytes In\t[total, mean]\t%d, %.2f\n", m.BytesIn.Total, m.BytesIn.Mean) fmt.Fprintf(w, "Bytes Out\t[total, mean]\t%d, %.2f\n", m.BytesOut.Total, m.BytesOut.Mean) fmt.Fprintf(w, "Success\t[ratio]\t%.2f%%\n", m.Success*100) From 17be39916978895b70b19da1d88651d3b028547c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Senart?= Date: Tue, 11 Nov 2014 10:46:05 +0100 Subject: [PATCH 2/4] Start ticker closer to work loop --- lib/attack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/attack.go b/lib/attack.go index 82288e48..4e859839 100644 --- a/lib/attack.go +++ b/lib/attack.go @@ -124,13 +124,13 @@ func TLSConfig(c *tls.Config) func(*Attacker) { // as soon as they arrive. func (a *Attacker) Attack(tr Targeter, rate uint64, du time.Duration) chan *Result { resc := make(chan *Result) - throttle := time.NewTicker(time.Duration(1e9 / rate)) hits := rate * uint64(du.Seconds()) wrk := a.workers if wrk == 0 || wrk > hits { wrk = hits } share := hits / wrk + throttle := time.NewTicker(time.Duration(1e9 / rate)) var wg sync.WaitGroup for i := uint64(0); i < wrk; i++ { From 7c585ad723153a22327169e90e91343da0e2f516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Senart?= Date: Tue, 11 Nov 2014 10:51:56 +0100 Subject: [PATCH 3/4] Fix order of magnitude passed to ValueAtQuantile --- lib/metrics.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/metrics.go b/lib/metrics.go index fd6317cc..f5242b27 100644 --- a/lib/metrics.go +++ b/lib/metrics.go @@ -91,10 +91,10 @@ func NewMetrics(r Results) *Metrics { m.Duration = r[len(r)-1].Timestamp.Sub(r[0].Timestamp) m.Wait = latest.Sub(r[len(r)-1].Timestamp) m.Latencies.Mean = time.Duration(hist.Mean()) - m.Latencies.P50 = time.Duration(hist.ValueAtQuantile(0.50)) - m.Latencies.P95 = time.Duration(hist.ValueAtQuantile(0.95)) - m.Latencies.P99 = time.Duration(hist.ValueAtQuantile(0.99)) - m.Latencies.P999 = time.Duration(hist.ValueAtQuantile(0.999)) + m.Latencies.P50 = time.Duration(hist.ValueAtQuantile(50)) + m.Latencies.P95 = time.Duration(hist.ValueAtQuantile(95)) + m.Latencies.P99 = time.Duration(hist.ValueAtQuantile(99)) + m.Latencies.P999 = time.Duration(hist.ValueAtQuantile(99.9)) m.BytesIn.Mean = float64(m.BytesIn.Total) / float64(m.Requests) m.BytesOut.Mean = float64(m.BytesOut.Total) / float64(m.Requests) m.Success = float64(success) / float64(m.Requests) From 5320e3179b06cba65c250bb8305b6631e4e18ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Senart?= Date: Wed, 12 Nov 2014 20:11:59 +0100 Subject: [PATCH 4/4] Update README --- README.md | 57 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 1662a470..a93abfa8 100644 --- a/README.md +++ b/README.md @@ -159,50 +159,51 @@ Specifies the kind of report to be generated. It defaults to text. ##### text ``` -Requests [total] 1200 -Duration [total, attack, wait] 10.094965987s, 9.949883921s, 145.082066ms -Latencies [mean, 50, 95, 99, max] 113.172398ms, 108.272568ms, 140.18235ms, 247.771566ms, 264.815246ms -Bytes In [total, mean] 3714690, 3095.57 -Bytes Out [total, mean] 0, 0.00 -Success [ratio] 55.42% -Status Codes [code:count] 0:535 200:665 +Requests [total] 3000 +Duration [total, attack, wait] 10.000908765s, 10.000333187s, 575.578us +Latencies [mean, min, 50, 95, 99, 999, max] 290.959305ms, 239.872us, 655.359us, 1.871052799s, 2.239496191s, 2.318794751s, 2.327768447s +Bytes In [total, mean] 17478504, 5826.17 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 99.90% +Status Codes [code:count] 200:2997 0:3 Error Set: -Get http://localhost:6060: dial tcp 127.0.0.1:6060: connection refused -Get http://localhost:6060: read tcp 127.0.0.1:6060: connection reset by peer -Get http://localhost:6060: dial tcp 127.0.0.1:6060: connection reset by peer -Get http://localhost:6060: write tcp 127.0.0.1:6060: broken pipe -Get http://localhost:6060: net/http: transport closed before response was received -Get http://localhost:6060: http: can't write HTTP request on broken connection +Get http://:6060: read tcp 127.0.0.1:6060: connection reset by peer +Get http://:6060: net/http: transport closed before response was received +Get http://:6060: write tcp 127.0.0.1:6060: broken pipe ``` ##### json ```json { "latencies": { - "mean": 9093653647, - "50th": 2401223400, - "95th": 12553709381, - "99th": 12604629125, - "max": 12604629125 + "mean": 290954696, + "50th": 655359, + "95th": 1871708159, + "99th": 2239758335, + "999th": 2319450111, + "max": 2327768447, + "min": 239872 }, "bytes_in": { - "total": 782040, - "mean": 651.7 + "total": 17478504, + "mean": 5826.168 }, "bytes_out": { "total": 0, "mean": 0 }, - "duration": 9949883921, - "wait": 145082066, - "requests": 1200, - "success": 0.11666666666666667, + "duration": 10000333187, + "wait": 575578, + "requests": 3000, + "success": 0.999, "status_codes": { - "0": 1060, - "200": 140 + "0": 3, + "200": 2997 }, "errors": [ - "Get http://localhost:6060: dial tcp 127.0.0.1:6060: operation timed out" + "Get http://:6060: read tcp 127.0.0.1:6060: connection reset by peer", + "Get http://:6060: net/http: transport closed before response was received", + "Get http://:6060: write tcp 127.0.0.1:6060: broken pipe" ] } ``` @@ -214,7 +215,7 @@ out. Input a different number on the bottom left corner input field to change the moving average window size (in data points). -![Plot](http://i.imgur.com/oi0cgGq.png) +![Plot](http://i.imgur.com/uCdxdaC.png) ## Usage (Library)