From c953313b01d0881f0fe206947b54e0efb401be70 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Sat, 2 Dec 2017 11:29:16 +0000 Subject: [PATCH 1/4] move main report processing benchmarks in one place so they can share code and are easier to run in combination. We take advantage of the code sharing by generalising the report rendering benchmarks to read & merge reports from a dir. --- app/benchmark_internal_test.go | 45 ++++++++++++++++++++++++++++--- report/benchmark_internal_test.go | 40 --------------------------- 2 files changed, 41 insertions(+), 44 deletions(-) delete mode 100644 report/benchmark_internal_test.go diff --git a/app/benchmark_internal_test.go b/app/benchmark_internal_test.go index 40daacbd50..2f7fe21fee 100644 --- a/app/benchmark_internal_test.go +++ b/app/benchmark_internal_test.go @@ -4,6 +4,8 @@ import ( "flag" "net/http" "net/url" + "os" + "path/filepath" "testing" "github.com/weaveworks/scope/render" @@ -12,9 +14,43 @@ import ( ) var ( - benchReportFile = flag.String("bench-report-file", "", "report file to use for benchmarking (relative to this package)") + benchReportPath = flag.String("bench-report-path", "", "report file, or dir with files, to use for benchmarking (relative to this package)") ) +func readReportFiles(path string) ([]report.Report, error) { + reports := []report.Report{} + if err := filepath.Walk(path, + func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + rpt, err := report.MakeFromFile(p) + if err != nil { + return err + } + reports = append(reports, rpt) + return nil + }); err != nil { + return nil, err + } + return reports, nil +} + +func BenchmarkReportUnmarshal(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + b.StartTimer() + if _, err := readReportFiles(*benchReportPath); err != nil { + b.Fatal(err) + } + } +} + func BenchmarkTopologyList(b *testing.B) { benchmarkRender(b, func(report report.Report) { request := &http.Request{ @@ -26,11 +62,12 @@ func BenchmarkTopologyList(b *testing.B) { func benchmarkRender(b *testing.B, f func(report.Report)) { r := fixture.Report - if *benchReportFile != "" { - var err error - if r, err = report.MakeFromFile(*benchReportFile); err != nil { + if *benchReportPath != "" { + reports, err := readReportFiles(*benchReportPath) + if err != nil { b.Fatal(err) } + r = NewSmartMerger().Merge(reports) } b.ReportAllocs() diff --git a/report/benchmark_internal_test.go b/report/benchmark_internal_test.go deleted file mode 100644 index 568372b50c..0000000000 --- a/report/benchmark_internal_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package report - -import ( - "flag" - "os" - "path/filepath" - "testing" -) - -var ( - benchReportPath = flag.String("bench-report-path", "", "report file, or dir with files, to use for benchmarking (relative to this package)") -) - -func BenchmarkReportUnmarshal(b *testing.B) { - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - b.StartTimer() - if err := readReportFiles(*benchReportPath); err != nil { - b.Fatal(err) - } - } -} - -func readReportFiles(path string) error { - return filepath.Walk(path, - func(p string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - if _, err := MakeFromFile(p); err != nil { - return err - } - return nil - }) -} From bc0c8324be8a68910f2059af9290e0c78bdb6023 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Sat, 2 Dec 2017 11:51:16 +0000 Subject: [PATCH 2/4] add report merging benchmark --- app/benchmark_internal_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/benchmark_internal_test.go b/app/benchmark_internal_test.go index 2f7fe21fee..6a58c7af84 100644 --- a/app/benchmark_internal_test.go +++ b/app/benchmark_internal_test.go @@ -51,6 +51,22 @@ func BenchmarkReportUnmarshal(b *testing.B) { } } +func BenchmarkReportMerge(b *testing.B) { + reports, err := readReportFiles(*benchReportPath) + if err != nil { + b.Fatal(err) + } + merger := NewSmartMerger() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + b.StartTimer() + merger.Merge(reports) + } +} + func BenchmarkTopologyList(b *testing.B) { benchmarkRender(b, func(report report.Report) { request := &http.Request{ From acde0ea2948c66c200918607f1254ed2e33fa96e Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Sat, 2 Dec 2017 13:03:53 +0000 Subject: [PATCH 3/4] add some more topology rendering benchmarks --- app/benchmark_internal_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/benchmark_internal_test.go b/app/benchmark_internal_test.go index 6a58c7af84..84b2e1b98c 100644 --- a/app/benchmark_internal_test.go +++ b/app/benchmark_internal_test.go @@ -101,10 +101,22 @@ func BenchmarkTopologyHosts(b *testing.B) { benchmarkOneTopology(b, "hosts") } +func BenchmarkTopologyControllers(b *testing.B) { + benchmarkOneTopology(b, "kube-controllers") +} + +func BenchmarkTopologyPods(b *testing.B) { + benchmarkOneTopology(b, "pods") +} + func BenchmarkTopologyContainers(b *testing.B) { benchmarkOneTopology(b, "containers") } +func BenchmarkTopologyProcesses(b *testing.B) { + benchmarkOneTopology(b, "processes") +} + func benchmarkOneTopology(b *testing.B, topologyID string) { benchmarkRender(b, func(report report.Report) { renderer, filter, err := topologyRegistry.RendererForTopology(topologyID, url.Values{}, report) From 4162b5d7342e1c0815202dcf3d633d48c417f8a0 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Sat, 2 Dec 2017 12:18:56 +0000 Subject: [PATCH 4/4] allocate less memory in LatestMap merging Most maps we merge have the same keys, or at least one set of keys is the subset of the other. Therefore, allocate a result slice capable of holding only the max of number keys, rather than the sum. --- extras/generate_latest_map | 6 +++++- report/latest_map_generated.go | 12 ++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/extras/generate_latest_map b/extras/generate_latest_map index 183c35ac0b..f34b59ffab 100755 --- a/extras/generate_latest_map +++ b/extras/generate_latest_map @@ -83,7 +83,11 @@ function generate_latest_map() { case n.entries == nil: return m } - out := make([]${entry_type}, 0, len(m.entries)+len(n.entries)) + l := len(m.entries) + if len(n.entries) > l { + l = len(n.entries) + } + out := make([]${entry_type}, 0, l) i, j := 0, 0 for i < len(m.entries) { diff --git a/report/latest_map_generated.go b/report/latest_map_generated.go index 52df7b113e..e4e5dd628a 100644 --- a/report/latest_map_generated.go +++ b/report/latest_map_generated.go @@ -51,7 +51,11 @@ func (m StringLatestMap) Merge(n StringLatestMap) StringLatestMap { case n.entries == nil: return m } - out := make([]stringLatestEntry, 0, len(m.entries)+len(n.entries)) + l := len(m.entries) + if len(n.entries) > l { + l = len(n.entries) + } + out := make([]stringLatestEntry, 0, l) i, j := 0, 0 for i < len(m.entries) { @@ -268,7 +272,11 @@ func (m NodeControlDataLatestMap) Merge(n NodeControlDataLatestMap) NodeControlD case n.entries == nil: return m } - out := make([]nodeControlDataLatestEntry, 0, len(m.entries)+len(n.entries)) + l := len(m.entries) + if len(n.entries) > l { + l = len(n.entries) + } + out := make([]nodeControlDataLatestEntry, 0, l) i, j := 0, 0 for i < len(m.entries) {