From 6913d42f272d95004d04d279ba95518ca8405b60 Mon Sep 17 00:00:00 2001 From: Simon Howe Date: Thu, 21 Jan 2016 12:48:50 +0100 Subject: [PATCH 1/5] Adds 0.12.0 changelog --- CHANGELOG.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e902fd97af..aaa2e88588 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,62 @@ +## Release 0.12.0 + +New features and enhancements: +- Details panel redesign + [#752](https://github.com/weaveworks/scope/pull/752) +- k8s: Use service account token by default and improve error logging + [#808](https://github.com/weaveworks/scope/pull/808) +- Probes now use TLS against scope.weave.works by default + [#785](https://github.com/weaveworks/scope/pull/785) +- Gather per-process CPU and memory metrics. + [#767](https://github.com/weaveworks/scope/pull/767) +- Allow dismissing a disconnected terminal w/ \ + [#819](https://github.com/weaveworks/scope/pull/819) +- k8s: Filter out pause as a system container + [#823](https://github.com/weaveworks/scope/pull/823) +- k8s: Render container names from label "io.kubernetes.container.name" + [#810](https://github.com/weaveworks/scope/pull/810) + +Bug fixes: +- General k8s fixups + [#834](https://github.com/weaveworks/scope/pull/834) +- Use argv\[0\] for process name, differentiate scope app and probe. + [#796](https://github.com/weaveworks/scope/pull/796) +- Don't panic if you don't understand the message on the control WS. + [#793](https://github.com/weaveworks/scope/pull/793) +- Highlight a single unconnected node on hover. + [#790](https://github.com/weaveworks/scope/pull/790) +- Fixes to Terminal resizing and key support + [#766](https://github.com/weaveworks/scope/pull/766) + [#780](https://github.com/weaveworks/scope/pull/780) + [#817](https://github.com/weaveworks/scope/pull/817) +- Correctly collapse nodes in the Container Images view when they use non-standard port. + [#824](https://github.com/weaveworks/scope/pull/824) +- Stop scope crashing chrome when we get "long" edges. + [#837](https://github.com/weaveworks/scope/pull/837) +- Fix node controls so they behave independently across nodes + [#797](https://github.com/weaveworks/scope/pull/797) + +Build improvements and cleanup: +- Update to latest tools.git + [#816](https://github.com/weaveworks/scope/pull/816) +- Update to latest go-dockerclient + [#788](https://github.com/weaveworks/scope/pull/788) +- Speed up builds + [#775](https://github.com/weaveworks/scope/pull/775) + [#789](https://github.com/weaveworks/scope/pull/789) +- Speed up tests + [#799](https://github.com/weaveworks/scope/pull/799) + [#807](https://github.com/weaveworks/scope/pull/807) +- Split and move xfer package. + [#794](https://github.com/weaveworks/scope/pull/794) +- Add more tests to procspy + [#751](https://github.com/weaveworks/scope/pull/751) + [#781](https://github.com/weaveworks/scope/pull/781) +- Build example app in container. + [#831](https://github.com/weaveworks/scope/pull/831) +- Various improvements to build & test + [#829](https://github.com/weaveworks/scope/pull/829) + ## Release 0.11.1 Bug fix: From 1d8663029f041f5c1ba8b088ba997526e1d43ff9 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Wed, 27 Jan 2016 14:00:28 -0800 Subject: [PATCH 2/5] Update release notes. --- CHANGELOG.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aaa2e88588..8849478542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,20 +1,20 @@ ## Release 0.12.0 New features and enhancements: -- Details panel redesign +- New, interactive contextual details panel [#752](https://github.com/weaveworks/scope/pull/752) +- Gather per-process CPU and memory metrics + [#767](https://github.com/weaveworks/scope/pull/767) - k8s: Use service account token by default and improve error logging [#808](https://github.com/weaveworks/scope/pull/808) +- k8s: Filter out pause as a system container to declutter view + [#823](https://github.com/weaveworks/scope/pull/823) +- k8s: Render container names from label "io.kubernetes.container.name" + [#810](https://github.com/weaveworks/scope/pull/810) - Probes now use TLS against scope.weave.works by default [#785](https://github.com/weaveworks/scope/pull/785) -- Gather per-process CPU and memory metrics. - [#767](https://github.com/weaveworks/scope/pull/767) - Allow dismissing a disconnected terminal w/ \ [#819](https://github.com/weaveworks/scope/pull/819) -- k8s: Filter out pause as a system container - [#823](https://github.com/weaveworks/scope/pull/823) -- k8s: Render container names from label "io.kubernetes.container.name" - [#810](https://github.com/weaveworks/scope/pull/810) Bug fixes: - General k8s fixups From d0d6cac4485b8189919f2acaf9413f83a601b766 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Thu, 28 Jan 2016 14:48:47 -0800 Subject: [PATCH 3/5] Use different key to count IPs, so we don't clash with containers. Fixes #870 --- render/mapping.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/render/mapping.go b/render/mapping.go index 3c4f0ad526..ed9147533f 100644 --- a/render/mapping.go +++ b/render/mapping.go @@ -24,6 +24,7 @@ const ( TheInternetMajor = "The Internet" containersKey = "containers" + ipsKey = "ips" podsKey = "pods" processesKey = "processes" servicesKey = "services" @@ -330,7 +331,7 @@ func MapContainer2IP(m RenderableNode, _ report.Networks) RenderableNodes { } id := report.MakeScopedEndpointNodeID(scope, addr, "") node := NewRenderableNodeWith(id, "", "", "", m) - node.Counters = node.Counters.Add(containersKey, 1) + node.Counters = node.Counters.Add(ipsKey, 1) result[id] = node } } @@ -343,7 +344,7 @@ func MapContainer2IP(m RenderableNode, _ report.Networks) RenderableNodes { ip, port := mapping[1], mapping[2] id := report.MakeScopedEndpointNodeID("", ip, port) node := NewRenderableNodeWith(id, "", "", "", m.WithParents(report.EmptySets)) - node.Counters = node.Counters.Add(containersKey, 1) + node.Counters = node.Counters.Add(ipsKey, 1) result[id] = node } } @@ -357,7 +358,7 @@ func MapContainer2IP(m RenderableNode, _ report.Networks) RenderableNodes { func MapIP2Container(n RenderableNode, _ report.Networks) RenderableNodes { // If an IP is shared between multiple containers, we can't // reliably attribute an connection based on its IP - if count, _ := n.Node.Counters.Lookup(containersKey); count > 1 { + if count, _ := n.Node.Counters.Lookup(ipsKey); count > 1 { return RenderableNodes{} } From 34447fd502341904069e496fbe9b2d776da61687 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Fri, 29 Jan 2016 10:45:20 -0800 Subject: [PATCH 4/5] Close response body for docker stats calls. --- probe/docker/container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/probe/docker/container.go b/probe/docker/container.go index cd33e57916..308dc08c68 100644 --- a/probe/docker/container.go +++ b/probe/docker/container.go @@ -185,6 +185,7 @@ func (c *container) StartGatheringStats() error { log.Printf("docker container: %v", err) return } + defer resp.Body.Close() c.Lock() c.statsConn = conn @@ -201,7 +202,6 @@ func (c *container) StartGatheringStats() error { stats := &docker.Stats{} decoder := json.NewDecoder(resp.Body) - for err := decoder.Decode(&stats); err != io.EOF; err = decoder.Decode(&stats) { if err != nil { log.Printf("docker container: error reading event, did container stop? %v", err) From 68a3748f01fed71fe8ce7dedd8e3d67a372a0223 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Fri, 29 Jan 2016 11:35:29 -0800 Subject: [PATCH 5/5] Don't allocate lots of docker.Stats structs, do some copying instead --- Makefile | 1 + probe/docker/container.go | 66 +++++++++++++++------------------- probe/docker/container_test.go | 1 - 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index 5168008430..a2759992d2 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,7 @@ $(SCOPE_EXE) $(RUNSVINIT) lint tests shell: $(SCOPE_BACKEND_BUILD_UPTODATE) $(SUDO) docker run $(RM) $(RUN_FLAGS) \ -v $(shell pwd):/go/src/github.com/weaveworks/scope \ -v $(shell pwd)/.pkg:/go/pkg \ + --net=host \ -e GOARCH -e GOOS -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL \ -e CIRCLE_NODE_INDEX -e COVERDIR -e SLOW \ $(SCOPE_BACKEND_BUILD_IMAGE) SCOPE_VERSION=$(SCOPE_VERSION) GO_BUILD_INSTALL_DEPS=$(GO_BUILD_INSTALL_DEPS) $@ diff --git a/probe/docker/container.go b/probe/docker/container.go index 308dc08c68..8e8707f46c 100644 --- a/probe/docker/container.go +++ b/probe/docker/container.go @@ -10,7 +10,6 @@ import ( "net/http" "net/http/httputil" "net/url" - "strconv" "strings" "sync" "time" @@ -95,8 +94,9 @@ type container struct { sync.RWMutex container *docker.Container statsConn ClientConn - latestStats *docker.Stats - pendingStats []*docker.Stats + latestStats docker.Stats + pendingStats [20]docker.Stats + numPending int } // NewContainer creates a new Container @@ -197,10 +197,9 @@ func (c *container) StartGatheringStats() error { log.Printf("docker container: stopped collecting stats for %s", c.container.ID) c.statsConn = nil - c.latestStats = nil }() - stats := &docker.Stats{} + var stats docker.Stats decoder := json.NewDecoder(resp.Body) for err := decoder.Decode(&stats); err != io.EOF; err = decoder.Decode(&stats) { if err != nil { @@ -209,11 +208,16 @@ func (c *container) StartGatheringStats() error { } c.Lock() - c.latestStats = stats - c.pendingStats = append(c.pendingStats, stats) + if c.numPending >= len(c.pendingStats) { + log.Printf("docker container: dropping stats.") + } else { + c.latestStats = stats + c.pendingStats[c.numPending] = stats + c.numPending++ + } c.Unlock() - stats = &docker.Stats{} + stats = docker.Stats{} } }() @@ -230,7 +234,6 @@ func (c *container) StopGatheringStats() { c.statsConn.Close() c.statsConn = nil - c.latestStats = nil return } @@ -259,22 +262,22 @@ func (c *container) ports(localAddrs []net.IP) report.StringSet { return report.MakeStringSet(ports...) } -func (c *container) memoryUsageMetric() report.Metric { +func (c *container) memoryUsageMetric(stats []docker.Stats) report.Metric { result := report.MakeMetric() - for _, s := range c.pendingStats { + for _, s := range stats { result = result.Add(s.Read, float64(s.MemoryStats.Usage)) } return result } -func (c *container) cpuPercentMetric() report.Metric { +func (c *container) cpuPercentMetric(stats []docker.Stats) report.Metric { result := report.MakeMetric() - if len(c.pendingStats) < 2 { + if len(stats) < 2 { return result } - previous := c.pendingStats[0] - for _, s := range c.pendingStats[1:] { + previous := stats[0] + for _, s := range stats[1:] { // Copies from docker/api/client/stats.go#L205 cpuDelta := float64(s.CPUStats.CPUUsage.TotalUsage - previous.CPUStats.CPUUsage.TotalUsage) systemDelta := float64(s.CPUStats.SystemCPUUsage - previous.CPUStats.SystemCPUUsage) @@ -293,15 +296,18 @@ func (c *container) cpuPercentMetric() report.Metric { } func (c *container) metrics() report.Metrics { + if c.numPending == 0 { + return report.Metrics{} + } + pendingStats := c.pendingStats[:c.numPending] result := report.Metrics{ - MemoryUsage: c.memoryUsageMetric(), - CPUTotalUsage: c.cpuPercentMetric(), + MemoryUsage: c.memoryUsageMetric(pendingStats), + CPUTotalUsage: c.cpuPercentMetric(pendingStats), } - // Keep the latest report to help with relative metric reporting. - if len(c.pendingStats) > 0 { - c.pendingStats = c.pendingStats[len(c.pendingStats)-1:] - } + // leave one stat to help with relative metrics + c.pendingStats[0] = c.pendingStats[c.numPending-1] + c.numPending = 1 return result } @@ -348,23 +354,7 @@ func (c *container) GetNode(hostID string, localAddrs []net.IP) report.Node { } result = AddLabels(result, c.container.Config.Labels) - - if c.latestStats == nil { - return result - } - - result = result.WithLatests(map[string]string{ - MemoryMaxUsage: strconv.FormatUint(c.latestStats.MemoryStats.MaxUsage, 10), - MemoryUsage: strconv.FormatUint(c.latestStats.MemoryStats.Usage, 10), - MemoryFailcnt: strconv.FormatUint(c.latestStats.MemoryStats.Failcnt, 10), - MemoryLimit: strconv.FormatUint(c.latestStats.MemoryStats.Limit, 10), - - // CPUPercpuUsage: strconv.FormatUint(stats.CPUStats.CPUUsage.PercpuUsage, 10), - CPUUsageInUsermode: strconv.FormatUint(c.latestStats.CPUStats.CPUUsage.UsageInUsermode, 10), - CPUTotalUsage: strconv.FormatUint(c.latestStats.CPUStats.CPUUsage.TotalUsage, 10), - CPUUsageInKernelmode: strconv.FormatUint(c.latestStats.CPUStats.CPUUsage.UsageInKernelmode, 10), - CPUSystemCPUUsage: strconv.FormatUint(c.latestStats.CPUStats.SystemCPUUsage, 10), - }).WithMetrics(c.metrics()) + result = result.WithMetrics(c.metrics()) return result } diff --git a/probe/docker/container_test.go b/probe/docker/container_test.go index 9308203917..6104569689 100644 --- a/probe/docker/container_test.go +++ b/probe/docker/container_test.go @@ -79,7 +79,6 @@ func TestContainer(t *testing.T) { "docker_image_id": "baz", "docker_label_foo1": "bar1", "docker_label_foo2": "bar2", - "docker_memory_usage": "12345", }).WithSets(report.EmptySets. Add("docker_container_ports", report.MakeStringSet("1.2.3.4:80->80/tcp", "81/tcp")). Add("docker_container_ips", report.MakeStringSet("1.2.3.4")).