Rebase mdap-rc1 with upstream updates

CHANGELOG.md

@@ -1,3 +1,20 @@
+## Release 1.11.4
+
+This release contains a few fixes, one of which should improve
+resource usage on hosts that have a lot of TCP connections.
+
+- Improve eBPF connection tracker to reduce the number of times it
+  restarts and falls back to a less efficient mechanism.
+	[#3653](https://github.com/weaveworks/scope/pull/3653)
+- Add reporter name to probe error logs. Thanks to @princerachit
+	[#3363](https://github.com/weaveworks/scope/pull/3363)
+- Defer metrics registration until we need it
+	[#3605](https://github.com/weaveworks/scope/pull/3605)
+- Remove unused metric SpyDuration
+	[#3646](https://github.com/weaveworks/scope/pull/3646)
+- Remove quay.io from release script
+	[#3657](https://github.com/weaveworks/scope/pull/3657)
+
 ## Release 1.11.3
 
 This is a bugfix release, which should improve some cases where

LICENSE

@@ -176,7 +176,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2014-2018 Weaveworks Ltd.
+   Copyright 2014-2019 Weaveworks Ltd.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -42,15 +42,16 @@ If you would like to see your name added, let us know on Slack, or send a PR ple
 
 ## <a name="getting-started"></a>Getting Started
 
+**Ensure your computer is behind a firewall that blocks port 4040** then,
+
 ```console
 sudo curl -L git.io/scope -o /usr/local/bin/scope
 sudo chmod a+x /usr/local/bin/scope
 scope launch
 ```
 
 This script downloads and runs a recent Scope image from Docker Hub.
-Now, open your web browser to **http://localhost:4040**. (If you're using
-boot2docker, replace localhost with the output of `boot2docker ip`.)
+Now, open your web browser to **http://localhost:4040**.
 
 For instructions on installing Scope on [Kubernetes](https://www.weave.works/docs/scope/latest/installing/#k8s), [DCOS](https://www.weave.works/docs/scope/latest/installing/#dcos), or [ECS](https://www.weave.works/docs/scope/latest/installing/#ecs), see [the docs](https://www.weave.works/docs/scope/latest/introducing/).
 

app/multitenant/aws_collector.go

@@ -86,7 +86,7 @@ var (
 	}, []string{"method", "status_code"})
 )
 
-func init() {
+func registerAWSCollectorMetrics() {
 	prometheus.MustRegister(dynamoRequestDuration)
 	prometheus.MustRegister(dynamoConsumedCapacity)
 	prometheus.MustRegister(dynamoValueSize)
@@ -98,6 +98,8 @@ func init() {
 	prometheus.MustRegister(natsRequests)
 }
 
+var registerAWSCollectorMetricsOnce sync.Once
+
 // AWSCollector is a Collector which can also CreateTables
 type AWSCollector interface {
 	app.Collector
@@ -149,6 +151,7 @@ type watchKey struct {
 // NewAWSCollector the elastic reaper of souls
 // https://github.com/aws/aws-sdk-go/wiki/common-examples
 func NewAWSCollector(config AWSCollectorConfig) (AWSCollector, error) {
+	registerAWSCollectorMetricsOnce.Do(registerAWSCollectorMetrics)
 	var nc *nats.Conn
 	if config.NatsHost != "" {
 		var err error

app/multitenant/memcache_client.go

@@ -39,12 +39,14 @@ var (
 	}, []string{"method", "status_code"})
 )
 
-func init() {
+func registerMemcacheClientMetrics() {
 	prometheus.MustRegister(memcacheRequests)
 	prometheus.MustRegister(memcacheHits)
 	prometheus.MustRegister(memcacheRequestDuration)
 }
 
+var registerMemcacheClientMetricsOnce sync.Once
+
 // MemcacheClient is a memcache client that gets its server list from SRV
 // records, and periodically updates that ServerList.
 type MemcacheClient struct {
@@ -72,6 +74,7 @@ type MemcacheConfig struct {
 // NewMemcacheClient creates a new MemcacheClient that gets its server list
 // from SRV and updates the server list on a regular basis.
 func NewMemcacheClient(config MemcacheConfig) *MemcacheClient {
+	registerMemcacheClientMetricsOnce.Do(registerMemcacheClientMetrics)
 	var servers memcache.ServerList
 	client := memcache.NewFromSelector(&servers)
 	client.Timeout = config.Timeout

app/multitenant/s3_client.go

@@ -2,6 +2,7 @@ package multitenant
 
 import (
 	"bytes"
+	"sync"
 
 	"context"
 	"github.com/aws/aws-sdk-go/aws"
@@ -28,12 +29,15 @@ type S3Store struct {
 	bucketName string
 }
 
-func init() {
+func registerS3ClientMetrics() {
 	prometheus.MustRegister(s3RequestDuration)
 }
 
+var registerS3ClientMetricsOnce sync.Once
+
 // NewS3Client creates a new S3 client.
 func NewS3Client(config *aws.Config, bucketName string) S3Store {
+	registerS3ClientMetricsOnce.Do(registerS3ClientMetrics)
 	return S3Store{
 		s3:         s3.New(session.New(config)),
 		bucketName: bucketName,

app/multitenant/sqs_control_router.go

@@ -30,10 +30,12 @@ var (
 	}, []string{"method", "status_code"})
 )
 
-func init() {
+func registerSQSMetrics() {
 	prometheus.MustRegister(sqsRequestDuration)
 }
 
+var registerSQSMetricsOnce sync.Once
+
 // sqsControlRouter:
 // Creates a queue for every probe that connects to it, and a queue for
 // responses back to it.  When it receives a request, posts it to the
@@ -64,6 +66,7 @@ type sqsResponseMessage struct {
 
 // NewSQSControlRouter the harbinger of death
 func NewSQSControlRouter(config *aws.Config, userIDer UserIDer, prefix string, rpcTimeout time.Duration) app.ControlRouter {
+	registerSQSMetricsOnce.Do(registerSQSMetrics)
 	result := &sqsControlRouter{
 		service:          sqs.New(session.New(config)),
 		responseQueueURL: nil,

bin/release

@@ -235,21 +235,19 @@ publish() {
 }
 
 tag_images() {
-    echo "== Tagging images for docker hub and quay.io as user $DOCKERHUB_USER"
+    echo "== Tagging images for docker hub as user $DOCKERHUB_USER"
     for IMAGE in "scope" "cloud-agent"; do
         $SUDO docker tag "$DOCKERHUB_USER/$IMAGE" "$DOCKERHUB_USER/$IMAGE:$VERSION"
         $SUDO docker tag "$DOCKERHUB_USER/$IMAGE:$VERSION" "$DOCKERHUB_USER/$IMAGE:latest_release"
-        $SUDO docker tag "$DOCKERHUB_USER/$IMAGE:$VERSION" "quay.io/$DOCKERHUB_USER/$IMAGE:$VERSION"
     done
     echo "** Docker images tagged"
 }
 
 push_images() {
-    echo "== Pushing images on docker hub and quay.io as user $DOCKERHUB_USER"
+    echo "== Pushing images on docker hub as user $DOCKERHUB_USER"
     for IMAGE in "scope" "cloud-agent"; do
         $SUDO docker push "$DOCKERHUB_USER/$IMAGE:$VERSION"
         $SUDO docker push "$DOCKERHUB_USER/$IMAGE:latest_release"
-        $SUDO docker push "quay.io/$DOCKERHUB_USER/$IMAGE:$VERSION"
     done
     echo "** Docker images pushed"
 }

examples/cri/deploy.yaml

@@ -25,7 +25,7 @@ spec:
           args:
             - '--no-probe'
           env: []
-          image: weaveworks/scope:1.11.3
+          image: weaveworks/scope:1.11.4
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 4040

examples/cri/ds.yaml

@@ -34,7 +34,7 @@ spec:
                 fieldRef:
                   apiVersion: v1
                   fieldPath: spec.nodeName
-          image: weaveworks/scope:1.11.3
+          image: weaveworks/scope:1.11.4
           imagePullPolicy: IfNotPresent
           securityContext:
             privileged: true

examples/docker/docker-compose-probe-v1.yml

@@ -1,5 +1,5 @@
 probe:
-  image: weaveworks/scope:1.11.3
+  image: weaveworks/scope:1.11.4
   net: "host"
   pid: "host"
   privileged: true

examples/docker/docker-compose-probe-v2.yml

@@ -1,7 +1,7 @@
 version: '2'
 services:
   probe:
-    image: weaveworks/scope:1.11.3
+    image: weaveworks/scope:1.11.4
     network_mode: "host"
     pid: "host"
     privileged: true

examples/k8s/deploy.yaml

@@ -28,7 +28,7 @@ spec:
           args:
             - '--no-probe'
           env: []
-          image: weaveworks/scope:1.11.3
+          image: weaveworks/scope:1.11.4
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 4040

examples/k8s/ds.yaml

@@ -31,7 +31,7 @@ spec:
             - '--probe.docker.bridge=docker0'
             - '--probe.docker=true'
             - 'weave-scope-app.weave.svc.cluster.local.:80'
-          image: weaveworks/scope:1.11.3
+          image: weaveworks/scope:1.11.4
           imagePullPolicy: IfNotPresent
           resources:
             requests:

examples/k8s/probe-deploy.yaml

@@ -35,7 +35,7 @@ spec:
             - 'weave-scope-app.weave.svc.cluster.local.:80'
           command:
             - /home/weave/scope
-          image: 'docker.io/weaveworks/scope:1.11.3'
+          image: 'docker.io/weaveworks/scope:1.11.4'
           imagePullPolicy: IfNotPresent
           resources:
             requests:

examples/mesos/minimesos.json

@@ -12,7 +12,7 @@
   "container": {
     "type": "DOCKER",
     "docker": {
-      "image": "weaveworks/scope:1.11.3",
+      "image": "weaveworks/scope:1.11.4",
       "network": "HOST",
       "privileged": true,
       "parameters": [

probe/probe.go

@@ -156,7 +156,7 @@ func (p *Probe) tick() {
 		err := ticker.Tick()
 		metrics.MeasureSince([]string{ticker.Name(), "ticker"}, t)
 		if err != nil {
-			log.Errorf("error doing ticker: %v", err)
+			log.Errorf("Error doing ticker: %v", err)
 		}
 	}
 }
@@ -173,7 +173,7 @@ func (p *Probe) report() report.Report {
 			}
 			metrics.MeasureSince([]string{rep.Name(), "reporter"}, t)
 			if err != nil {
-				log.Errorf("error generating report: %v", err)
+				log.Errorf("Error generating %s report: %v", rep.Name(), err)
 				newReport = report.MakeReport() // empty is OK to merge
 			}
 			reports <- newReport
@@ -198,7 +198,7 @@ func (p *Probe) tag(r report.Report) report.Report {
 		}
 		metrics.MeasureSince([]string{tagger.Name(), "tagger"}, t)
 		if err != nil {
-			log.Errorf("error applying tagger: %v", err)
+			log.Errorf("Error applying tagger: %v", err)
 		}
 	}
 	return r
@@ -227,7 +227,7 @@ ForLoop:
 		})
 	}
 	if err := p.publisher.Publish(rpt); err != nil {
-		log.Infof("publish: %v", err)
+		log.Infof("Publish: %v", err)
 	}
 }
 

prog/app.go

@@ -10,6 +10,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/goji/httpauth"
@@ -46,11 +47,13 @@ var (
 	}, []string{"method", "route", "status_code", "ws"})
 )
 
-func init() {
+func registerAppMetrics() {
 	prometheus.MustRegister(requestDuration)
 	billing.MustRegisterMetrics()
 }
 
+var registerAppMetricsOnce sync.Once
+
 // Router creates the mux for all the various app components.
 func router(collector app.Collector, controlRouter app.ControlRouter, pipeRouter app.PipeRouter, externalUI bool, capabilities map[string]bool, metricsGraphURL string) http.Handler {
 	router := mux.NewRouter().SkipClean(true)
@@ -208,6 +211,8 @@ func appMain(flags appFlags) {
 	setLogFormatter(flags.logPrefix)
 	runtime.SetBlockProfileRate(flags.blockProfileRate)
 
+	registerAppMetricsOnce.Do(registerAppMetrics)
+
 	traceCloser := tracing.NewFromEnv(fmt.Sprintf("scope-%s", flags.serviceName))
 	defer traceCloser.Close()
 

site/installing.md

@@ -112,7 +112,7 @@ After it’s been launched, open your browser to `http://localhost:4040`.
 **Docker Compose Format Version 1:**
 
     scope:
-      image: weaveworks/scope:1.11.3
+      image: weaveworks/scope:1.11.4
       net: "host"
       pid: "host"
       privileged: true
@@ -128,7 +128,7 @@ After it’s been launched, open your browser to `http://localhost:4040`.
     version: '2'
     services:
       scope:
-        image: weaveworks/scope:1.11.3
+        image: weaveworks/scope:1.11.4
         network_mode: "host"
         pid: "host"
         privileged: true
@@ -168,6 +168,11 @@ Allowable parameters for the launcher URL:
 
 The URL is: http://localhost:4040.
 
+>**Note:** Do not expose the Scope service to the Internet, e.g. by
+   changing the type to NodePort or LoadBalancer. Scope allows anyone
+   with access to the user interface control over your hosts and
+   containers.
+
 ### Kubernetes (local clone)
 
 A simple way to get Scope running in a Kubernetes setting is to

site/plugins.md

@@ -34,7 +34,7 @@ Official Weave Scope plugins can be found at [Weaveworks Plugins](https://github
 	* Number of HTTP requests per seconds.
 	* Number of HTTP responses code per second (per code).
 
->**Note:** The HTTP Statistics plugin requires a [recent kernel version with ebpf support](https://github.com/iovisor/bcc/blob/master/INSTALL.md#kernel-configuration) and it will not compile on [dlite](https://github.com/nlf/dlite) or on boot2docker hosts.
+>**Note:** The HTTP Statistics plugin requires a [recent kernel version with ebpf support](https://github.com/iovisor/bcc/blob/master/INSTALL.md#kernel-configuration) and it will not compile on [dlite](https://github.com/nlf/dlite) hosts.
 
 * [Traffic Control](https://github.com/weaveworks-plugins/scope-traffic-control): This plugin allows you to modify latency and packet loss for a specific container via controls from the container's detailed view in the Scope user interface.