Merge pull request #2094 from weaveworks/maybe-old-master

Fix force-push to master

extras/dialer/.gitignore

@@ -0,0 +1 @@
+bin/

extras/dialer/Dockerfile

@@ -0,0 +1,2 @@
+FROM golang:1.7
+ADD ./bin/dialer /go/bin

extras/dialer/README.md

@@ -0,0 +1,46 @@
+## Overview
+
+The dialer scripts can be used to test Scope with a high number of
+containers and connections.
+
+The dialer server is a TCP server that holds incoming connections
+forever.
+
+The dialer client is a TCP client that opens n connection to a server
+and holds them forever.
+
+The `listener` script starts a dialer server and prints its address for
+usage with the `dialer` script. The `dialer` scripts starts up to n
+(default 50) client containers, each opening a random (1-20) number of
+connections.
+
+`time-scope-probe` then can be used to measure the scheduled time
+(utime + stime) of the scope-probe process on the host. The results
+can be used to compare performance under different scenarios/branches.
+
+## Usage example
+
+```
+# Start a listener
+./tools/dialer/listener
+Listening on :8082
+IP addr + port: 172.17.0.2:8082
+
+
+# Start the dialer script with a maximum of 10 dialer containers
+# (default 50)
+./tools/dialer/dialer 172.17.0.2:8082 10
+
+
+# Start time-scope-probe to measure the scheduled time of scope-probe
+# every 3 seconds (default 10 seconds) for 3 times (default 60 times)
+sudo ./tools/dialer/time-scope-probe 3 3
+...
+```
+
+## Build dialer container
+
+```
+go build -o bin/dialer
+docker build -t dialer .
+```

extras/dialer/dialer

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -eu
+if [ $# -lt 1 ]; then
+  echo "Usage: $0 <ip:port> [<max_dialer>]" >&2
+  exit 1
+fi
+
+readonly addr=$1
+readonly max_dialer=${2:-50}
+
+dialer=()
+trap 'echo -n "stopping ... "; for c in "${dialer[@]}"; do docker rm -f "$c" >/dev/null; done; echo "done"' EXIT
+
+while true; do
+  rand=$(( ( RANDOM % max_dialer )  + 1 ))
+  dialer+=("$(docker run -d dialer /go/bin/dialer connect "$addr" "$rand")")
+
+  if [ ${#dialer[@]} -gt "$max_dialer" ]; then
+    container=${dialer[$rand]}
+    docker rm -f "$container" >/dev/null &
+    unset dialer[$rand]
+    dialer=("${dialer[@]}")
+  fi
+
+  sleep $(( rand % 3 ))
+done

extras/dialer/dialer.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+)
+
+func connect(url string, numConn int) {
+	fmt.Printf("Establishing %d TCP connections to %s\n", numConn, url)
+	for x := 0; x < numConn; x++ {
+		_, err := net.Dial("tcp", url)
+
+		if err != nil {
+			fmt.Printf("Error: %v\n", err)
+			os.Exit(1)
+		}
+	}
+
+	// wait forever
+	select {}
+}
+
+func listen(url string) {
+	l, err := net.Listen("tcp", url)
+	if err != nil {
+		fmt.Println("Error listening:", err.Error())
+		os.Exit(1)
+	}
+	defer l.Close()
+	fmt.Println("Listening on " + url)
+	for {
+		_, err := l.Accept()
+		if err != nil {
+			fmt.Println("Error accepting: ", err.Error())
+			os.Exit(1)
+		}
+	}
+
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Println("Not enough arguments")
+		os.Exit(1)
+	}
+
+	verb := os.Args[1]
+
+	if verb == "connect" {
+		if len(os.Args) != 4 {
+			fmt.Println("Not enough arguments")
+			os.Exit(1)
+		}
+
+		url := os.Args[2]
+		numConn, err := strconv.Atoi(os.Args[3])
+		if err != nil {
+			fmt.Printf("Error with second argument\n")
+			os.Exit(1)
+		}
+
+		connect(url, numConn)
+	}
+	if verb == "listen" {
+		if len(os.Args) != 3 {
+			fmt.Println("Not enough arguments")
+			os.Exit(1)
+		}
+
+		port := os.Args[2]
+		listen(":" + port)
+	}
+
+}

extras/dialer/listener

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -eu
+readonly port=${1:-8082}
+(
+sleep 5 # wait for dialerserver to start
+echo "IP addr + port: $(docker inspect -f '{{ .NetworkSettings.IPAddress }}' dialerserver):$port"
+) &
+docker run --rm --name dialerserver dialer /go/bin/dialer listen "$port"

extras/dialer/time-scope-probe

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -eu
+if [ $EUID -ne 0 ]; then
+  echo "You must be root!" >&2
+  exit 1
+fi
+
+readonly interval_num=${1:-60}
+readonly interval_sleep=${2:-10}
+
+TIME_U1=0
+TIME_K1=0
+TIME_T1=0
+TIME_U2=0
+TIME_K2=0
+TIME_T2=0
+
+if [ "$(pidof scope-probe)" == "" ]; then
+  echo "No scope-probe process running - aborting" >&2
+  exit 1
+fi
+
+for ((i=0; i<"$interval_num"; i++)); do
+  sleep "$interval_sleep"
+  TIME_U1=$TIME_U2
+  TIME_K1=$TIME_K2
+  TIME_T1=$TIME_T2
+
+  TIME_U2=$(gawk '{print $14"*10"}'<"/proc/$(pidof scope-probe)/stat" | bc)
+  TIME_K2=$(gawk '{print $15"*10"}'<"/proc/$(pidof scope-probe)/stat" | bc)
+  TIME_T2=$(( TIME_U2 + TIME_K2 ))
+  echo "utime $(( TIME_U2 - TIME_U1)) + stime $(( TIME_K2 - TIME_K1 )) = $(( TIME_T2 - TIME_T1 ))"
+done

probe/awsecs/client.go

@@ -73,7 +73,7 @@ func (c ecsClient) getServices() map[string]*ecs.Service {
 				}
 
 				for _, failure := range resp.Failures {
-					log.Warnf("Failed to describe ECS service %s, ECS service report may be incomplete: %s", failure.Arn, failure.Reason)
+					log.Warnf("Failed to describe ECS service %s, ECS service report may be incomplete: %s", *failure.Arn, *failure.Reason)
 				}
 
 				lock.Lock()
@@ -110,7 +110,7 @@ func (c ecsClient) getTasks(taskArns []string) (map[string]*ecs.Task, error) {
 	}
 
 	for _, failure := range resp.Failures {
-		log.Warnf("Failed to describe ECS task %s, ECS service report may be incomplete: %s", failure.Arn, failure.Reason)
+		log.Warnf("Failed to describe ECS task %s, ECS service report may be incomplete: %s", *failure.Arn, *failure.Reason)
 	}
 
 	results := make(map[string]*ecs.Task, len(resp.Tasks))

prog/app.go

@@ -270,7 +270,10 @@ func appMain(flags appFlags) {
 
 	handler := router(collector, controlRouter, pipeRouter, flags.externalUI)
 	if flags.logHTTP {
-		handler = middleware.LogFailed.Wrap(handler)
+		handler = middleware.Log{
+			LogRequestHeaders: flags.logHTTPHeaders,
+			LogSuccess:        false,
+		}.Wrap(handler)
 	}
 
 	server := &graceful.Server{

prog/main.go

@@ -112,12 +112,13 @@ type probeFlags struct {
 }
 
 type appFlags struct {
-	window      time.Duration
-	listen      string
-	stopTimeout time.Duration
-	logLevel    string
-	logPrefix   string
-	logHTTP     bool
+	window         time.Duration
+	listen         string
+	stopTimeout    time.Duration
+	logLevel       string
+	logPrefix      string
+	logHTTP        bool
+	logHTTPHeaders bool
 
 	weaveEnabled   bool
 	weaveAddr      string
@@ -298,6 +299,7 @@ func main() {
 	flag.StringVar(&flags.app.logLevel, "app.log.level", "info", "logging threshold level: debug|info|warn|error|fatal|panic")
 	flag.StringVar(&flags.app.logPrefix, "app.log.prefix", "<app>", "prefix for each log line")
 	flag.BoolVar(&flags.app.logHTTP, "app.log.http", false, "Log individual HTTP requests")
+	flag.BoolVar(&flags.app.logHTTPHeaders, "app.log.httpHeaders", false, "Log HTTP headers. Needs app.log.http to be enabled.")
 
 	flag.StringVar(&flags.app.weaveAddr, "app.weave.addr", app.DefaultWeaveURL, "Address on which to contact WeaveDNS")
 	flag.StringVar(&flags.app.weaveHostname, "app.weave.hostname", app.DefaultHostname, "Hostname to advertise in WeaveDNS")