Work on convienience

* Make workers retry connection to server until work is done
* Implement message from server to worker if all tests are finished (and workers can shutdown)
* Implement skipSSLverify to connect to untrusted S3 endpoints
* Change worker names in k8s to gosbench-worker
* Log in debug mode on Kubernetes

common/configFile.go

@@ -21,11 +21,12 @@ const (
 
 // S3Configuration contains all information to connect to a certain S3 endpoint
 type S3Configuration struct {
-	AccessKey string        `yaml:"access_key"`
-	SecretKey string        `yaml:"secret_key"`
-	Region    string        `yaml:"region"`
-	Endpoint  string        `yaml:"endpoint"`
-	Timeout   time.Duration `yaml:"timeout"`
+	AccessKey     string        `yaml:"access_key"`
+	SecretKey     string        `yaml:"secret_key"`
+	Region        string        `yaml:"region"`
+	Endpoint      string        `yaml:"endpoint"`
+	Timeout       time.Duration `yaml:"timeout"`
+	SkipSSLVerify bool          `yaml:"skipSSLverify"`
 }
 
 // GrafanaConfiguration contains all information necessary to add annotations

examples/example_config.yaml

@@ -4,15 +4,18 @@ s3_config:
   - access_key: abc
     secret_key: as
     region: eu-central-1
-    endpoint:
+    endpoint: https://my.rgw.endpoint:8080
+    skipSSLverify: false
   - access_key: def
     secret_key: as
     region: eu-central-2
-    endpoint:
+    endpoint: https://my.rgw.endpoint:8080
+    skipSSLverify: false
   - access_key: ghi
     secret_key: as
     region: eu-central-3
-    endpoint:
+    endpoint: https://my.rgw.endpoint:8080
+    skipSSLverify: false
 
 # For generating annotations when we start/stop testcases
 # https://grafana.com/docs/http_api/annotations/#create-annotation

k8s/gosbench.yaml

@@ -4,18 +4,11 @@ kind: ConfigMap
 data:
   config.yml: |-
     s3_config:
-      - access_key: abc
-        secret_key: as
+      - access_key: 8MKI...
+        secret_key: Y7quDBT...
         region: eu-central-1
-        endpoint:
-      - access_key: def
-        secret_key: as
-        region: eu-central-2
-        endpoint:
-      - access_key: ghi
-        secret_key: as
-        region: eu-central-3
-        endpoint:
+        endpoint: https://172.30.196.58:443
+        skipSSLverify: true
 
     # For generating annotations when we start/stop testcases
     # https://grafana.com/docs/http_api/annotations/#create-annotation
@@ -71,7 +64,7 @@ metadata:
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: worker1
+  name: gosbench-worker1
   labels:
     app: gosbench-worker1
 spec:
@@ -81,17 +74,18 @@ spec:
         app: gosbench
     spec:
       containers:
-        - name: worker
-          image: quay.io/mulbc/goroom-worker
-          command: ['./main', '-s', 'gosbench-server:2000']
+        - name: gosbench-worker
+          image: quay.io/mulbc/goroom-worker:dev4
+          imagePullPolicy: Always
+          command: ['./main', '-d', '-s', 'gosbench-server:2000']
           ports:
             - containerPort: 8888
       restartPolicy: Never
 ---
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: worker2
+  name: gosbench-worker2
   labels:
     app: gosbench-worker2
 spec:
@@ -101,9 +95,10 @@ spec:
         app: gosbench
     spec:
       containers:
-        - name: worker
-          image: quay.io/mulbc/goroom-worker
-          command: ['./main', '-s', 'gosbench-server:2000']
+        - name: gosbench-worker
+          image: quay.io/mulbc/goroom-worker:dev4
+          imagePullPolicy: Always
+          command: ['./main', '-d', '-s', 'gosbench-server:2000']
           ports:
             - containerPort: 8888
       restartPolicy: Never
@@ -126,7 +121,8 @@ spec:
     spec:
       containers:
         - name: server
-          image: quay.io/mulbc/goroom-server
+          image: quay.io/mulbc/goroom-server:dev4
+          imagePullPolicy: Always
           command: ['./main', '-c', '/app/config/config.yml']
           ports:
             - containerPort: 2000
@@ -164,7 +160,7 @@ spec:
     - port: 8888
       targetPort: 8888
   selector:
-    job-name: worker1
+    job-name: gosbench-worker1
 ---
 apiVersion: v1
 kind: Service
@@ -178,5 +174,5 @@ spec:
     - port: 8888
       targetPort: 8888
   selector:
-    job-name: worker2
+    job-name: gosbench-worker2
 ...

server/main.go

@@ -7,7 +7,6 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"net"
-	"os"
 	"time"
 
 	"github.com/mulbc/gosbench/common"
@@ -143,8 +142,10 @@ func scheduleTests(config common.Testconf) {
 		log.WithField("test", testNumber).Infof("GRAFANA: ?from=%d&to=%d", startTime, stopTime)
 	}
 	log.Info("All performance tests finished")
-	time.Sleep(30 * time.Second)
-	os.Exit(0)
+	for {
+		workerConnection := <-readyWorkers
+		shutdownWorker(workerConnection)
+	}
 }
 
 func executeTestOnWorker(conn *net.Conn, config *common.WorkerConf, doneChannel chan bool, continueWorkers chan bool) {
@@ -173,3 +174,9 @@ func executeTestOnWorker(conn *net.Conn, config *common.WorkerConf, doneChannel
 		}
 	}
 }
+
+func shutdownWorker(conn *net.Conn) {
+	encoder := json.NewEncoder(*conn)
+	log.WithField("Worker", (*conn).RemoteAddr()).Info("Shutting down worker")
+	encoder.Encode(common.WorkerMessage{Message: "shutdown"})
+}

worker/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"math/rand"
@@ -49,15 +50,15 @@ func main() {
 		err := connectToServer(serverAddress)
 		if err != nil {
 			log.WithError(err).Fatal("Issues with server connection")
+			time.Sleep(time.Second)
 		}
 	}
 }
 
 func connectToServer(serverAddress string) error {
 	conn, err := net.Dial("tcp", serverAddress)
 	if err != nil {
-		log.WithError(err).Error("Could not connect to the server")
-		os.Exit(0)
+		return errors.New("Could not establish connection to server yet")
 	}
 	encoder := json.NewEncoder(conn)
 	decoder := json.NewDecoder(conn)
@@ -73,7 +74,7 @@ func connectToServer(serverAddress string) error {
 		if err != nil {
 			log.WithField("message", response).WithError(err).Error("Server responded unusually - reconnecting")
 			conn.Close()
-			return nil
+			return errors.New("Issue when receiving work from server")
 		}
 		log.Tracef("Response: %+v", response)
 		switch response.Message {
@@ -99,6 +100,9 @@ func connectToServer(serverAddress string) error {
 			encoder.Encode(common.WorkerMessage{Message: "work done"})
 			// Work is done - return to being a ready worker by reconnecting
 			return nil
+		case "shutdown":
+			log.Info("Server told us to shut down - all work is done for today")
+			os.Exit(0)
 		}
 	}
 }

worker/s3.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"io"
 	"net/http"
@@ -36,7 +37,6 @@ func init() {
 	if err != nil {
 		log.WithError(err).Fatalf("Failed to create the Prometheus exporter:")
 	}
-	hc = &http.Client{Transport: new(ochttp.Transport)}
 
 	if err := view.Register([]*view.View{
 		ochttp.ClientSentBytesDistribution,
@@ -67,6 +67,14 @@ func InitS3(config common.S3Configuration) {
 	// Session should be shared where possible to take advantage of
 	// configuration and credential caching. See the session package for
 	// more information.
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: config.SkipSSLVerify},
+	}
+	tr2 := &ochttp.Transport{Base: tr}
+	// tr2.(*http.RoundTripper).TLSClientConfig = &tls.Config{InsecureSkipVerify: config.SkipSSLVerify}
+	hc = &http.Client{
+		Transport: tr2,
+	}
 
 	sess := session.Must(session.NewSession(&aws.Config{
 		HTTPClient: hc,
@@ -78,6 +86,7 @@ func InitS3(config common.S3Configuration) {
 	}))
 	// Use this Session to do things that are hidden from the performance monitoring
 	housekeepingSess := session.Must(session.NewSession(&aws.Config{
+		HTTPClient: &http.Client{Transport: tr},
 		// TODO Also set the remaining S3 connection details...
 		Region:           &config.Region,
 		Credentials:      credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, ""),