Added support for persisting logs in 'emptyDir' volume attached to ag…

…ones controller.

The logs are saved in 100MB chunks under `/home/agones/logs` up to a specified limit (10000 MB by default).

Example:

```shell
$ ls -lh /home/agones/logs
total 757764
-rw-r--r--    1 agones   agones    100.0M Feb 22 21:22 agones-controller-20190222_212001-2019-02-22T21-22-37.499.log
-rw-r--r--    1 agones   agones    100.0M Feb 22 21:26 agones-controller-20190222_212001-2019-02-22T21-26-31.904.log
-rw-r--r--    1 agones   agones    100.0M Feb 22 21:27 agones-controller-20190222_212001-2019-02-22T21-27-01.546.log
-rw-r--r--    1 agones   agones    100.0M Feb 22 21:27 agones-controller-20190222_212001-2019-02-22T21-27-42.934.log
-rw-r--r--    1 agones   agones    100.0M Feb 22 21:28 agones-controller-20190222_212001-2019-02-22T21-28-46.574.log
-rw-r--r--    1 agones   agones    100.0M Feb 22 21:32 agones-controller-20190222_212001-2019-02-22T21-32-31.224.log
-rw-r--r--    1 agones   agones    100.0M Feb 22 21:33 agones-controller-20190222_212001-2019-02-22T21-33-30.350.log
-rw-r--r--    1 agones   agones     40.0M Feb 22 21:41 agones-controller-20190222_212001.log

```

Gopkg.toml

@@ -103,3 +103,7 @@
 [[constraint]]
   name = "fortio.org/fortio"
   version = "1.3.1"
+
+[[constraint]]
+  name = "gopkg.in/natefinch/lumberjack.v2"
+  version = "2.1.0"

cmd/controller/main.go

@@ -16,6 +16,7 @@
 package main
 
 import (
+	"io"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -39,8 +40,10 @@ import (
 	"github.com/heptiolabs/healthcheck"
 	"github.com/pkg/errors"
 	prom "github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+	lumberjack "gopkg.in/natefinch/lumberjack.v2"
 	extclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/client-go/informers"
@@ -63,6 +66,8 @@ const (
 	numWorkersFlag               = "num-workers"
 	apiServerSustainedQPSFlag    = "api-server-qps"
 	apiServerBurstQPSFlag        = "api-server-qps-burst"
+	logDirFlag                   = "log-dir"
+	logSizeLimitMBFlag           = "log-size-limit-mb"
 	kubeconfigFlag               = "kubeconfig"
 	defaultResync                = 30 * time.Second
 )
@@ -71,9 +76,32 @@ var (
 	logger = runtime.NewLoggerWithSource("main")
 )
 
+func setupLogging(logDir string, logSizeLimitMB int) {
+	logFileName := filepath.Join(logDir, "agones-controller-"+time.Now().Format("20060102_150405")+".log")
+
+	const maxLogSizeMB = 100
+	maxBackups := (logSizeLimitMB - maxLogSizeMB) / maxLogSizeMB
+	logger.WithField("filename", logFileName).WithField("numbackups", maxBackups).Info("logging to file")
+	logrus.SetOutput(
+		io.MultiWriter(
+			logrus.StandardLogger().Out,
+			&lumberjack.Logger{
+				Filename:   logFileName,
+				MaxSize:    maxLogSizeMB,
+				MaxBackups: maxBackups,
+			},
+		),
+	)
+}
+
 // main starts the operator for the gameserver CRD
 func main() {
 	ctlConf := parseEnvFlags()
+
+	if ctlConf.LogDir != "" {
+		setupLogging(ctlConf.LogDir, ctlConf.LogSizeLimitMB)
+	}
+
 	logger.WithField("version", pkg.Version).
 		WithField("ctlConf", ctlConf).Info("starting gameServer operator...")
 
@@ -203,6 +231,8 @@ func parseEnvFlags() config {
 	viper.SetDefault(numWorkersFlag, 64)
 	viper.SetDefault(apiServerSustainedQPSFlag, 100)
 	viper.SetDefault(apiServerBurstQPSFlag, 200)
+	viper.SetDefault(logDirFlag, "")
+	viper.SetDefault(logSizeLimitMBFlag, 10000) // 10 GB, will be split into 100 MB chunks
 
 	pflag.String(sidecarImageFlag, viper.GetString(sidecarImageFlag), "Flag to overwrite the GameServer sidecar image that is used. Can also use SIDECAR env variable")
 	pflag.String(sidecarCPULimitFlag, viper.GetString(sidecarCPULimitFlag), "Flag to overwrite the GameServer sidecar container's cpu limit. Can also use SIDECAR_CPU_LIMIT env variable")
@@ -219,6 +249,8 @@ func parseEnvFlags() config {
 	pflag.Int32(numWorkersFlag, 64, "Number of controller workers per resource type")
 	pflag.Int32(apiServerSustainedQPSFlag, 100, "Maximum sustained queries per second to send to the API server")
 	pflag.Int32(apiServerBurstQPSFlag, 200, "Maximum burst queries per second to send to the API server")
+	pflag.String(logDirFlag, viper.GetString(logDirFlag), "If set, store logs in a given directory.")
+	pflag.Int32(logSizeLimitMBFlag, 1000, "Log file size limit in MB")
 	pflag.Parse()
 
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
@@ -238,6 +270,8 @@ func parseEnvFlags() config {
 	runtime.Must(viper.BindEnv(numWorkersFlag))
 	runtime.Must(viper.BindEnv(apiServerSustainedQPSFlag))
 	runtime.Must(viper.BindEnv(apiServerBurstQPSFlag))
+	runtime.Must(viper.BindEnv(logDirFlag))
+	runtime.Must(viper.BindEnv(logSizeLimitMBFlag))
 
 	request, err := resource.ParseQuantity(viper.GetString(sidecarCPURequestFlag))
 	if err != nil {
@@ -265,6 +299,8 @@ func parseEnvFlags() config {
 		NumWorkers:            int(viper.GetInt32(numWorkersFlag)),
 		APIServerSustainedQPS: int(viper.GetInt32(apiServerSustainedQPSFlag)),
 		APIServerBurstQPS:     int(viper.GetInt32(apiServerBurstQPSFlag)),
+		LogDir:                viper.GetString(logDirFlag),
+		LogSizeLimitMB:        int(viper.GetInt32(logSizeLimitMBFlag)),
 	}
 }
 
@@ -285,6 +321,8 @@ type config struct {
 	NumWorkers            int
 	APIServerSustainedQPS int
 	APIServerBurstQPS     int
+	LogDir                string
+	LogSizeLimitMB        int
 }
 
 // validate ensures the ctlConfig data is valid.

install/helm/agones/templates/controller.yaml

@@ -98,6 +98,12 @@ spec:
           value: {{ .Values.agones.controller.apiServerQPS | quote }}
         - name: API_SERVER_QPS_BURST
           value: {{ .Values.agones.controller.apiServerQPSBurst | quote }}
+{{- if .Values.agones.controller.persistentLogs }}
+        - name: LOG_DIR
+          value: "/home/agones/logs"
+        - name: LOG_SIZE_LIMIT_MB
+          value: {{ .Values.agones.controller.persistentLogsSizeLimitMB | quote }}
+{{- end }}
         livenessProbe:
           httpGet:
             path: /live
@@ -114,10 +120,19 @@ spec:
         - name: certs
           mountPath: /home/agones/certs
           readOnly: true
+{{- if .Values.agones.controller.persistentLogs }}
+        - name: logs
+          mountPath: /home/agones/logs
+          readOnly: false
+{{- end }}
       volumes:
       - name: certs
         secret:
           secretName: {{ template "agones.fullname" . }}-cert
+{{- if .Values.agones.controller.persistentLogs }}
+      - name: logs
+        emptyDir: {}
+{{- end }}
 {{- if .Values.agones.image.controller.pullSecret }}
       imagePullSecrets:
         - name: {{.Values.agones.image.controller.pullSecret}}

install/helm/agones/values.yaml

@@ -49,6 +49,8 @@ agones:
                 operator: Exists
     generateTLS: true
     safeToEvict: false
+    persistentLogs: true
+    persistentLogsSizeLimitMB: 10000
     numWorkers: 100
     apiServerQPS: 400
     apiServerQPSBurst: 500

install/yaml/install.yaml

@@ -1089,6 +1089,10 @@ spec:
           value: "400"
         - name: API_SERVER_QPS_BURST
           value: "500"
+        - name: LOG_DIR
+          value: "/home/agones/logs"
+        - name: LOG_SIZE_LIMIT_MB
+          value: "10000"
         livenessProbe:
           httpGet:
             path: /live
@@ -1101,10 +1105,15 @@ spec:
         - name: certs
           mountPath: /home/agones/certs
           readOnly: true
+        - name: logs
+          mountPath: /home/agones/logs
+          readOnly: false
       volumes:
       - name: certs
         secret:
           secretName: agones-manual-cert
+      - name: logs
+        emptyDir: {}
 
 ---
 # Source: agones/templates/ping.yaml

site/content/en/docs/Installation/helm.md

@@ -164,6 +164,8 @@ The following tables lists the configurable parameters of the Agones chart and t
 
 | Parameter                                           | Description                                                                                     | Default                |
 | --------------------------------------------------- | ----------------------------------------------------------------------------------------------- | ---------------------- |
+| `agones.controller.persistentLogs`                  | Store Agones controller logs in a temporary volume attached to a container for debugging        | `true`                 |
+| `agones.controller.persistentLogsSizeLimitMB`       | Maximum total size of all Agones container logs in MB                                           | `10000`                |
 
 {{% /feature %}}