controller: made QPS, burst QPS and number of workers externally conf…

…igurable and increased their default values signfificantly based on scale test results
googleforgames · Jan 23, 2019 · e255d66 · e255d66
1 parent c62be56
commit e255d66
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 33 deletions.
diff --git a/cmd/controller/main.go b/cmd/controller/main.go
@@ -49,18 +49,20 @@ import (
 )
 
 const (
-	enableMetricsFlag     = "metrics"
-	sidecarImageFlag      = "sidecar-image"
-	sidecarCPURequestFlag = "sidecar-cpu-request"
-	sidecarCPULimitFlag   = "sidecar-cpu-limit"
-	pullSidecarFlag       = "always-pull-sidecar"
-	minPortFlag           = "min-port"
-	maxPortFlag           = "max-port"
-	certFileFlag          = "cert-file"
-	keyFileFlag           = "key-file"
-	kubeconfigFlag        = "kubeconfig"
-	workers               = 2
-	defaultResync         = 30 * time.Second
+	enableMetricsFlag         = "metrics"
+	sidecarImageFlag          = "sidecar-image"
+	sidecarCPURequestFlag     = "sidecar-cpu-request"
+	sidecarCPULimitFlag       = "sidecar-cpu-limit"
+	pullSidecarFlag           = "always-pull-sidecar"
+	minPortFlag               = "min-port"
+	maxPortFlag               = "max-port"
+	certFileFlag              = "cert-file"
+	keyFileFlag               = "key-file"
+	numWorkersFlag            = "num-workers"
+	apiServerSustainedQPSFlag = "api-server-qps"
+	apiServerBurstQPSFlag     = "api-server-qps-burst"
+	kubeconfigFlag            = "kubeconfig"
+	defaultResync             = 30 * time.Second
 )
 
 var (
@@ -83,6 +85,9 @@ func main() {
 		logger.WithError(err).Fatal("Could not create in cluster config")
 	}
 
+	clientConf.QPS = float32(ctlConf.APIServerSustainedQPS)
+	clientConf.Burst = ctlConf.APIServerBurstQPS
+
 	kubeClient, err := kubernetes.NewForConfig(clientConf)
 	if err != nil {
 		logger.WithError(err).Fatal("Could not create the kubernetes clientset")
@@ -147,7 +152,7 @@ func main() {
 
 	for _, r := range rs {
 		go func(rr runner) {
-			if runErr := rr.Run(workers, stop); runErr != nil {
+			if runErr := rr.Run(ctlConf.NumWorkers, stop); runErr != nil {
 				logger.WithError(runErr).Fatalf("could not start runner: %T", rr)
 			}
 		}(r)
@@ -171,6 +176,9 @@ func parseEnvFlags() config {
 	viper.SetDefault(certFileFlag, filepath.Join(base, "certs/server.crt"))
 	viper.SetDefault(keyFileFlag, filepath.Join(base, "certs/server.key"))
 	viper.SetDefault(enableMetricsFlag, true)
+	viper.SetDefault(numWorkersFlag, 64)
+	viper.SetDefault(apiServerSustainedQPSFlag, 100)
+	viper.SetDefault(apiServerBurstQPSFlag, 200)
 
 	pflag.String(sidecarImageFlag, viper.GetString(sidecarImageFlag), "Flag to overwrite the GameServer sidecar image that is used. Can also use SIDECAR env variable")
 	pflag.String(sidecarCPULimitFlag, viper.GetString(sidecarCPULimitFlag), "Flag to overwrite the GameServer sidecar container's cpu limit. Can also use SIDECAR_CPU_LIMIT env variable")
@@ -182,6 +190,9 @@ func parseEnvFlags() config {
 	pflag.String(certFileFlag, viper.GetString(certFileFlag), "Optional. Path to the crt file")
 	pflag.String(kubeconfigFlag, viper.GetString(kubeconfigFlag), "Optional. kubeconfig to run the controller out of the cluster. Only use it for debugging as webhook won't works.")
 	pflag.Bool(enableMetricsFlag, viper.GetBool(enableMetricsFlag), "Flag to activate metrics of Agones. Can also use METRICS env variable.")
+	pflag.Int32(numWorkersFlag, 64, "Number of controller workers per resource type")
+	pflag.Int32(apiServerSustainedQPSFlag, 100, "Maximum sustained queries per second to send to the API server")
+	pflag.Int32(apiServerBurstQPSFlag, 200, "Maximum burst queries per second to send to the API server")
 	pflag.Parse()
 
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
@@ -196,6 +207,9 @@ func parseEnvFlags() config {
 	runtime.Must(viper.BindEnv(kubeconfigFlag))
 	runtime.Must(viper.BindEnv(enableMetricsFlag))
 	runtime.Must(viper.BindPFlags(pflag.CommandLine))
+	runtime.Must(viper.BindEnv(numWorkersFlag))
+	runtime.Must(viper.BindEnv(apiServerSustainedQPSFlag))
+	runtime.Must(viper.BindEnv(apiServerBurstQPSFlag))
 
 	request, err := resource.ParseQuantity(viper.GetString(sidecarCPURequestFlag))
 	if err != nil {
@@ -208,31 +222,37 @@ func parseEnvFlags() config {
 	}
 
 	return config{
-		MinPort:           int32(viper.GetInt64(minPortFlag)),
-		MaxPort:           int32(viper.GetInt64(maxPortFlag)),
-		SidecarImage:      viper.GetString(sidecarImageFlag),
-		SidecarCPURequest: request,
-		SidecarCPULimit:   limit,
-		AlwaysPullSidecar: viper.GetBool(pullSidecarFlag),
-		KeyFile:           viper.GetString(keyFileFlag),
-		CertFile:          viper.GetString(certFileFlag),
-		KubeConfig:        viper.GetString(kubeconfigFlag),
-		Metrics:           viper.GetBool(enableMetricsFlag),
+		MinPort:               int32(viper.GetInt64(minPortFlag)),
+		MaxPort:               int32(viper.GetInt64(maxPortFlag)),
+		SidecarImage:          viper.GetString(sidecarImageFlag),
+		SidecarCPURequest:     request,
+		SidecarCPULimit:       limit,
+		AlwaysPullSidecar:     viper.GetBool(pullSidecarFlag),
+		KeyFile:               viper.GetString(keyFileFlag),
+		CertFile:              viper.GetString(certFileFlag),
+		KubeConfig:            viper.GetString(kubeconfigFlag),
+		Metrics:               viper.GetBool(enableMetricsFlag),
+		NumWorkers:            int(viper.GetInt32(numWorkersFlag)),
+		APIServerSustainedQPS: int(viper.GetInt32(apiServerSustainedQPSFlag)),
+		APIServerBurstQPS:     int(viper.GetInt32(apiServerBurstQPSFlag)),
 	}
 }
 
 // config stores all required configuration to create a game server controller.
 type config struct {
-	MinPort           int32
-	MaxPort           int32
-	SidecarImage      string
-	SidecarCPURequest resource.Quantity
-	SidecarCPULimit   resource.Quantity
-	AlwaysPullSidecar bool
-	Metrics           bool
-	KeyFile           string
-	CertFile          string
-	KubeConfig        string
+	MinPort               int32
+	MaxPort               int32
+	SidecarImage          string
+	SidecarCPURequest     resource.Quantity
+	SidecarCPULimit       resource.Quantity
+	AlwaysPullSidecar     bool
+	Metrics               bool
+	KeyFile               string
+	CertFile              string
+	KubeConfig            string
+	NumWorkers            int
+	APIServerSustainedQPS int
+	APIServerBurstQPS     int
 }
 
 // validate ensures the ctlConfig data is valid.

diff --git a/install/helm/agones/README.md b/install/helm/agones/README.md
@@ -111,6 +111,9 @@ The following tables lists the configurable parameters of the Agones chart and t
 | `agones.controller.tolerations`                     | (⚠️ **development feature**⚠️) Controller [toleration][toleration] labels for pod assignment                                   | `[]`                   |
 | `agones.controller.affinity`                        | (⚠️ **development feature**⚠️) Controller [affinity](affinity) settings for pod assignment                                     | `{}`                   |
 | `agones.controller.generateTLS`                     | Set to true to generate TLS certificates or false to provide your own certificates in `certs/*` | `true`                 |
+| `agones.controller.numWorkers`                      | Number of workers to spin per resource type                                                     | `64`                   |
+| `agones.controller.apiServerQPS`                    | Maximum sustained queries per second that controller should be making against API Server        | `100`                  |
+| `agones.controller.apiServerQPSBurst`               | Maximum burst queries per second that controller should be making against API Server            | `200`                  |
 | `agones.ping.install`                               | Whether to install the [ping service][ping]                                                     | `true`                 |
 | `agones.ping.replicas`                              | The number of replicas to run in the deployment                                                 | `2`                    | 
 | `agones.ping.http.expose`                           | Expose the http ping service via a Service                                                      | `true`                 | 

diff --git a/install/helm/agones/templates/controller.yaml b/install/helm/agones/templates/controller.yaml
@@ -85,6 +85,12 @@ spec:
           value: {{ .Values.agones.metrics.enabled | quote }}
         - name: SIDECAR_CPU_LIMIT
           value: {{ .Values.agones.image.sdk.cpuLimit | quote }}
+        - name: NUM_WORKERS
+          value: {{ .Values.agones.controller.numWorkers | quote }}
+        - name: API_SERVER_QPS
+          value: {{ .Values.agones.controller.apiServerQPS | quote }}
+        - name: API_SERVER_QPS_BURST
+          value: {{ .Values.agones.controller.apiServerQPSBurst | quote }}
         livenessProbe:
           httpGet:
             path: /live

diff --git a/install/helm/agones/values.yaml b/install/helm/agones/values.yaml
@@ -32,6 +32,9 @@ agones:
     affinity: {}
     generateTLS: true
     safeToEvict: false
+    numWorkers: 64
+    apiServerQPS: 100
+    apiServerQPSBurst: 200
     http:
       port: 8080
     healthCheck:

diff --git a/install/yaml/install.yaml b/install/yaml/install.yaml
@@ -1035,6 +1035,12 @@ spec:
           value: "true"
         - name: SIDECAR_CPU_LIMIT
           value: "0"
+        - name: NUM_WORKERS
+          value: "64"
+        - name: API_SERVER_QPS
+          value: "100"
+        - name: API_SERVER_QPS_BURST
+          value: "200"
         livenessProbe:
           httpGet:
             path: /live