From 5613e92b4d31cb52819515f3382eb9e28fa02c73 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Wed, 23 Jan 2019 10:34:40 -0800 Subject: [PATCH] controller: made QPS, burst QPS and number of workers externally configurable and increased their default values signfificantly based on scale test results --- cmd/controller/main.go | 86 ++++++++++++------- install/helm/agones/README.md | 3 + install/helm/agones/templates/controller.yaml | 6 ++ install/helm/agones/values.yaml | 3 + install/yaml/install.yaml | 6 ++ 5 files changed, 71 insertions(+), 33 deletions(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index b695baaa4c..8cc705d830 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -49,18 +49,20 @@ import ( ) const ( - enableMetricsFlag = "metrics" - sidecarImageFlag = "sidecar-image" - sidecarCPURequestFlag = "sidecar-cpu-request" - sidecarCPULimitFlag = "sidecar-cpu-limit" - pullSidecarFlag = "always-pull-sidecar" - minPortFlag = "min-port" - maxPortFlag = "max-port" - certFileFlag = "cert-file" - keyFileFlag = "key-file" - kubeconfigFlag = "kubeconfig" - workers = 2 - defaultResync = 30 * time.Second + enableMetricsFlag = "metrics" + sidecarImageFlag = "sidecar-image" + sidecarCPURequestFlag = "sidecar-cpu-request" + sidecarCPULimitFlag = "sidecar-cpu-limit" + pullSidecarFlag = "always-pull-sidecar" + minPortFlag = "min-port" + maxPortFlag = "max-port" + certFileFlag = "cert-file" + keyFileFlag = "key-file" + numWorkersFlag = "num-workers" + apiServerSustainedQPSFlag = "api-server-qps" + apiServerBurstQPSFlag = "api-server-qps-burst" + kubeconfigFlag = "kubeconfig" + defaultResync = 30 * time.Second ) var ( @@ -83,6 +85,9 @@ func main() { logger.WithError(err).Fatal("Could not create in cluster config") } + clientConf.QPS = float32(ctlConf.APIServerSustainedQPS) + clientConf.Burst = ctlConf.APIServerBurstQPS + kubeClient, err := kubernetes.NewForConfig(clientConf) if err != nil { logger.WithError(err).Fatal("Could not create the kubernetes clientset") @@ -147,7 +152,7 @@ func main() { for _, r := range rs { go func(rr runner) { - if runErr := rr.Run(workers, stop); runErr != nil { + if runErr := rr.Run(ctlConf.NumWorkers, stop); runErr != nil { logger.WithError(runErr).Fatalf("could not start runner: %T", rr) } }(r) @@ -171,6 +176,9 @@ func parseEnvFlags() config { viper.SetDefault(certFileFlag, filepath.Join(base, "certs/server.crt")) viper.SetDefault(keyFileFlag, filepath.Join(base, "certs/server.key")) viper.SetDefault(enableMetricsFlag, true) + viper.SetDefault(numWorkersFlag, 64) + viper.SetDefault(apiServerSustainedQPSFlag, 100) + viper.SetDefault(apiServerBurstQPSFlag, 200) pflag.String(sidecarImageFlag, viper.GetString(sidecarImageFlag), "Flag to overwrite the GameServer sidecar image that is used. Can also use SIDECAR env variable") pflag.String(sidecarCPULimitFlag, viper.GetString(sidecarCPULimitFlag), "Flag to overwrite the GameServer sidecar container's cpu limit. Can also use SIDECAR_CPU_LIMIT env variable") @@ -182,6 +190,9 @@ func parseEnvFlags() config { pflag.String(certFileFlag, viper.GetString(certFileFlag), "Optional. Path to the crt file") pflag.String(kubeconfigFlag, viper.GetString(kubeconfigFlag), "Optional. kubeconfig to run the controller out of the cluster. Only use it for debugging as webhook won't works.") pflag.Bool(enableMetricsFlag, viper.GetBool(enableMetricsFlag), "Flag to activate metrics of Agones. Can also use METRICS env variable.") + pflag.Int32(numWorkersFlag, 64, "Number of controller workers per resource type") + pflag.Int32(apiServerSustainedQPSFlag, 100, "Maximum sustained queries per second to send to the API server") + pflag.Int32(apiServerBurstQPSFlag, 200, "Maximum burst queries per second to send to the API server") pflag.Parse() viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) @@ -196,6 +207,9 @@ func parseEnvFlags() config { runtime.Must(viper.BindEnv(kubeconfigFlag)) runtime.Must(viper.BindEnv(enableMetricsFlag)) runtime.Must(viper.BindPFlags(pflag.CommandLine)) + runtime.Must(viper.BindEnv(numWorkersFlag)) + runtime.Must(viper.BindEnv(apiServerSustainedQPSFlag)) + runtime.Must(viper.BindEnv(apiServerBurstQPSFlag)) request, err := resource.ParseQuantity(viper.GetString(sidecarCPURequestFlag)) if err != nil { @@ -208,31 +222,37 @@ func parseEnvFlags() config { } return config{ - MinPort: int32(viper.GetInt64(minPortFlag)), - MaxPort: int32(viper.GetInt64(maxPortFlag)), - SidecarImage: viper.GetString(sidecarImageFlag), - SidecarCPURequest: request, - SidecarCPULimit: limit, - AlwaysPullSidecar: viper.GetBool(pullSidecarFlag), - KeyFile: viper.GetString(keyFileFlag), - CertFile: viper.GetString(certFileFlag), - KubeConfig: viper.GetString(kubeconfigFlag), - Metrics: viper.GetBool(enableMetricsFlag), + MinPort: int32(viper.GetInt64(minPortFlag)), + MaxPort: int32(viper.GetInt64(maxPortFlag)), + SidecarImage: viper.GetString(sidecarImageFlag), + SidecarCPURequest: request, + SidecarCPULimit: limit, + AlwaysPullSidecar: viper.GetBool(pullSidecarFlag), + KeyFile: viper.GetString(keyFileFlag), + CertFile: viper.GetString(certFileFlag), + KubeConfig: viper.GetString(kubeconfigFlag), + Metrics: viper.GetBool(enableMetricsFlag), + NumWorkers: int(viper.GetInt32(numWorkersFlag)), + APIServerSustainedQPS: int(viper.GetInt32(apiServerSustainedQPSFlag)), + APIServerBurstQPS: int(viper.GetInt32(apiServerBurstQPSFlag)), } } // config stores all required configuration to create a game server controller. type config struct { - MinPort int32 - MaxPort int32 - SidecarImage string - SidecarCPURequest resource.Quantity - SidecarCPULimit resource.Quantity - AlwaysPullSidecar bool - Metrics bool - KeyFile string - CertFile string - KubeConfig string + MinPort int32 + MaxPort int32 + SidecarImage string + SidecarCPURequest resource.Quantity + SidecarCPULimit resource.Quantity + AlwaysPullSidecar bool + Metrics bool + KeyFile string + CertFile string + KubeConfig string + NumWorkers int + APIServerSustainedQPS int + APIServerBurstQPS int } // validate ensures the ctlConfig data is valid. diff --git a/install/helm/agones/README.md b/install/helm/agones/README.md index 1f22d92f3a..cbb5f4e5cd 100644 --- a/install/helm/agones/README.md +++ b/install/helm/agones/README.md @@ -111,6 +111,9 @@ The following tables lists the configurable parameters of the Agones chart and t | `agones.controller.tolerations` | (⚠️ **development feature**⚠️) Controller [toleration][toleration] labels for pod assignment | `[]` | | `agones.controller.affinity` | (⚠️ **development feature**⚠️) Controller [affinity](affinity) settings for pod assignment | `{}` | | `agones.controller.generateTLS` | Set to true to generate TLS certificates or false to provide your own certificates in `certs/*` | `true` | +| `agones.controller.numWorkers` | Number of workers to spin per resource type | `64` | +| `agones.controller.apiServerQPS` | Maximum sustained queries per second that controller should be making against API Server | `100` | +| `agones.controller.apiServerQPSBurst` | Maximum burst queries per second that controller should be making against API Server | `200` | | `agones.ping.install` | Whether to install the [ping service][ping] | `true` | | `agones.ping.replicas` | The number of replicas to run in the deployment | `2` | | `agones.ping.http.expose` | Expose the http ping service via a Service | `true` | diff --git a/install/helm/agones/templates/controller.yaml b/install/helm/agones/templates/controller.yaml index c96481f80c..fa01602b32 100644 --- a/install/helm/agones/templates/controller.yaml +++ b/install/helm/agones/templates/controller.yaml @@ -85,6 +85,12 @@ spec: value: {{ .Values.agones.metrics.enabled | quote }} - name: SIDECAR_CPU_LIMIT value: {{ .Values.agones.image.sdk.cpuLimit | quote }} + - name: NUM_WORKERS + value: {{ .Values.agones.controller.numWorkers | quote }} + - name: API_SERVER_QPS + value: {{ .Values.agones.controller.apiServerQPS | quote }} + - name: API_SERVER_QPS_BURST + value: {{ .Values.agones.controller.apiServerQPSBurst | quote }} livenessProbe: httpGet: path: /live diff --git a/install/helm/agones/values.yaml b/install/helm/agones/values.yaml index 102f9a3466..0384ead883 100644 --- a/install/helm/agones/values.yaml +++ b/install/helm/agones/values.yaml @@ -32,6 +32,9 @@ agones: affinity: {} generateTLS: true safeToEvict: false + numWorkers: 64 + apiServerQPS: 100 + apiServerQPSBurst: 200 http: port: 8080 healthCheck: diff --git a/install/yaml/install.yaml b/install/yaml/install.yaml index c7622af87a..5ca73139ca 100644 --- a/install/yaml/install.yaml +++ b/install/yaml/install.yaml @@ -1035,6 +1035,12 @@ spec: value: "true" - name: SIDECAR_CPU_LIMIT value: "0" + - name: NUM_WORKERS + value: "64" + - name: API_SERVER_QPS + value: "100" + - name: API_SERVER_QPS_BURST + value: "200" livenessProbe: httpGet: path: /live