diff --git a/cmd/extensions/main.go b/cmd/extensions/main.go index d01be204f0..f6cb453536 100644 --- a/cmd/extensions/main.go +++ b/cmd/extensions/main.go @@ -68,6 +68,8 @@ const ( apiServerSustainedQPSFlag = "api-server-qps" apiServerBurstQPSFlag = "api-server-qps-burst" readinessShutdownDuration = "readiness-shutdown-duration" + httpPort = "http-port" + webhookPort = "webhook-port" ) var ( @@ -138,7 +140,7 @@ func main() { logger.WithError(err).Fatal("Could not initialize cloud product") } // https server and the items that share the Mux for routing - httpsServer := https.NewServer(ctlConf.CertFile, ctlConf.KeyFile) + httpsServer := https.NewServer(ctlConf.CertFile, ctlConf.KeyFile, ctlConf.WebhookPort) cancelTLS, err := httpsServer.WatchForCertificateChanges() if err != nil { logger.WithError(err).Fatal("Got an error while watching certificate changes") @@ -150,7 +152,10 @@ func main() { agonesInformerFactory := externalversions.NewSharedInformerFactory(agonesClient, defaultResync) kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, defaultResync) - server := &httpserver.Server{Logger: logger} + server := &httpserver.Server{ + Port: ctlConf.HTTPPort, + Logger: logger, + } var health healthcheck.Handler // Stackdriver metrics @@ -249,6 +254,8 @@ func parseEnvFlags() config { viper.SetDefault(logDirFlag, "") viper.SetDefault(logLevelFlag, "Info") viper.SetDefault(logSizeLimitMBFlag, 10000) // 10 GB, will be split into 100 MB chunks + viper.SetDefault(httpPort, "8080") + viper.SetDefault(webhookPort, "8081") pflag.String(keyFileFlag, viper.GetString(keyFileFlag), "Optional. Path to the key file") pflag.String(certFileFlag, viper.GetString(certFileFlag), "Optional. Path to the crt file") @@ -262,6 +269,8 @@ func parseEnvFlags() config { pflag.Int32(numWorkersFlag, 64, "Number of controller workers per resource type") pflag.Int32(apiServerSustainedQPSFlag, 100, "Maximum sustained queries per second to send to the API server") pflag.Int32(apiServerBurstQPSFlag, 200, "Maximum burst queries per second to send to the API server") + pflag.String(httpPort, viper.GetString(httpPort), "Port for the HTTP server. Defaults to 8080, can also use HTTP_PORT env variable") + pflag.String(webhookPort, viper.GetString(webhookPort), "Port for the Webhook. Defaults to 8081, can also use WEBHOOK_PORT env variable") pflag.String(logDirFlag, viper.GetString(logDirFlag), "If set, store logs in a given directory.") pflag.Int32(logSizeLimitMBFlag, 1000, "Log file size limit in MB") pflag.String(logLevelFlag, viper.GetString(logLevelFlag), "Agones Log level") @@ -288,6 +297,8 @@ func parseEnvFlags() config { runtime.Must(viper.BindEnv(logLevelFlag)) runtime.Must(viper.BindEnv(logDirFlag)) runtime.Must(viper.BindEnv(logSizeLimitMBFlag)) + runtime.Must(viper.BindEnv(httpPort)) + runtime.Must(viper.BindEnv(webhookPort)) runtime.Must(viper.BindEnv(allocationBatchWaitTime)) runtime.Must(viper.BindPFlags(pflag.CommandLine)) runtime.Must(viper.BindEnv(readinessShutdownDuration)) @@ -311,6 +322,8 @@ func parseEnvFlags() config { LogDir: viper.GetString(logDirFlag), LogLevel: viper.GetString(logLevelFlag), LogSizeLimitMB: int(viper.GetInt32(logSizeLimitMBFlag)), + HTTPPort: viper.GetString(httpPort), + WebhookPort: viper.GetString(webhookPort), AllocationBatchWaitTime: viper.GetDuration(allocationBatchWaitTime), ReadinessShutdownDuration: viper.GetDuration(readinessShutdownDuration), } @@ -333,6 +346,8 @@ type config struct { LogDir string LogLevel string LogSizeLimitMB int + HTTPPort string + WebhookPort string AllocationBatchWaitTime time.Duration ReadinessShutdownDuration time.Duration } diff --git a/install/helm/agones/templates/extensions-deployment.yaml b/install/helm/agones/templates/extensions-deployment.yaml index f190de06ea..3745af240b 100644 --- a/install/helm/agones/templates/extensions-deployment.yaml +++ b/install/helm/agones/templates/extensions-deployment.yaml @@ -41,7 +41,7 @@ spec: {{- end }} {{- if and (.Values.agones.metrics.prometheusServiceDiscovery) (.Values.agones.metrics.prometheusEnabled) }} prometheus.io/scrape: "true" - prometheus.io/port: "8080" + prometheus.io/port: {{ .Values.agones.extensions.http.port | quote }} prometheus.io/path: "/metrics" {{- end }} {{- if .Values.agones.extensions.annotations }} @@ -93,6 +93,10 @@ spec: {{- end }} serviceAccountName: {{ .Values.agones.serviceaccount.controller.name }} terminationGracePeriodSeconds: {{ mul .Values.agones.extensions.readiness.periodSeconds .Values.agones.extensions.readiness.failureThreshold 3 }} + {{- if .Values.agones.extensions.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + {{- end }} containers: - name: agones-extensions image: "{{ .Values.agones.image.registry }}/{{ .Values.agones.image.extensions.name}}:{{ default .Values.agones.image.tag .Values.agones.image.extensions.tag }}" @@ -107,7 +111,7 @@ spec: - name: STACKDRIVER_EXPORTER value: {{ .Values.agones.metrics.stackdriverEnabled | quote }} - name: STACKDRIVER_LABELS - value: {{ .Values.agones.metrics.stackdriverLabels | quote }} + value: {{ .Values.agones.metrics.stackdriverLabels | quote }} - name: GCP_PROJECT_ID value: {{ .Values.agones.metrics.stackdriverProjectID | quote }} - name: NUM_WORKERS @@ -142,11 +146,15 @@ spec: value: "agones-extensions" - name: READINESS_SHUTDOWN_DURATION value: {{ mul .Values.agones.extensions.readiness.periodSeconds .Values.agones.extensions.readiness.failureThreshold 2 }}s + - name: WEBHOOK_PORT + value: {{ .Values.agones.extensions.webhooks.port | quote }} + - name: HTTP_PORT + value: {{ .Values.agones.extensions.http.port | quote }} ports: - name: webhooks - containerPort: 8081 + containerPort: {{ .Values.agones.extensions.webhooks.port }} - name: http - containerPort: 8080 + containerPort: {{ .Values.agones.extensions.http.port }} livenessProbe: httpGet: path: /live @@ -158,7 +166,7 @@ spec: readinessProbe: httpGet: path: /ready - port: 8080 + port: {{ .Values.agones.extensions.http.port }} initialDelaySeconds: {{ .Values.agones.extensions.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.agones.extensions.readiness.periodSeconds }} failureThreshold: {{ .Values.agones.extensions.readiness.failureThreshold }} diff --git a/install/helm/agones/templates/extensions-metrics-service.yaml b/install/helm/agones/templates/extensions-metrics-service.yaml index a71a6ff579..7b170393f1 100644 --- a/install/helm/agones/templates/extensions-metrics-service.yaml +++ b/install/helm/agones/templates/extensions-metrics-service.yaml @@ -28,5 +28,5 @@ spec: agones.dev/role: extensions ports: - name: metrics - port: {{ .Values.agones.controller.http.port }} + port: {{ .Values.agones.extensions.http.port }} targetPort: http diff --git a/install/helm/agones/templates/service.yaml b/install/helm/agones/templates/service.yaml index 384c80d3d1..bfd76349da 100644 --- a/install/helm/agones/templates/service.yaml +++ b/install/helm/agones/templates/service.yaml @@ -32,5 +32,5 @@ spec: port: 443 targetPort: webhooks - name: web - port: {{ .Values.agones.controller.http.port }} + port: {{ .Values.agones.extensions.http.port }} targetPort: http diff --git a/install/helm/agones/values.yaml b/install/helm/agones/values.yaml index 249db19229..c7755235cf 100644 --- a/install/helm/agones/values.yaml +++ b/install/helm/agones/values.yaml @@ -100,6 +100,15 @@ agones: # memory: 256Mi nodeSelector: {} annotations: {} + # Determines if the Agones extensions should operate in hostNetwork mode. + # + # This setting is necessary for certain managed Kubernetes clusters (e.g., AWS EKS) that use custom + # CNI plugins (such as Calico or Cilium) because the AWS-managed control plane cannot communicate + # with pod IP CIDRs. + # + # Note: The default port may conflicts with others on the host network. Therefore, if + # running in hostNetwork mode, you should change `http.port` and `webhooks.port` to an available port. + hostNetwork: false tolerations: - key: "agones.dev/agones-system" operator: "Equal" @@ -125,6 +134,8 @@ agones: numWorkers: 100 apiServerQPS: 400 apiServerQPSBurst: 500 + webhooks: + port: 8081 http: port: 8080 healthCheck: diff --git a/install/yaml/install.yaml b/install/yaml/install.yaml index 8a50e99232..7687bace59 100644 --- a/install/yaml/install.yaml +++ b/install/yaml/install.yaml @@ -17237,7 +17237,7 @@ spec: - name: STACKDRIVER_EXPORTER value: "false" - name: STACKDRIVER_LABELS - value: "" + value: "" - name: GCP_PROJECT_ID value: "" - name: NUM_WORKERS @@ -17270,6 +17270,10 @@ spec: value: "agones-extensions" - name: READINESS_SHUTDOWN_DURATION value: 18s + - name: WEBHOOK_PORT + value: "8081" + - name: HTTP_PORT + value: "8080" ports: - name: webhooks containerPort: 8081 diff --git a/pkg/util/https/server.go b/pkg/util/https/server.go index 7d472fd8c7..4d7c4e50c0 100644 --- a/pkg/util/https/server.go +++ b/pkg/util/https/server.go @@ -53,16 +53,18 @@ type Server struct { tls tls certFile string keyFile string + port string } // NewServer returns a Server instance. -func NewServer(certFile, keyFile string) *Server { +func NewServer(certFile, keyFile string, port string) *Server { mux := http.NewServeMux() wh := &Server{ Mux: mux, certFile: certFile, keyFile: keyFile, + port: port, } wh.logger = runtime.NewLoggerWithType(wh) wh.setupServer() @@ -73,7 +75,7 @@ func NewServer(certFile, keyFile string) *Server { func (s *Server) setupServer() { s.tls = &http.Server{ - Addr: ":8081", + Addr: ":" + s.port, Handler: s.Mux, TLSConfig: &cryptotls.Config{ GetCertificate: s.getCertificate, @@ -129,7 +131,7 @@ func (s *Server) Run(ctx context.Context, _ int) error { _ = s.tls.Shutdown(context.Background()) }() - s.logger.WithField("server", s).Infof("https server started") + s.logger.WithField("server", s).Infof("https server started on port :" + s.port) err := s.tls.ListenAndServeTLS(s.certFile, s.keyFile) if err == http.ErrServerClosed { @@ -137,7 +139,7 @@ func (s *Server) Run(ctx context.Context, _ int) error { return nil } - return errors.Wrap(err, "Could not listen on :8081") + return errors.Wrap(err, "Could not listen on :"+s.port) } // defaultHandler Handles all the HTTP requests diff --git a/pkg/util/https/server_test.go b/pkg/util/https/server_test.go index a03739f453..f9c6232128 100644 --- a/pkg/util/https/server_test.go +++ b/pkg/util/https/server_test.go @@ -41,7 +41,7 @@ func (ts *testServer) ListenAndServeTLS(certFile, keyFile string) error { func TestServerRun(t *testing.T) { t.Parallel() - s := NewServer("", "") + s := NewServer("", "", "") ts := &testServer{server: httptest.NewUnstartedServer(s.Mux)} s.tls = ts diff --git a/pkg/util/httpserver/server.go b/pkg/util/httpserver/server.go index 2a4811fdc5..6b624c29e6 100644 --- a/pkg/util/httpserver/server.go +++ b/pkg/util/httpserver/server.go @@ -31,6 +31,7 @@ import ( //nolint:govet // ignore field alignment complaint, this is a singleton type Server struct { http.ServeMux + Port string Logger *logrus.Entry } @@ -38,8 +39,11 @@ type Server struct { // Run runs an http server on port :8080. func (s *Server) Run(ctx context.Context, _ int) error { s.Logger.Info("Starting http server...") + if s.Port == "" { + s.Port = "8080" + } srv := &http.Server{ - Addr: ":8080", + Addr: ":" + s.Port, Handler: s, } go func() { @@ -51,7 +55,7 @@ func (s *Server) Run(ctx context.Context, _ int) error { if err == http.ErrServerClosed { s.Logger.WithError(err).Info("http server closed") } else { - wrappedErr := errors.Wrap(err, "Could not listen on :8080") + wrappedErr := errors.Wrap(err, "Could not listen on :"+s.Port) runtime.HandleError(s.Logger.WithError(wrappedErr), wrappedErr) } }