Add Option to Use Host Network and Configure Ports (#3895)

Add support for hostNetwork mode in Agones extensions. Required for compatibility with managed Kubernetes clusters (e.g., AWS EKS) using custom CNI plugins (e.g., Calico, Cilium). Along the way: * Refactor HTTP(S) server configuration to use configurable port from env * Extend httpPort description * Changes to helm template
googleforgames · Jul 17, 2024 · 37aafc0 · 37aafc0
1 parent 91e3851
commit 37aafc0
Show file tree

Hide file tree

Showing 9 changed files with 61 additions and 17 deletions.
diff --git a/cmd/extensions/main.go b/cmd/extensions/main.go
@@ -68,6 +68,8 @@ const (
 	apiServerSustainedQPSFlag    = "api-server-qps"
 	apiServerBurstQPSFlag        = "api-server-qps-burst"
 	readinessShutdownDuration    = "readiness-shutdown-duration"
+	httpPort                     = "http-port"
+	webhookPort                  = "webhook-port"
 )
 
 var (
@@ -138,7 +140,7 @@ func main() {
 		logger.WithError(err).Fatal("Could not initialize cloud product")
 	}
 	// https server and the items that share the Mux for routing
-	httpsServer := https.NewServer(ctlConf.CertFile, ctlConf.KeyFile)
+	httpsServer := https.NewServer(ctlConf.CertFile, ctlConf.KeyFile, ctlConf.WebhookPort)
 	cancelTLS, err := httpsServer.WatchForCertificateChanges()
 	if err != nil {
 		logger.WithError(err).Fatal("Got an error while watching certificate changes")
@@ -150,7 +152,10 @@ func main() {
 	agonesInformerFactory := externalversions.NewSharedInformerFactory(agonesClient, defaultResync)
 	kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, defaultResync)
 
-	server := &httpserver.Server{Logger: logger}
+	server := &httpserver.Server{
+		Port:   ctlConf.HTTPPort,
+		Logger: logger,
+	}
 	var health healthcheck.Handler
 
 	// Stackdriver metrics
@@ -249,6 +254,8 @@ func parseEnvFlags() config {
 	viper.SetDefault(logDirFlag, "")
 	viper.SetDefault(logLevelFlag, "Info")
 	viper.SetDefault(logSizeLimitMBFlag, 10000) // 10 GB, will be split into 100 MB chunks
+	viper.SetDefault(httpPort, "8080")
+	viper.SetDefault(webhookPort, "8081")
 
 	pflag.String(keyFileFlag, viper.GetString(keyFileFlag), "Optional. Path to the key file")
 	pflag.String(certFileFlag, viper.GetString(certFileFlag), "Optional. Path to the crt file")
@@ -262,6 +269,8 @@ func parseEnvFlags() config {
 	pflag.Int32(numWorkersFlag, 64, "Number of controller workers per resource type")
 	pflag.Int32(apiServerSustainedQPSFlag, 100, "Maximum sustained queries per second to send to the API server")
 	pflag.Int32(apiServerBurstQPSFlag, 200, "Maximum burst queries per second to send to the API server")
+	pflag.String(httpPort, viper.GetString(httpPort), "Port for the HTTP server. Defaults to 8080, can also use HTTP_PORT env variable")
+	pflag.String(webhookPort, viper.GetString(webhookPort), "Port for the Webhook. Defaults to 8081, can also use WEBHOOK_PORT env variable")
 	pflag.String(logDirFlag, viper.GetString(logDirFlag), "If set, store logs in a given directory.")
 	pflag.Int32(logSizeLimitMBFlag, 1000, "Log file size limit in MB")
 	pflag.String(logLevelFlag, viper.GetString(logLevelFlag), "Agones Log level")
@@ -288,6 +297,8 @@ func parseEnvFlags() config {
 	runtime.Must(viper.BindEnv(logLevelFlag))
 	runtime.Must(viper.BindEnv(logDirFlag))
 	runtime.Must(viper.BindEnv(logSizeLimitMBFlag))
+	runtime.Must(viper.BindEnv(httpPort))
+	runtime.Must(viper.BindEnv(webhookPort))
 	runtime.Must(viper.BindEnv(allocationBatchWaitTime))
 	runtime.Must(viper.BindPFlags(pflag.CommandLine))
 	runtime.Must(viper.BindEnv(readinessShutdownDuration))
@@ -311,6 +322,8 @@ func parseEnvFlags() config {
 		LogDir:                    viper.GetString(logDirFlag),
 		LogLevel:                  viper.GetString(logLevelFlag),
 		LogSizeLimitMB:            int(viper.GetInt32(logSizeLimitMBFlag)),
+		HTTPPort:                  viper.GetString(httpPort),
+		WebhookPort:               viper.GetString(webhookPort),
 		AllocationBatchWaitTime:   viper.GetDuration(allocationBatchWaitTime),
 		ReadinessShutdownDuration: viper.GetDuration(readinessShutdownDuration),
 	}
@@ -333,6 +346,8 @@ type config struct {
 	LogDir                    string
 	LogLevel                  string
 	LogSizeLimitMB            int
+	HTTPPort                  string
+	WebhookPort               string
 	AllocationBatchWaitTime   time.Duration
 	ReadinessShutdownDuration time.Duration
 }

diff --git a/install/helm/agones/templates/extensions-deployment.yaml b/install/helm/agones/templates/extensions-deployment.yaml
@@ -41,7 +41,7 @@ spec:
 {{- end }}
 {{- if and (.Values.agones.metrics.prometheusServiceDiscovery) (.Values.agones.metrics.prometheusEnabled) }}
         prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
+        prometheus.io/port: {{ .Values.agones.extensions.http.port | quote }}
         prometheus.io/path: "/metrics"
 {{- end }}
 {{- if .Values.agones.extensions.annotations }}
@@ -93,6 +93,10 @@ spec:
       {{- end }}
       serviceAccountName: {{ .Values.agones.serviceaccount.controller.name }}
       terminationGracePeriodSeconds: {{ mul .Values.agones.extensions.readiness.periodSeconds .Values.agones.extensions.readiness.failureThreshold 3 }}
+      {{- if .Values.agones.extensions.hostNetwork }}
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      {{- end }}
       containers:
       - name: agones-extensions
         image: "{{ .Values.agones.image.registry }}/{{ .Values.agones.image.extensions.name}}:{{ default .Values.agones.image.tag .Values.agones.image.extensions.tag }}"
@@ -107,7 +111,7 @@ spec:
         - name: STACKDRIVER_EXPORTER
           value: {{ .Values.agones.metrics.stackdriverEnabled | quote }}
         - name: STACKDRIVER_LABELS
-          value: {{ .Values.agones.metrics.stackdriverLabels | quote }}       
+          value: {{ .Values.agones.metrics.stackdriverLabels | quote }}
         - name: GCP_PROJECT_ID
           value: {{ .Values.agones.metrics.stackdriverProjectID | quote }}
         - name: NUM_WORKERS
@@ -142,11 +146,15 @@ spec:
           value: "agones-extensions"
         - name: READINESS_SHUTDOWN_DURATION
           value: {{ mul .Values.agones.extensions.readiness.periodSeconds .Values.agones.extensions.readiness.failureThreshold 2 }}s
+        - name: WEBHOOK_PORT
+          value: {{ .Values.agones.extensions.webhooks.port | quote }}
+        - name: HTTP_PORT
+          value: {{ .Values.agones.extensions.http.port | quote }}
         ports:
         - name: webhooks
-          containerPort: 8081
+          containerPort: {{ .Values.agones.extensions.webhooks.port }}
         - name: http
-          containerPort: 8080
+          containerPort: {{ .Values.agones.extensions.http.port }}
         livenessProbe:
           httpGet:
             path: /live
@@ -158,7 +166,7 @@ spec:
         readinessProbe:
           httpGet:
             path: /ready
-            port: 8080
+            port: {{ .Values.agones.extensions.http.port }}
           initialDelaySeconds: {{ .Values.agones.extensions.readiness.initialDelaySeconds }}
           periodSeconds: {{ .Values.agones.extensions.readiness.periodSeconds }}
           failureThreshold: {{ .Values.agones.extensions.readiness.failureThreshold }}

diff --git a/install/helm/agones/templates/extensions-metrics-service.yaml b/install/helm/agones/templates/extensions-metrics-service.yaml
@@ -28,5 +28,5 @@ spec:
     agones.dev/role: extensions
   ports:
     - name: metrics
-      port: {{ .Values.agones.controller.http.port }}
+      port: {{ .Values.agones.extensions.http.port }}
       targetPort: http
diff --git a/install/helm/agones/templates/service.yaml b/install/helm/agones/templates/service.yaml
@@ -32,5 +32,5 @@ spec:
       port: 443
       targetPort: webhooks
     - name: web
-      port: {{ .Values.agones.controller.http.port }}
+      port: {{ .Values.agones.extensions.http.port }}
       targetPort: http
diff --git a/install/helm/agones/values.yaml b/install/helm/agones/values.yaml
@@ -100,6 +100,15 @@ agones:
       #   memory: 256Mi
     nodeSelector: {}
     annotations: {}
+  # Determines if the Agones extensions should operate in hostNetwork mode.
+  #
+  # This setting is necessary for certain managed Kubernetes clusters (e.g., AWS EKS) that use custom
+  # CNI plugins (such as Calico or Cilium) because the AWS-managed control plane cannot communicate
+  # with pod IP CIDRs.
+  #
+  # Note: The default port may conflicts with others on the host network. Therefore, if
+  # running in hostNetwork mode, you should change `http.port` and `webhooks.port` to an available port.
+    hostNetwork: false
     tolerations:
     - key: "agones.dev/agones-system"
       operator: "Equal"
@@ -125,6 +134,8 @@ agones:
     numWorkers: 100
     apiServerQPS: 400
     apiServerQPSBurst: 500
+    webhooks:
+      port: 8081
     http:
       port: 8080
     healthCheck:

diff --git a/install/yaml/install.yaml b/install/yaml/install.yaml
@@ -17237,7 +17237,7 @@ spec:
         - name: STACKDRIVER_EXPORTER
           value: "false"
         - name: STACKDRIVER_LABELS
-          value: ""       
+          value: ""
         - name: GCP_PROJECT_ID
           value: ""
         - name: NUM_WORKERS
@@ -17270,6 +17270,10 @@ spec:
           value: "agones-extensions"
         - name: READINESS_SHUTDOWN_DURATION
           value: 18s
+        - name: WEBHOOK_PORT
+          value: "8081"
+        - name: HTTP_PORT
+          value: "8080"
         ports:
         - name: webhooks
           containerPort: 8081

diff --git a/pkg/util/https/server.go b/pkg/util/https/server.go
@@ -53,16 +53,18 @@ type Server struct {
 	tls        tls
 	certFile   string
 	keyFile    string
+	port       string
 }
 
 // NewServer returns a Server instance.
-func NewServer(certFile, keyFile string) *Server {
+func NewServer(certFile, keyFile string, port string) *Server {
 	mux := http.NewServeMux()
 
 	wh := &Server{
 		Mux:      mux,
 		certFile: certFile,
 		keyFile:  keyFile,
+		port:     port,
 	}
 	wh.logger = runtime.NewLoggerWithType(wh)
 	wh.setupServer()
@@ -73,7 +75,7 @@ func NewServer(certFile, keyFile string) *Server {
 
 func (s *Server) setupServer() {
 	s.tls = &http.Server{
-		Addr:    ":8081",
+		Addr:    ":" + s.port,
 		Handler: s.Mux,
 		TLSConfig: &cryptotls.Config{
 			GetCertificate: s.getCertificate,
@@ -129,15 +131,15 @@ func (s *Server) Run(ctx context.Context, _ int) error {
 		_ = s.tls.Shutdown(context.Background())
 	}()
 
-	s.logger.WithField("server", s).Infof("https server started")
+	s.logger.WithField("server", s).Infof("https server started on port :" + s.port)
 
 	err := s.tls.ListenAndServeTLS(s.certFile, s.keyFile)
 	if err == http.ErrServerClosed {
 		s.logger.WithError(err).Info("https server closed")
 		return nil
 	}
 
-	return errors.Wrap(err, "Could not listen on :8081")
+	return errors.Wrap(err, "Could not listen on :"+s.port)
 }
 
 // defaultHandler Handles all the HTTP requests

diff --git a/pkg/util/https/server_test.go b/pkg/util/https/server_test.go
@@ -41,7 +41,7 @@ func (ts *testServer) ListenAndServeTLS(certFile, keyFile string) error {
 func TestServerRun(t *testing.T) {
 	t.Parallel()
 
-	s := NewServer("", "")
+	s := NewServer("", "", "")
 	ts := &testServer{server: httptest.NewUnstartedServer(s.Mux)}
 	s.tls = ts
 

diff --git a/pkg/util/httpserver/server.go b/pkg/util/httpserver/server.go
@@ -31,15 +31,19 @@ import (
 //nolint:govet // ignore field alignment complaint, this is a singleton
 type Server struct {
 	http.ServeMux
+	Port string
 
 	Logger *logrus.Entry
 }
 
 // Run runs an http server on port :8080.
 func (s *Server) Run(ctx context.Context, _ int) error {
 	s.Logger.Info("Starting http server...")
+	if s.Port == "" {
+		s.Port = "8080"
+	}
 	srv := &http.Server{
-		Addr:    ":8080",
+		Addr:    ":" + s.Port,
 		Handler: s,
 	}
 	go func() {
@@ -51,7 +55,7 @@ func (s *Server) Run(ctx context.Context, _ int) error {
 		if err == http.ErrServerClosed {
 			s.Logger.WithError(err).Info("http server closed")
 		} else {
-			wrappedErr := errors.Wrap(err, "Could not listen on :8080")
+			wrappedErr := errors.Wrap(err, "Could not listen on :"+s.Port)
 			runtime.HandleError(s.Logger.WithError(wrappedErr), wrappedErr)
 		}
 	}