Sync Pod host ports back to GameServer in GCP

cmd/controller/main.go

@@ -28,6 +28,7 @@ import (
 	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
 	"agones.dev/agones/pkg/client/clientset/versioned"
 	"agones.dev/agones/pkg/client/informers/externalversions"
+	"agones.dev/agones/pkg/cloudproduct"
 	"agones.dev/agones/pkg/fleetautoscalers"
 	"agones.dev/agones/pkg/fleets"
 	"agones.dev/agones/pkg/gameserverallocations"
@@ -78,6 +79,7 @@ const (
 	logSizeLimitMBFlag           = "log-size-limit-mb"
 	kubeconfigFlag               = "kubeconfig"
 	allocationBatchWaitTime      = "allocation-batch-wait-time"
+	cloudProductFlag             = "cloud-product"
 	defaultResync                = 30 * time.Second
 )
 
@@ -105,6 +107,7 @@ func setupLogging(logDir string, logSizeLimitMB int) {
 
 // main starts the operator for the gameserver CRD
 func main() {
+	ctx := signals.NewSigKillContext()
 	ctlConf := parseEnvFlags()
 
 	if ctlConf.LogDir != "" {
@@ -154,6 +157,11 @@ func main() {
 		logger.WithError(err).Fatal("Could not create the agones api clientset")
 	}
 
+	cloudProduct, err := cloudproduct.New(ctx, ctlConf.CloudProduct, kubeClient)
+	if err != nil {
+		logger.WithError(err).Fatal("Could not initialize cloud provider")
+	}
+
 	// https server and the items that share the Mux for routing
 	httpsServer := https.NewServer(ctlConf.CertFile, ctlConf.KeyFile)
 	wh := webhooks.NewWebHook(httpsServer.Mux)
@@ -207,7 +215,7 @@ func main() {
 		ctlConf.MinPort, ctlConf.MaxPort, ctlConf.SidecarImage, ctlConf.AlwaysPullSidecar,
 		ctlConf.SidecarCPURequest, ctlConf.SidecarCPULimit,
 		ctlConf.SidecarMemoryRequest, ctlConf.SidecarMemoryLimit, ctlConf.SdkServiceAccount,
-		kubeClient, kubeInformerFactory, extClient, agonesClient, agonesInformerFactory)
+		kubeClient, kubeInformerFactory, extClient, agonesClient, agonesInformerFactory, cloudProduct)
 	gsSetController := gameserversets.NewController(wh, health, gsCounter,
 		kubeClient, extClient, agonesClient, agonesInformerFactory)
 	fleetController := fleets.NewController(wh, health, kubeClient, extClient, agonesClient, agonesInformerFactory)
@@ -219,8 +227,6 @@ func main() {
 	rs = append(rs,
 		httpsServer, gsCounter, gsController, gsSetController, fleetController, fasController, gasController, server)
 
-	ctx := signals.NewSigKillContext()
-
 	kubeInformerFactory.Start(ctx.Done())
 	agonesInformerFactory.Start(ctx.Done())
 
@@ -264,6 +270,7 @@ func parseEnvFlags() config {
 	viper.SetDefault(logDirFlag, "")
 	viper.SetDefault(logLevelFlag, "Info")
 	viper.SetDefault(logSizeLimitMBFlag, 10000) // 10 GB, will be split into 100 MB chunks
+	viper.SetDefault(cloudProductFlag, cloudproduct.AutoDetect)
 
 	pflag.String(sidecarImageFlag, viper.GetString(sidecarImageFlag), "Flag to overwrite the GameServer sidecar image that is used. Can also use SIDECAR env variable")
 	pflag.String(sidecarCPULimitFlag, viper.GetString(sidecarCPULimitFlag), "Flag to overwrite the GameServer sidecar container's cpu limit. Can also use SIDECAR_CPU_LIMIT env variable")
@@ -288,6 +295,7 @@ func parseEnvFlags() config {
 	pflag.Int32(logSizeLimitMBFlag, 1000, "Log file size limit in MB")
 	pflag.String(logLevelFlag, viper.GetString(logLevelFlag), "Agones Log level")
 	pflag.Duration(allocationBatchWaitTime, viper.GetDuration(allocationBatchWaitTime), "Flag to configure the waiting period between allocations batches")
+	pflag.String(cloudProductFlag, viper.GetString(cloudProductFlag), "Cloud product. Set to 'auto' to auto-detect, set to 'generic' to force generic behavior, set to 'gke-autopilot' for GKE Autopilot. Can also use CLOUD_PRODUCT env variable.")
 	runtime.FeaturesBindFlags()
 	pflag.Parse()
 
@@ -315,6 +323,7 @@ func parseEnvFlags() config {
 	runtime.Must(viper.BindEnv(logDirFlag))
 	runtime.Must(viper.BindEnv(logSizeLimitMBFlag))
 	runtime.Must(viper.BindEnv(allocationBatchWaitTime))
+	runtime.Must(viper.BindEnv(cloudProductFlag))
 	runtime.Must(viper.BindPFlags(pflag.CommandLine))
 	runtime.Must(runtime.FeaturesBindEnv())
 
@@ -364,6 +373,7 @@ func parseEnvFlags() config {
 		LogSizeLimitMB:          int(viper.GetInt32(logSizeLimitMBFlag)),
 		StackdriverLabels:       viper.GetString(stackdriverLabels),
 		AllocationBatchWaitTime: viper.GetDuration(allocationBatchWaitTime),
+		CloudProduct:            viper.GetString(cloudProductFlag),
 	}
 }
 
@@ -392,6 +402,7 @@ type config struct {
 	LogLevel                string
 	LogSizeLimitMB          int
 	AllocationBatchWaitTime time.Duration
+	CloudProduct            string
 }
 
 // validate ensures the ctlConfig data is valid.

install/helm/agones/templates/controller.yaml

@@ -115,6 +115,8 @@ spec:
           value: {{ .Values.agones.featureGates | quote }}
         - name: ALLOCATION_BATCH_WAIT_TIME
           value: {{ .Values.agones.controller.allocationBatchWaitTime | quote }}
+        - name: CLOUD_PRODUCT
+          value: {{ .Values.agones.cloudProduct | quote }}
 {{- if .Values.agones.controller.persistentLogs }}
         - name: LOG_DIR
           value: "/home/agones/logs"

install/helm/agones/templates/serviceaccounts/controller.yaml

@@ -50,6 +50,11 @@ rules:
 - apiGroups: [""]
   resources: ["nodes", "secrets"]
   verbs: ["list", "watch"]
+{{- if eq .Values.agones.cloudProduct "auto" }}
+- apiGroups: ["admissionregistration.k8s.io"] # only needed for cloudProduct detection
+  resources: ["mutatingwebhookconfigurations"]
+  verbs: ["get"]
+{{- end}}
 - apiGroups: ["apiextensions.k8s.io"]
   resources: ["customresourcedefinitions"]
   verbs: ["get"]

install/helm/agones/values.yaml

@@ -44,6 +44,7 @@ agones:
       annotations: {}
   createPriorityClass: true
   priorityClassName: agones-system
+  cloudProduct: "auto"
   controller:
     resources: {}
       # requests:

install/yaml/install.yaml

@@ -14342,6 +14342,9 @@ rules:
 - apiGroups: [""]
   resources: ["nodes", "secrets"]
   verbs: ["list", "watch"]
+- apiGroups: ["admissionregistration.k8s.io"] # only needed for cloudProduct detection
+  resources: ["mutatingwebhookconfigurations"]
+  verbs: ["get"]
 - apiGroups: ["apiextensions.k8s.io"]
   resources: ["customresourcedefinitions"]
   verbs: ["get"]
@@ -14726,6 +14729,8 @@ spec:
           value: ""
         - name: ALLOCATION_BATCH_WAIT_TIME
           value: "500ms"
+        - name: CLOUD_PRODUCT
+          value: "auto"
         - name: LOG_DIR
           value: "/home/agones/logs"
         - name: LOG_SIZE_LIMIT_MB

pkg/cloudproduct/cloudproduct.go

@@ -0,0 +1,84 @@
+// Copyright 2022 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudproduct
+
+import (
+	"context"
+	"fmt"
+
+	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
+	"agones.dev/agones/pkg/cloudproduct/generic"
+	"agones.dev/agones/pkg/cloudproduct/gke"
+	"agones.dev/agones/pkg/util/runtime"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+// CloudProduct provides a generic interface that abstracts cloud product
+// specific functionality. Users should call New() to instantiate a
+// specific cloud product interface.
+type CloudProduct interface {
+	// SyncPodPortsToGameServer runs after a Pod has been assigned to a Node and before we sync
+	// Pod host ports to the GameServer status.
+	SyncPodPortsToGameServer(*agonesv1.GameServer, *corev1.Pod) error
+}
+
+const (
+	// If --cloud-product=auto, auto-detect
+	AutoDetect = "auto"
+
+	genericProduct = "generic"
+)
+
+var (
+	logger           = runtime.NewLoggerWithSource("cloudproduct")
+	productDetectors = []func(context.Context, *kubernetes.Clientset) string{gke.Detect}
+)
+
+// New instantiates a new CloudProduct interface by product name.
+func New(ctx context.Context, product string, kc *kubernetes.Clientset) (CloudProduct, error) {
+	product = autoDetect(ctx, product, kc)
+
+	switch product {
+	case "gke-autopilot":
+		return gke.Autopilot()
+	case genericProduct:
+		return generic.New()
+	}
+	return nil, fmt.Errorf("unknown cloud product: %q", product)
+}
+
+func autoDetect(ctx context.Context, product string, kc *kubernetes.Clientset) string {
+	if product != AutoDetect {
+		logger.Infof("Cloud product forced to %q, skipping auto-detection", product)
+		return product
+	}
+	for _, detect := range productDetectors {
+		product = detect(ctx, kc)
+		if product != "" {
+			logger.Infof("Cloud product detected as %q", product)
+			return product
+		}
+	}
+	logger.Infof("Cloud product defaulted to %q", genericProduct)
+	return genericProduct
+}
+
+// MustNewGeneric returns the "generic" cloud product, panicking if an error is encountered.
+func MustNewGeneric(ctx context.Context) CloudProduct {
+	c, err := New(ctx, genericProduct, nil)
+	runtime.Must(err)
+	return c
+}

pkg/cloudproduct/doc.go

@@ -0,0 +1,16 @@
+// Copyright 2022 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cloudproduct provides an abstraction layer to product specific functionality
+package cloudproduct

pkg/cloudproduct/generic/doc.go

@@ -0,0 +1,17 @@
+// Copyright 2022 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package generic provides the generic cloud provider. The generic cloud provider
+// should be usable on all clouds, but provides no special functionality.
+package generic

pkg/cloudproduct/generic/generic.go

@@ -0,0 +1,25 @@
+// Copyright 2022 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package generic
+
+import (
+	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func New() (*generic, error) { return &generic{}, nil }
+
+type generic struct{}
+
+func (*generic) SyncPodPortsToGameServer(*agonesv1.GameServer, *corev1.Pod) error { return nil }

pkg/cloudproduct/gke/doc.go

@@ -0,0 +1,16 @@
+// Copyright 2022 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package gke provides the GKE cloud product interfaces for GKE Standard and Autopilot
+package gke

pkg/cloudproduct/gke/gke.go

@@ -0,0 +1,84 @@
+// Copyright 2022 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package gke
+
+import (
+	"context"
+	"encoding/json"
+
+	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
+	"agones.dev/agones/pkg/util/runtime"
+	"cloud.google.com/go/compute/metadata"
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+const (
+	workloadDefaulterWebhook     = "workload-defaulter.config.common-webhooks.networking.gke.io"
+	noWorkloadDefaulter          = "failed to get MutatingWebhookConfigurations/workload-defaulter.config.common-webhooks.networking.gke.io (error expected if not on GKE Autopilot)"
+	hostPortAssignmentAnnotation = "autopilot.gke.io/host-port-assignment"
+)
+
+var logger = runtime.NewLoggerWithSource("gke")
+
+type gkeAutopilot struct{}
+
+// hostPortAssignment is the JSON structure of the `host-port-assignment` annotation
+type hostPortAssignment struct {
+	Min           int32           `json:"min,omitempty"`
+	Max           int32           `json:"max,omitempty"`
+	PortsAssigned map[int32]int32 `json:"portsAssigned,omitempty"` // old -> new
+}
+
+func Detect(ctx context.Context, kc *kubernetes.Clientset) string {
+	if !metadata.OnGCE() {
+		return ""
+	}
+	// Look for the workload defaulter - this is the current best method to detect Autopilot
+	if _, err := kc.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(
+		ctx, workloadDefaulterWebhook, metav1.GetOptions{}); err != nil {
+		logger.WithError(err).WithField("reason", noWorkloadDefaulter).Info(
+			"Assuming GKE Standard and defaulting to generic provider")
+		return "" // GKE standard, but we don't need an interface for it just yet.
+	}
+	logger.Info("Running on GKE Autopilot (skip detection with --cloud-product=gke-autopilot)")
+	return "gke-autopilot"
+}
+
+func Autopilot() (*gkeAutopilot, error) { return &gkeAutopilot{}, nil }
+
+func (*gkeAutopilot) SyncPodPortsToGameServer(gs *agonesv1.GameServer, pod *corev1.Pod) error {
+	// If applyGameServerAddressAndPort has already filled in Status, SyncPodPortsToGameServer
+	// has already run. Skip syncing from the Pod again - this avoids having to reason
+	// about whether we're re-applying the old->new mapping.
+	if len(gs.Status.Ports) == len(gs.Spec.Ports) {
+		return nil
+	}
+	annotation, ok := pod.ObjectMeta.Annotations[hostPortAssignmentAnnotation]
+	if !ok {
+		return nil
+	}
+	var hpa hostPortAssignment
+	if err := json.Unmarshal([]byte(annotation), &hpa); err != nil {
+		return errors.Wrapf(err, "could not unmarshal annotation %s (value %q)", hostPortAssignmentAnnotation, annotation)
+	}
+	for i, p := range gs.Spec.Ports {
+		if newPort, ok := hpa.PortsAssigned[p.HostPort]; ok {
+			gs.Spec.Ports[i].HostPort = newPort
+		}
+	}
+	return nil
+}