Enable SplitControllerAndExtensions for e2e testing and changed value…

…s yaml for deployment (#2923)
googleforgames · Jan 30, 2023 · 5ca4a36 · 5ca4a36
1 parent 7d814b2
commit 5ca4a36
Show file tree

Hide file tree

Showing 9 changed files with 51 additions and 51 deletions.
diff --git a/build/Makefile b/build/Makefile
@@ -64,7 +64,7 @@ KIND_CONTAINER_NAME=$(KIND_PROFILE)-control-plane
 GS_TEST_IMAGE ?= us-docker.pkg.dev/agones-images/examples/simple-game-server:0.14
 
 # Enable all alpha feature gates. Keep in sync with `false` (alpha) entries in pkg/util/runtime/features.go:featureDefaults
-ALPHA_FEATURE_GATES ?= "PlayerAllocationFilter=true&PlayerTracking=true&ResetMetricsOnDelete=true&SafeToEvict=true&PodHostname=true&Example=true"
+ALPHA_FEATURE_GATES ?= "PlayerAllocationFilter=true&PlayerTracking=true&ResetMetricsOnDelete=true&SafeToEvict=true&PodHostname=true&SplitControllerAndExtensions=true&Example=true"
 
 # Build with Windows support
 WITH_WINDOWS=1
@@ -391,7 +391,6 @@ install: LOG_LEVEL := "debug"
 install: EXTERNAL_IP ?= $(shell $(DOCKER_RUN) kubectl get services agones-allocator -n agones-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
 install: FEATURE_GATES ?= $(ALPHA_FEATURE_GATES)
 install: SERVICE_MONITOR := false
-install: CONTROLLER_SAFE_TO_EVICT := false # TODO(#2797): Remove when `SplitControllerAndExtensions` => safe-to-evict=true
 install: HELM_ARGS ?=
 install: $(ensure-build-image) install-custom-pull-secret
 	-$(DOCKER_RUN) bash -c '[[ $$(helm status agones -n agones-system --output json | jq -r ".info.status") =~ (failed|pending-.*) ]] && helm uninstall agones --namespace=agones-system || true'
@@ -408,7 +407,6 @@ install: $(ensure-build-image) install-custom-pull-secret
 		--set agones.featureGates=$(FEATURE_GATES) \
 		--set agones.allocator.service.loadBalancerIP=$(EXTERNAL_IP) \
 		--set agones.metrics.serviceMonitor.enabled=$(SERVICE_MONITOR) \
-		--set agones.controller.safeToEvict=$(CONTROLLER_SAFE_TO_EVICT) \
 		$(HELM_ARGS) \
 		agones $(mount_path)/install/helm/agones/
 

diff --git a/build/e2e-image/e2e.sh b/build/e2e-image/e2e.sh
@@ -22,8 +22,7 @@ echo $FEATURES
 echo $REGISTRY
 set -e
 echo "installing current release"
-# TODO(#2797): Remove when `SplitControllerAndExtensions` => safe-to-evict=true
-DOCKER_RUN= make install FEATURE_GATES='"'$FEATURES'"' REGISTRY='"'$REGISTRY'"' CONTROLLER_SAFE_TO_EVICT=true
+DOCKER_RUN= make install FEATURE_GATES='"'$FEATURES'"' REGISTRY='"'$REGISTRY'"' 
 echo "starting e2e test"
 DOCKER_RUN= make test-e2e ARGS=-parallel=16 E2E_USE_GOTESTSUM=true GOTESTSUM_VERBOSE=true FEATURE_GATES='"'$FEATURES'"' CLOUD_PRODUCT='"'$CLOUD_PRODUCT'"'
 echo "completed e2e test"
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
@@ -286,7 +286,7 @@ steps:
 
 - name: 'e2e-runner'
   args:
-    - 'CustomFasSyncInterval=false&SDKGracefulTermination=false&StateAllocationFilter=false&PlayerAllocationFilter=true&PlayerTracking=true&ResetMetricsOnDelete=true&SafeToEvict=true&PodHostname=true&Example=true'
+    - 'CustomFasSyncInterval=false&SDKGracefulTermination=false&StateAllocationFilter=false&PlayerAllocationFilter=true&PlayerTracking=true&ResetMetricsOnDelete=true&SafeToEvict=true&PodHostname=true&SplitControllerAndExtensions=true&Example=true'
     - 'generic'
     - 'e2e-test-cluster'
     - 'us-west1-c'
@@ -317,7 +317,7 @@ steps:
 
 - name: 'e2e-runner'
   args:
-    - 'CustomFasSyncInterval=false&SDKGracefulTermination=false&StateAllocationFilter=false&PlayerAllocationFilter=true&PlayerTracking=true&ResetMetricsOnDelete=true&SafeToEvict=true&PodHostname=true&Example=true'
+    - 'CustomFasSyncInterval=false&SDKGracefulTermination=false&StateAllocationFilter=false&PlayerAllocationFilter=true&PlayerTracking=true&ResetMetricsOnDelete=true&SafeToEvict=true&PodHostname=true&SplitControllerAndExtensions=true&Example=true'
     - 'gke-autopilot'
     - 'gke-autopilot-e2e-test-cluster-1-24'
     - 'us-west1'
@@ -328,12 +328,12 @@ steps:
     - e2e-wait-to-become-leader
 
 #
-# Run the e2e tests on GKE Autopilot without FeatureGates, except SafeToEvict, which we need on Autopilot
+# Run the e2e tests on GKE Autopilot without FeatureGates, except SafeToEvict and SplitControllerAndExtensions, which we need on Autopilot
 #
 
 - name: 'e2e-runner'
   args:
-    - 'SafeToEvict=true'
+    - 'SafeToEvict=true&SplitControllerAndExtensions=true'
     - 'gke-autopilot'
     - 'gke-autopilot-e2e-test-cluster-1-24'
     - 'us-west1'

diff --git a/install/helm/agones/defaultfeaturegates.yaml b/install/helm/agones/defaultfeaturegates.yaml
@@ -25,9 +25,9 @@ PlayerTracking: false
 ResetMetricsOnDelete: false
 SafeToEvict: false
 PodHostname: false
+SplitControllerAndExtensions: false
 
 # Pre-Alpha features
-SplitControllerAndExtensions: false
 
 # Example feature
 Example: false

diff --git a/install/helm/agones/templates/controller.yaml b/install/helm/agones/templates/controller.yaml
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+{{- $featureGates := include "agones.featureGates" . | fromYaml }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -36,7 +37,9 @@ spec:
   template:
     metadata:
       annotations:
+{{- if not $featureGates.SplitControllerAndExtensions }}
         cluster-autoscaler.kubernetes.io/safe-to-evict: {{ .Values.agones.controller.safeToEvict | quote }}
+{{- end }}
 {{- if .Values.agones.controller.generateTLS }}
         revision/tls-cert: {{ .Release.Revision | quote }}
 {{- end }}

diff --git a/install/helm/agones/templates/extensions-deployment.yaml b/install/helm/agones/templates/extensions-deployment.yaml
@@ -38,30 +38,29 @@ spec:
   template:
     metadata:
       annotations:
-        cluster-autoscaler.kubernetes.io/safe-to-evict: {{ .Values.agones.controller.safeToEvict | quote }}
-{{- if .Values.agones.controller.generateTLS }}
+{{- if .Values.agones.extensions.generateTLS }}
         revision/tls-cert: {{ .Release.Revision | quote }}
 {{- end }}
-{{- if .Values.agones.controller.annotations }}
-{{- toYaml .Values.agones.controller.annotations | nindent 8 }}
+{{- if .Values.agones.extensions.annotations }}
+{{- toYaml .Values.agones.extensions.annotations | nindent 8 }}
 {{- end }}
       labels:
         agones.dev/role: extensions
         app: {{ template "agones.name" . }}
         release: {{ .Release.Name }}
         heritage: {{ .Release.Service }}
     spec:
-      {{- if .Values.agones.controller.affinity }}
+      {{- if .Values.agones.extensions.affinity }}
       affinity:
-{{ toYaml .Values.agones.controller.affinity | indent 8 }}
+{{ toYaml .Values.agones.extensions.affinity | indent 8 }}
       {{- end }}
-      {{- if .Values.agones.controller.nodeSelector }}
+      {{- if .Values.agones.extensions.nodeSelector }}
       nodeSelector:
-{{ toYaml .Values.agones.controller.nodeSelector | indent 8 }}
+{{ toYaml .Values.agones.extensions.nodeSelector | indent 8 }}
       {{- end }}
-      {{- if .Values.agones.controller.tolerations }}
+      {{- if .Values.agones.extensions.tolerations }}
       tolerations:
-{{ toYaml .Values.agones.controller.tolerations | indent 8 }}
+{{ toYaml .Values.agones.extensions.tolerations | indent 8 }}
       {{- end }}
       {{- if .Values.agones.createPriorityClass }}
       priorityClassName: {{ .Values.agones.priorityClassName }}
@@ -70,29 +69,29 @@ spec:
       containers:
       - name: agones-extensions
         image: "{{ .Values.agones.image.registry }}/{{ .Values.agones.image.extensions.name}}:{{ default .Values.agones.image.tag .Values.agones.image.extensions.tag }}"
-        imagePullPolicy: {{ .Values.agones.image.controller.pullPolicy }}
+        imagePullPolicy: {{ .Values.agones.image.extensions.pullPolicy }}
         env:
         - name: GCP_PROJECT_ID
           value: {{ .Values.agones.metrics.stackdriverProjectID | quote }}
         - name: NUM_WORKERS
-          value: {{ .Values.agones.controller.numWorkers | quote }}
+          value: {{ .Values.agones.extensions.numWorkers | quote }}
         - name: API_SERVER_QPS
-          value: {{ .Values.agones.controller.apiServerQPS | quote }}
+          value: {{ .Values.agones.extensions.apiServerQPS | quote }}
         - name: API_SERVER_QPS_BURST
-          value: {{ .Values.agones.controller.apiServerQPSBurst | quote }}
+          value: {{ .Values.agones.extensions.apiServerQPSBurst | quote }}
         - name: LOG_LEVEL
-          value: {{ .Values.agones.controller.logLevel | quote }}
+          value: {{ .Values.agones.extensions.logLevel | quote }}
         - name: FEATURE_GATES
           value: {{ .Values.agones.featureGates | quote }}
         - name: ALLOCATION_BATCH_WAIT_TIME
-          value: {{ .Values.agones.controller.allocationBatchWaitTime | quote }}
+          value: {{ .Values.agones.extensions.allocationBatchWaitTime | quote }}
         - name: CLOUD_PRODUCT
           value: {{ .Values.agones.cloudProduct | quote }}
-{{- if .Values.agones.controller.persistentLogs }}
+{{- if .Values.agones.extensions.persistentLogs }}
         - name: LOG_DIR
           value: "/home/agones/logs"
         - name: LOG_SIZE_LIMIT_MB
-          value: {{ .Values.agones.controller.persistentLogsSizeLimitMB | quote }}
+          value: {{ .Values.agones.extensions.persistentLogsSizeLimitMB | quote }}
 {{- end }}
         - name: POD_NAME
           valueFrom:
@@ -113,13 +112,13 @@ spec:
           httpGet:
             path: /live
             port: http
-          initialDelaySeconds: {{ .Values.agones.controller.healthCheck.initialDelaySeconds }}
-          periodSeconds: {{ .Values.agones.controller.healthCheck.periodSeconds }}
-          failureThreshold: {{ .Values.agones.controller.healthCheck.failureThreshold }}
-          timeoutSeconds: {{ .Values.agones.controller.healthCheck.timeoutSeconds }}
+          initialDelaySeconds: {{ .Values.agones.extensions.healthCheck.initialDelaySeconds }}
+          periodSeconds: {{ .Values.agones.extensions.healthCheck.periodSeconds }}
+          failureThreshold: {{ .Values.agones.extensions.healthCheck.failureThreshold }}
+          timeoutSeconds: {{ .Values.agones.extensions.healthCheck.timeoutSeconds }}
         resources:
-{{- if .Values.agones.controller.resources }}
-{{ toYaml .Values.agones.controller.resources | indent 10 }}
+{{- if .Values.agones.extensions.resources }}
+{{ toYaml .Values.agones.extensions.resources | indent 10 }}
 {{- else}}
           limits:
             ephemeral-storage: {{ add .Values.agones.controller.persistentLogsSizeLimitMB 100 | printf "%dMi" }}
@@ -130,7 +129,7 @@ spec:
         - name: certs
           mountPath: /home/agones/certs
           readOnly: true
-{{- if .Values.agones.controller.persistentLogs }}
+{{- if .Values.agones.extensions.persistentLogs }}
         - name: logs
           mountPath: /home/agones/logs
           readOnly: false
@@ -139,17 +138,17 @@ spec:
       - name: certs
         secret:
           secretName: {{ template "agones.fullname" . }}-cert
-{{- if .Values.agones.controller.customCertSecretPath }}
+{{- if .Values.agones.extensions.customCertSecretPath }}
           items:
-{{- toYaml .Values.agones.controller.customCertSecretPath | nindent 10 }}
+{{- toYaml .Values.agones.extensions.customCertSecretPath | nindent 10 }}
 {{- end }}
-{{- if .Values.agones.controller.persistentLogs }}
+{{- if .Values.agones.extensions.persistentLogs }}
       - name: logs
         emptyDir: {}
 {{- end }}
-{{- if .Values.agones.image.controller.pullSecret }}
+{{- if .Values.agones.image.extensions.pullSecret }}
       imagePullSecrets:
-        - name: {{.Values.agones.image.controller.pullSecret}}
+        - name: {{.Values.agones.image.extensions.pullSecret}}
 {{- end }}
 ---
 apiVersion: policy/v1

diff --git a/install/helm/agones/values.yaml b/install/helm/agones/values.yaml
@@ -45,7 +45,7 @@ agones:
   createPriorityClass: true
   priorityClassName: agones-system
   cloudProduct: "auto"
-  controller:
+  controller: &controllerValues
     resources: {}
       # requests:
       #   cpu: 1
@@ -95,6 +95,7 @@ agones:
       timeoutSeconds: 1
     allocationBatchWaitTime: 500ms
   extensions:
+    <<: *controllerValues
     pdb:
         minAvailable: 1
     replicas: 2
@@ -223,6 +224,7 @@ agones:
     # extensions settings ignored unless `SplitControllerAndExtensions` feature gate is enabled
     extensions:
       name: agones-extensions
+      pullPolicy: IfNotPresent
     sdk:
       name: agones-sdk
       cpuRequest: 30m

diff --git a/install/yaml/install.yaml b/install/yaml/install.yaml
@@ -15505,7 +15505,6 @@ spec:
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 apiVersion: apps/v1
 kind: Deployment
 metadata:

diff --git a/pkg/util/runtime/features.go b/pkg/util/runtime/features.go
@@ -60,12 +60,12 @@ const (
 	// FeaturePodHostname enables the Pod Hostname being assigned the name of the GameServer
 	FeaturePodHostname = "PodHostname"
 
-	////////////////
-	// "Pre"-Alpha features
-
 	// FeatureSplitControllerAndExtensions is a feature flag that will split agones-controller into two deployments
 	FeatureSplitControllerAndExtensions = "SplitControllerAndExtensions"
 
+	////////////////
+	// "Pre"-Alpha features
+
 	////////////////
 	// Example feature
 
@@ -106,14 +106,14 @@ var (
 		FeatureStateAllocationFilter:  true,
 
 		// Alpha features
-		FeaturePlayerAllocationFilter: false,
-		FeaturePlayerTracking:         false,
-		FeatureResetMetricsOnDelete:   false,
-		FeatureSafeToEvict:            false,
-		FeaturePodHostname:            false,
+		FeaturePlayerAllocationFilter:       false,
+		FeaturePlayerTracking:               false,
+		FeatureResetMetricsOnDelete:         false,
+		FeatureSafeToEvict:                  false,
+		FeaturePodHostname:                  false,
+		FeatureSplitControllerAndExtensions: false,
 
 		// Pre-Alpha features
-		FeatureSplitControllerAndExtensions: false,
 
 		// Example feature
 		FeatureExample: false,