Move GKEAutopilotExtendedDurationPods to Alpha in 1.28+ (#3729)

With 1.28+, we can comfortably support extended duration pods in
Agones on Autopilot. This change moves
`GKEAutopilotExtendedDurationPods` to Alpha, but documents that
it is supported in 1.28+ only. Additionally:

* I changed the e2e testing to support changing the gate only on
1.28+, including adding a step in the next version update
* I fixed alphabetization in feature stages

.github/ISSUE_TEMPLATE/kubernetes_update.md

@@ -54,6 +54,7 @@ List of items to do for upgrading to {version_1} {version_2} {version_3}
         - [ ] Recreate clusters with new scripts: `cd build; make GCP_PROJECT=agones-images gcloud-e2e-test-cluster`
     - [ ] Update the Cloud Build configuration to run e2e test on the new created clusters, and disable the e2e test on the cluster with the oldest supported K8s version
         - [ ] Update the `versionsAndRegions` variable to add the new supported version and remove the oldest supported K8s version in `cloudbuild.yaml` `submit-e2e-test-cloud-build` step
+        - [ ] Update the `featureWithGateByVersion` variable to add the new supported version and remove the oldest supported K8s version in `cloudbuild.yaml` `submit-e2e-test-cloud-build` step
         - [ ] Run `make lint` for code quality check.
         - [ ] Submit a PR to trigger the e2e tests and verfiy they all pass
     - [ ] After the PR that includes the above Cloud Build configuration change has been merged and all the existing pending PRs in the Cloud Build queue have picked up the new configuration, submit a separate PR to update the e2e clusters terraform module to remove the e2e cluster with the oldest supported K8s version.

ci/e2e-test-cloudbuild.yaml

@@ -28,7 +28,6 @@ steps:
   #
   # Run the e2e tests with FeatureGates inverted compared to Stable
   #
-  # Keep in sync with (the inverse of) pkg/util/runtime/features.go:featureDefaults
   - name: e2e-runner
     args:
       - ${_FEATURE_WITH_GATE}

cloudbuild.yaml

@@ -286,23 +286,30 @@ steps:
         pids=()
         cloudProducts=("generic" "gke-autopilot")
         declare -A versionsAndRegions=( [1.27]=us-east1 [1.28]=us-west1 [1.29]=europe-west1 )
+
+        # Keep in sync with (the inverse of) pkg/util/runtime/features.go:featureDefaults
+        featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&FleetAllocationOverflow=false&CountsAndLists=true&DisableResyncOnSDKServer=true&Example=true"
+        featureWithoutGate=""
+
+        # Use this if specific feature gates can only be supported on specific Kubernetes versions.
+        # TODO: When 1.28 becomes trailing version, promote GKEAutopilotExtendedDurationPods to beta.
+        declare -A featureWithGateByVersion=( [1.27]="${featureWithGate}" [1.28]="${featureWithGate}&GKEAutopilotExtendedDurationPods=true" [1.29]="${featureWithGate}&GKEAutopilotExtendedDurationPods=true")
+
         for cloudProduct in ${cloudProducts[@]}
         do
           for version in "${!versionsAndRegions[@]}"
           do
+            withGate=${featureWithGateByVersion[$version]}
             region=${versionsAndRegions[$version]}
             if [ $cloudProduct = generic ]
             then
-              featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&FleetAllocationOverflow=false&CountsAndLists=true&DisableResyncOnSDKServer=true&Example=true"
               testCluster="standard-e2e-test-cluster-${version//./-}"
             else
-              featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&FleetAllocationOverflow=false&CountsAndLists=true&DisableResyncOnSDKServer=true&Example=true"
               testCluster="gke-autopilot-e2e-test-cluster-${version//./-}"
             fi
-            featureWithoutGate=""
             testClusterLocation="${region}"
             { stdbuf -oL -eL gcloud builds submit . --suppress-logs --config=./ci/e2e-test-cloudbuild.yaml \
-              --substitutions _FEATURE_WITH_GATE=$featureWithGate,_FEATURE_WITHOUT_GATE=$featureWithoutGate,_CLOUD_PRODUCT=$cloudProduct,_TEST_CLUSTER_NAME=$testCluster,_TEST_CLUSTER_LOCATION=$testClusterLocation,_REGISTRY=${_REGISTRY},_PARENT_COMMIT_SHA=${COMMIT_SHA},_PARENT_BUILD_ID=${BUILD_ID} \
+              --substitutions _FEATURE_WITH_GATE=$withGate,_FEATURE_WITHOUT_GATE=$featureWithoutGate,_CLOUD_PRODUCT=$cloudProduct,_TEST_CLUSTER_NAME=$testCluster,_TEST_CLUSTER_LOCATION=$testClusterLocation,_REGISTRY=${_REGISTRY},_PARENT_COMMIT_SHA=${COMMIT_SHA},_PARENT_BUILD_ID=${BUILD_ID} \
               |& stdbuf -i0 -oL -eL grep -v " tarball " \
               |& stdbuf -i0 -oL -eL sed "s/^/${cloudProduct}-${version}: /"; } &
             pids+=($!)

install/helm/agones/defaultfeaturegates.yaml

@@ -18,13 +18,13 @@
 FleetAllocationOverflow: true
 
 # Alpha features
+CountsAndLists: false
+DisableResyncOnSDKServer: false
+GKEAutopilotExtendedDurationPods: false
 PlayerAllocationFilter: false
 PlayerTracking: false
-DisableResyncOnSDKServer: false
-CountsAndLists: false
 
 # Dev features
-GKEAutopilotExtendedDurationPods: false
 
 # Example feature
 Example: false

pkg/util/runtime/features.go

@@ -38,27 +38,27 @@ const (
 	////////////////
 	// Alpha features
 
-	// FeaturePlayerAllocationFilter is a feature flag that enables the ability for Allocations to filter based on
-	// player capacity.
-	FeaturePlayerAllocationFilter Feature = "PlayerAllocationFilter"
-
-	// FeaturePlayerTracking is a feature flag to enable/disable player tracking features.
-	FeaturePlayerTracking Feature = "PlayerTracking"
-
 	// FeatureCountsAndLists is a feature flag that enables/disables counts and lists feature
 	// (a generic implenetation of the player tracking feature).
 	FeatureCountsAndLists Feature = "CountsAndLists"
 
 	// FeatureDisableResyncOnSDKServer is a feature flag to enable/disable resync on SDK server.
 	FeatureDisableResyncOnSDKServer Feature = "DisableResyncOnSDKServer"
 
-	////////////////
-	// Dev features
-
 	// FeatureGKEAutopilotExtendedDurationPods enables the use of Extended Duration pods
-	// when Agones is running on Autopilot.
+	// when Agones is running on Autopilot. Available on 1.28+ only.
 	FeatureGKEAutopilotExtendedDurationPods = "GKEAutopilotExtendedDurationPods"
 
+	// FeaturePlayerAllocationFilter is a feature flag that enables the ability for Allocations to filter based on
+	// player capacity.
+	FeaturePlayerAllocationFilter Feature = "PlayerAllocationFilter"
+
+	// FeaturePlayerTracking is a feature flag to enable/disable player tracking features.
+	FeaturePlayerTracking Feature = "PlayerTracking"
+
+	////////////////
+	// Dev features
+
 	////////////////
 	// Example feature
 
@@ -102,13 +102,13 @@ var (
 		FeatureFleetAllocateOverflow: true,
 
 		// Alpha features
-		FeaturePlayerAllocationFilter:   false,
-		FeaturePlayerTracking:           false,
-		FeatureDisableResyncOnSDKServer: false,
-		FeatureCountsAndLists:           false,
+		FeatureCountsAndLists:                   false,
+		FeatureDisableResyncOnSDKServer:         false,
+		FeatureGKEAutopilotExtendedDurationPods: false,
+		FeaturePlayerAllocationFilter:           false,
+		FeaturePlayerTracking:                   false,
 
 		// Dev features
-		FeatureGKEAutopilotExtendedDurationPods: false,
 
 		// Example feature
 		FeatureExample: false,

site/content/en/docs/Guides/feature-stages.md

@@ -24,17 +24,31 @@ that can be found in the [Helm configuration]({{< ref "/docs/Installation/Instal
 
 The current set of `alpha` and `beta` feature gates:
 
-
+{{% feature expiryVersion="1.40.0" %}}
 | Feature Name                                                                                                          | Gate                           | Default  | Stage   | Since  |
 |-----------------------------------------------------------------------------------------------------------------------|--------------------------------|----------|---------|--------|
 | [Allocated GameServers are notified on relevant Fleet Updates][fleet-updates]                                         | `FleetAllocationOverflow`      | Enabled  | `Beta`  | 1.37.0 |
 | [CountsAndLists](https://github.com/googleforgames/agones/issues/2716)                                                | `CountsAndLists`               | Disabled | `Alpha` | 1.37.0 |
+| [DisableResyncOnSDKServer](https://github.com/googleforgames/agones/issues/3377)                                      | `DisableResyncOnSDKServer`     | Disabled | `Alpha` | 1.37.0 |
 | [GameServer player capacity filtering on GameServerAllocations](https://github.com/googleforgames/agones/issues/1239) | `PlayerAllocationFilter`       | Disabled | `Alpha` | 1.14.0 |
 | [Player Tracking]({{< ref "/docs/Guides/player-tracking.md" >}})                                                      | `PlayerTracking`               | Disabled | `Alpha` | 1.6.0  |
-| [DisableResyncOnSDKServer](https://github.com/googleforgames/agones/issues/3377)                                      | `DisableResyncOnSDKServer`     | Disabled | `Alpha` | 1.37.0 |
 | Example Gate (not in use)                                                                                             | `Example`                      | Disabled | None    | 0.13.0 |
 
 [fleet-updates]: {{% relref "./fleet-updates.md#notifying-gameservers-on-fleet-updatedownscale" %}}
+{{% /feature %}}
+{{% feature publishVersion="1.40.0" %}}
+| Feature Name                                                                                                                | Gate                               | Default  | Stage   | Since  |
+|-----------------------------------------------------------------------------------------------------------------------------|------------------------------------|----------|---------|--------|
+| [Allocated GameServers are notified on relevant Fleet Updates][fleet-updates]                                               | `FleetAllocationOverflow`          | Enabled  | `Beta`  | 1.37.0 |
+| [CountsAndLists](https://github.com/googleforgames/agones/issues/2716)                                                      | `CountsAndLists`                   | Disabled | `Alpha` | 1.37.0 |
+| [DisableResyncOnSDKServer](https://github.com/googleforgames/agones/issues/3377)                                            | `DisableResyncOnSDKServer`         | Disabled | `Alpha` | 1.37.0 |
+| [Support for Extended Duration Pods on GKE Autopilot (*1.28+ only*)](https://github.com/googleforgames/agones/issues/3386)  | `GKEAutopilotExtendedDurationPods` | Disabled | `Alpha` | 1.37.0 |
+| [GameServer player capacity filtering on GameServerAllocations](https://github.com/googleforgames/agones/issues/1239)       | `PlayerAllocationFilter`           | Disabled | `Alpha` | 1.14.0 |
+| [Player Tracking]({{< ref "/docs/Guides/player-tracking.md" >}})                                                            | `PlayerTracking`                   | Disabled | `Alpha` | 1.6.0  |
+| Example Gate (not in use)                                                                                                   | `Example`                          | Disabled | None    | 0.13.0 |
+
+[fleet-updates]: {{% relref "./fleet-updates.md#notifying-gameservers-on-fleet-updatedownscale" %}}
+{{% /feature %}}
 
 
 {{< alert title="Note" color="info" >}}