Update Supported Kubernetes to 1.24 1.25 1.26

.github/ISSUE_TEMPLATE/kubernetes_update.md

@@ -12,16 +12,16 @@ Agones supports and is tested against 3 releases of Kubernetes, targeting the ne
 List of items to do for upgrading to {version_1} {version_2} {version_3}
 
 - [ ] Update the cluster version of terraform submodules in `install/terraform/modules`
-    - [ ] Update Kubernetes version of GKE cluster to {version_2}
+    - [ ] Update Kubernetes version of GKE cluster (both `gke` and `gke-autopilot`) to {version_2}
     - [ ] Update Kubernetes version of AKS to the newest supported version in {version_1} {version_2} {version_3}
     - [ ] Update Kubernetes version of EKS to the newest supported version in {version_1} {version_2} {version_3}
-- [ ] Update kubectl in dev tooling to {version_2}
+- [ ] Update kubectl in dev tooling to {version_2}, the latest patch version can be found [here](https://kubernetes.io/releases/)
     - [ ] Update kubectl in `build/build-image/Dockerfile`
     - [ ] Update kubectl in `build/e2e-image/Dockerfile`
 - [ ] Update the Kubernetes version of the below test clusters to {version_2}
-    - [ ] Minikube (Get the patch version [here](https://github.com/kubernetes/minikube/blob/master/CHANGELOG.md))
-    - [ ] Kind (Confirm {version_2} is supported and get the patch version [here](https://github.com/kubernetes-sigs/kind/releases))
-- [ ] Update the k8s image used in the helm [pre-delete-hook](https://github.com/googleforgames/agones/blob/main/install/helm/agones/templates/hooks/pre_delete_hook.yaml) to {version_2}
+    - [ ] Minikube in `build/includes/minikube.mk` (Get the patch version [here](https://kubernetes.io/releases/) since minikube supports the latest Kubernetes release)
+    - [ ] Kind in `build/includes/kind.mk` (Confirm {version_2} is supported and get the patch version [here](https://github.com/kubernetes-sigs/kind/releases))
+- [ ] Update the k8s image used in the helm [pre-delete-hook](https://github.com/googleforgames/agones/blob/main/install/helm/agones/templates/hooks/pre_delete_hook.yaml) to {version_2} (Get the patch version [here](https://hub.docker.com/r/lachlanevenson/k8s-kubectl))
 - [ ] Update client-go in `go.mod` and `test/terraform/go.mod` to {version_2} by running `go get -u k8s.io/client-go@{CORRESPONDING_VERSION}` and `go get -u k8s.io/apiextensions-apiserver@{CORRESPONDING_VERSION}`, then re-run `go mod tidy` and `go mod vendor`
 - [ ] Update CRD API reference to {version_2}
     - [ ] Update links to k8s documentation in `site/assets/templates/crd-doc-config.json`
@@ -43,9 +43,12 @@ List of items to do for upgrading to {version_1} {version_2} {version_3}
     - [ ] Regenerate all client sdks: [make gen-all-sdk-grpc](https://github.com/googleforgames/agones/blob/main/build/README.md#make-gen-all-sdk-grpc)
     - [ ] Update the version number in C++ Cmake scripts [here](https://github.com/googleforgames/agones/blob/main/sdks/cpp/CMakeLists.txt#L100) and [here](https://github.com/googleforgames/agones/blob/main/sdks/cpp/cmake/prerequisites.cmake#L34)
 - [ ] Confirm the update works as expected by running e2e tests
-    - [ ] Update the Kubernetes version of the e2e clusters
-        - [ ] In `terraform/e2e/module.tf`, update variable `kubernetes_versions_standard` and `kubernetes_versions_autopilot` to the new versions to be supported
-        - [ ] Recreate cluster with new scripts: `cd build; make GCP_PROJECT=agones-images gcloud-e2e-test-cluster`
-    - [ ] Update the Cloud Build configuration to run e2e test on the new created clusters
-        - [ ] Update the `versionsAndRegions` variable to reflect new versions in `cloudbuild.yaml` `submit-e2e-test-cloud-build` step
+    - [ ] Add the new supported Kubernetes versions to the e2e clusters creation
+        - [ ] In `terraform/e2e/module.tf`, add the new supported version to the map `kubernetes_versions_standard` and `kubernetes_versions_autopilot`. Noted the location of the new clusters should have enough quota (CPU, In-use IP addresses) to create the cluster. And the new supported version is usually only available in RAPID channel.
+        - [ ] Recreate clusters with new scripts: `cd build; make GCP_PROJECT=agones-images gcloud-e2e-test-cluster`
+    - [ ] Update the Cloud Build configuration to run e2e test on the new created clusters, and disable the e2e test on the cluster with the oldest supported K8s version
+        - [ ] Update the `versionsAndRegions` variable to add the new supported version and remove the oldest supported K8s version in `cloudbuild.yaml` `submit-e2e-test-cloud-build` step
         - [ ] Submit a PR to trigger the e2e tests and verfiy they all pass
+    - [ ] After the PR that includes the above Cloud Build configuration change has been merged and all the existing pending PRs in the Cloud Build queue have picked up the new configuration, submit a separate PR to update the e2e clusters terraform module to remove the e2e cluster with the oldest supported K8s version.
+        - [ ] In `terraform/e2e/module.tf`, remove the oldest supported version from the map `kubernetes_versions_standard` and `kubernetes_versions_autopilot`.
+        - [ ] Destroy the old clusters with new scripts: `cd build; make GCP_PROJECT=agones-images gcloud-e2e-test-cluster`

build/build-image/Dockerfile

@@ -44,7 +44,7 @@ ENV PATH /usr/local/go/bin:/go/bin:$PATH
 RUN go install golang.org/x/tools/cmd/goimports@latest
 
 # the kubernetes version for the file
-ENV KUBERNETES_VER 1.24.9
+ENV KUBERNETES_VER 1.25.7
 
 # overwrite kubectl as we want a specific version
 RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VER}/bin/linux/amd64/kubectl && \

build/e2e-image/Dockerfile

@@ -15,7 +15,7 @@ RUN wget -q https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \
 ENV PATH /usr/local/go/bin:/go/bin:$PATH
 
 # install kubectl without gcloud as we want the last version
-ENV KUBECTL_VER 1.24.9
+ENV KUBECTL_VER 1.25.7
 RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VER}/bin/linux/amd64/kubectl && \
     chmod go+rx ./kubectl && \
     mv ./kubectl /usr/local/bin/kubectl

build/includes/kind.mk

@@ -26,7 +26,7 @@ kind-test-cluster: DOCKER_RUN_ARGS+=--network=host
 kind-test-cluster: $(ensure-build-image)
 	@if [ -z $$(kind get clusters | grep $(KIND_PROFILE)) ]; then\
 		echo "Could not find $(KIND_PROFILE) cluster. Creating...";\
-		kind create cluster --name $(KIND_PROFILE) --image kindest/node:v1.24.7 --wait 5m;\
+		kind create cluster --name $(KIND_PROFILE) --image kindest/node:v1.25.3 --wait 5m;\
 	fi
 
 # deletes the kind agones cluster

build/includes/minikube.mk

@@ -28,7 +28,7 @@ minikube_cert_mount := ~/.minikube:$(HOME)/.minikube
 # of the right version.
 minikube-test-cluster: DOCKER_RUN_ARGS+=--network=host -v $(minikube_cert_mount)
 minikube-test-cluster: $(ensure-build-image)
-	$(MINIKUBE) start --kubernetes-version v1.24.9 -p $(MINIKUBE_PROFILE) --driver $(MINIKUBE_DRIVER)
+	$(MINIKUBE) start --kubernetes-version v1.25.7 -p $(MINIKUBE_PROFILE) --driver $(MINIKUBE_DRIVER)
 
 # Connecting to minikube requires so enhanced permissions, so use this target
 # instead of `make shell` to start an interactive shell for development on minikube.

build/includes/sdk.mk

@@ -27,7 +27,7 @@ build_sdk_base_tag = agones-build-sdk-base:$(build_sdk_base_version)
 build_sdk_version = $(call sha_dir,$(build_path)/build-sdk-images/$(SDK_FOLDER)/*)
 build_sdk_base_remote_tag = $(REGISTRY)/$(build_sdk_base_tag)
 build_sdk_prefix = agones-build-sdk-
-grpc_release_tag = v1.50.1
+grpc_release_tag = v1.53.0
 sdk_build_folder = build-sdk-images/
 examples_folder = ../examples/
 SDK_FOLDER ?= go

build/terraform/e2e/gke-autopilot/module.tf

@@ -33,6 +33,7 @@ terraform {
 variable "project" {}
 variable "kubernetesVersion" {}
 variable "location" {}
+variable "releaseChannel" {}
 
 module "gke_cluster" {
   source = "../../../../install/terraform/modules/gke-autopilot"
@@ -41,6 +42,7 @@ module "gke_cluster" {
     "name"              = format("gke-autopilot-e2e-test-cluster-%s", replace(var.kubernetesVersion, ".", "-"))
     "project"           = var.project
     "location"          = var.location
+    "releaseChannel"    = var.releaseChannel
     "kubernetesVersion" = var.kubernetesVersion
   }
 

build/terraform/e2e/gke-standard/module.tf

@@ -33,6 +33,7 @@ terraform {
 variable "project" {}
 variable "kubernetesVersion" {}
 variable "location" {}
+variable "releaseChannel" {}
 
 variable "overrideName" {
   default = ""
@@ -44,6 +45,7 @@ module "gke_cluster" {
   cluster = {
     "name"                 = var.overrideName != "" ? var.overrideName : format("gke-standard-e2e-test-cluster-%s", replace(var.kubernetesVersion, ".", "-"))
     "location"             = var.location
+    "releaseChannel"       = var.releaseChannel
     "machineType"          = "e2-standard-4"
     "initialNodeCount"     = 10
     "enableImageStreaming" = true

build/terraform/e2e/module.tf

@@ -36,21 +36,22 @@ terraform {
 variable "project" {}
 variable "kubernetes_versions_standard" {
   description = "Create standard e2e test clusters with these k8s versions in these zones"
-  type        = map(string)
+  type        = map(list(string))
   default     = {
-    "1.23" = "us-east1-c"
-    "1.24" = "us-west1-c"
-    "1.25" = "us-central1-c"
+    "1.23" = ["us-east1-c", "UNSPECIFIED"]
+    "1.24" = ["us-west1-c", "UNSPECIFIED"]
+    "1.25" = ["us-central1-c", "UNSPECIFIED"]
+    "1.26" = ["asia-east1-c", "RAPID"]
   }
 }
 
 variable "kubernetes_versions_autopilot" {
   description = "Create Autopilot e2e test clusters with these k8s versions in these regions"
-  type        = map(string)
+  type        = map(list(string))
   default     = {
-    "1.23" = "us-east1"
-    "1.24" = "us-west1"
-    "1.25" = "us-central1"
+    "1.24" = ["us-west1", "REGULAR"]
+    "1.25" = ["us-central1", "REGULAR"]
+    "1.26" = ["asia-east1", "RAPID"]
   }
 }
 
@@ -59,15 +60,17 @@ module "gke_standard_cluster" {
   source = "./gke-standard"
   project = var.project
   kubernetesVersion = each.key
-  location = each.value
+  location = each.value[0]
+  releaseChannel = each.value[1]
 }
 
 module "gke_autopilot_cluster" {
   for_each = var.kubernetes_versions_autopilot
   source = "./gke-autopilot"
   project = var.project
   kubernetesVersion = each.key
-  location = each.value
+  location = each.value[0]
+  releaseChannel = each.value[1]
 }
 
 resource "google_compute_firewall" "udp" {

cloudbuild.yaml

@@ -300,17 +300,13 @@ steps:
       set -o pipefail
       pids=()
       cloudProducts=("generic" "gke-autopilot")
-      declare -A versionsAndRegions=( [1.23]=us-east1 [1.24]=us-west1 [1.25]=us-central1 )
+      declare -A versionsAndRegions=( [1.24]=us-west1 [1.25]=us-central1 [1.26]=asia-east1 )
 
       for cloudProduct in ${cloudProducts[@]}
       do
         for version in "${!versionsAndRegions[@]}"
         do
           region=${versionsAndRegions[$version]}
-          if [ $cloudProduct = gke-autopilot ] && [ $version = 1.23 ]
-          then
-            continue
-          fi
           if [ $cloudProduct = generic ]
           then
             featureWithGate="CustomFasSyncInterval=false&SafeToEvict=false&SDKGracefulTermination=false&StateAllocationFilter=false&PlayerAllocationFilter=true&PlayerTracking=true&ResetMetricsOnDelete=true&PodHostname=true&SplitControllerAndExtensions=true&FleetAllocationOverflow=true&Example=true"

go.mod

@@ -36,11 +36,11 @@ require (
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/fsnotify.v1 v1.4.7
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	k8s.io/api v0.24.9
+	k8s.io/api v0.25.7
 	k8s.io/apiextensions-apiserver v0.24.9
-	k8s.io/apimachinery v0.24.9
-	k8s.io/client-go v0.24.9
-	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
+	k8s.io/apimachinery v0.25.7
+	k8s.io/client-go v0.25.7
+	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
 )
 
 require (
@@ -55,7 +55,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful v2.16.0+incompatible // indirect
+	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
@@ -72,7 +72,7 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
 	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -101,14 +101,13 @@ require (
 	gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.51.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/gengo v0.0.0-20221011193443-fad74ee6edd9 // indirect
 	k8s.io/klog v1.0.0 // indirect
 	k8s.io/klog/v2 v2.90.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect
-	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )