Use GCS as the Terraform state backend (#2938)

* Use GCS to as the Terraform state backend

* fix format

* Make gcs bucket dynamic; Use gcs as terraform backend for prow cluster

* remove prow cluster

.github/ISSUE_TEMPLATE/kubernetes_update.md

@@ -13,8 +13,6 @@ List of items to do for upgrading to {version}:
     - [ ] GKE
     - [ ] Azure
     - [ ] EKS
-- [ ] Update prow cluster (even though we aren't using it yet, we should keep it in sync)
-    - [ ] Recreate cluster with new scripts: `cd build/terraform/prow; terraform apply -var project=agones-images`
 - [ ] Update e2e cluster
     - [ ] Recreate cluster with new scripts: `cd build/terraform/e2e; terraform apply -var project=agones-images`
 - [ ] Update kubectl in dev tooling

build/includes/google-cloud.mk

@@ -53,30 +53,17 @@ clean-gcloud-test-cluster: $(ensure-build-image)
 gcloud-e2e-test-cluster: GCP_PROJECT ?= $(shell $(current_project))
 gcloud-e2e-test-cluster: $(ensure-build-image)
 gcloud-e2e-test-cluster:
-	$(MAKE) terraform-init DIRECTORY=e2e
+	$(MAKE) terraform-init BUCKET=$(GCP_PROJECT)-e2e-infra-bucket-tfstate PREFIX=terraform/state DIRECTORY=e2e
 	docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) bash -c 'cd $(mount_path)/build/terraform/e2e && \
       	terraform apply -auto-approve -var project="$(GCP_PROJECT)"'
 
 # Deletes the gcloud e2e cluster and cleanup any left pvc volumes
 clean-gcloud-e2e-test-cluster: GCP_PROJECT ?= $(shell $(current_project))
 clean-gcloud-e2e-test-cluster: $(ensure-build-image)
 clean-gcloud-e2e-test-cluster:
-	$(MAKE) terraform-init DIRECTORY=e2e
+	$(MAKE) terraform-init BUCKET=$(GCP_PROJECT)-e2e-infra-bucket-tfstate PREFIX=terraform/state DIRECTORY=e2e
 	$(DOCKER_RUN) bash -c 'cd $(mount_path)/build/terraform/e2e && terraform destroy -var project=$(GCP_PROJECT) -auto-approve'
 
-# Creates a gcloud cluster for prow
-gcloud-prow-build-cluster: GCP_PROJECT ?= $(shell $(current_project))
-gcloud-prow-build-cluster: $(ensure-build-image)
-gcloud-prow-build-cluster:
-	$(MAKE) terraform-init DIRECTORY=prow
-	docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) bash -c 'cd $(mount_path)/build/terraform/prow && \
-      	terraform apply -auto-approve -var project="$(GCP_PROJECT)"'
-
-# Deletes the gcloud prow build cluster
-clean-gcloud-prow-build-cluster: $(ensure-build-image)
-	$(MAKE) terraform-init DIRECTORY=prow
-	$(DOCKER_RUN) bash -c 'cd $(mount_path)/build/terraform/prow && terraform destroy -var project=$(GCP_PROJECT) -auto-approve'
-
 # Pulls down authentication information for kubectl against a cluster, name can be specified through GCP_CLUSTER_NAME
 # (defaults to 'test-cluster')
 gcloud-auth-cluster: $(ensure-build-image)

build/includes/terraform.mk

@@ -23,7 +23,7 @@ terraform-init: TERRAFORM_BUILD_DIR ?= $(mount_path)/build/terraform/$(DIRECTORY
 terraform-init: $(ensure-build-image)
 terraform-init:
 	docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) bash -c '\
-	cd $(TERRAFORM_BUILD_DIR) && terraform init && gcloud auth application-default login'
+	cd $(TERRAFORM_BUILD_DIR) && terraform init -backend-config="bucket=$(BUCKET)" -backend-config="prefix=$(PREFIX)" && gcloud auth application-default login'
 
 terraform-clean: TERRAFORM_BUILD_DIR ?= $(mount_path)/build/terraform/gke
 terraform-clean:

build/terraform/e2e/gke-autopilot/module.tf

@@ -43,38 +43,4 @@ module "gke_cluster" {
   }
 
   udpFirewall = false // firewall is created at the project module level
-}
-
-provider "helm" {
-  kubernetes {
-    host                   = module.gke_cluster.host
-    token                  = module.gke_cluster.token
-    cluster_ca_certificate = module.gke_cluster.cluster_ca_certificate
-  }
-}
-
-resource "helm_release" "consul" {
-  repository = "https://helm.releases.hashicorp.com"
-  chart      = "consul"
-  name       = "consul"
-
-  set {
-    name  = "server.replicas"
-    value = "1"
-  }
-
-  set {
-    name  = "server.affinity"
-    value = "null"
-  }
-
-  set {
-    name  = "ui.service.type"
-    value = "ClusterIP"
-  }
-
-  set {
-    name  = "client.enabled"
-    value = "false"
-  }
 }

build/terraform/e2e/module.tf

@@ -14,6 +14,7 @@
 
 
 // Run:
+//  terraform init -backend-config="bucket=<YOUR_GCP_ProjectID>-e2e-infra-bucket-tfstate" -backend-config="prefix=terraform/state"
 //  terraform apply -var project="<YOUR_GCP_ProjectID>"
 
 terraform {
@@ -28,6 +29,8 @@ terraform {
       version = "~> 2.3"
     }
   }
+  backend "gcs" {
+  }
 }
 
 variable "project" {}

build/terraform/e2e/state-bucket/main.tf

@@ -0,0 +1,43 @@
+// Copyright 2023 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+// Run:
+//  terraform apply -var project="<YOUR_GCP_ProjectID>"
+
+// # GCS bucket for holding the Terraform state of the e2e Terraform config.
+
+terraform {
+  required_version = ">= 1.0.0"
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 4.25.0"
+    }
+  }
+}
+
+variable "project" {}
+
+resource "google_storage_bucket" "default" {
+  project                     = var.project
+  name                        = "${var.project}-e2e-infra-bucket-tfstate"
+  force_destroy               = false
+  uniform_bucket_level_access = true
+  location                    = "US"
+  storage_class               = "STANDARD"
+  versioning {
+    enabled = true
+  }
+}