From ae92064a67ecc04821b02c3a299f23c183f1bbc5 Mon Sep 17 00:00:00 2001 From: Alexander Apalikov Date: Fri, 28 Feb 2020 19:28:24 +0300 Subject: [PATCH 1/4] Switch from the normal to module terraform config For development installation using make target. --- build/includes/terraform.mk | 16 ++++++------ .../terraform-submodules/gke-local/module.tf | 25 ++++++++++++++++++- install/terraform/modules/gke/cluster.tf | 2 +- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/build/includes/terraform.mk b/build/includes/terraform.mk index 9b88e0d2e3..a152c4f12b 100644 --- a/build/includes/terraform.mk +++ b/build/includes/terraform.mk @@ -16,28 +16,25 @@ terraform-init: terraform-init: $(ensure-build-image) docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) bash -c '\ - cd $(mount_path)/install/terraform && terraform init && gcloud auth application-default login' + cd $(mount_path)/examples/terraform-submodules/gke-local && terraform init && gcloud auth application-default login' terraform-clean: - rm -r ../install/terraform/.terraform - rm ../install/terraform/terraform.tfstate* + rm -r ../examples/terraform-submodules/gke-local/.terraform + rm ../examples/terraform-submodules/gke-local/.tfstate* # Creates a cluster and install release version of Agones controller # Version could be specified by AGONES_VERSION gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4 gcloud-terraform-cluster: AGONES_VERSION ?= '' +gcloud-terraform-cluster: GCP_CLUSTER_NAME ?= test-cluster gcloud-terraform-cluster: $(ensure-build-image) gcloud-terraform-cluster: ifndef GCP_PROJECT $(eval GCP_PROJECT=$(shell sh -c "gcloud config get-value project 2> /dev/null")) endif $(DOCKER_RUN) bash -c 'export TF_VAR_agones_version=$(AGONES_VERSION) && \ - cd $(mount_path)/install/terraform && terraform apply -auto-approve -var values_file="" \ - -var chart="agones" \ - -var "cluster={name=\"$(GCP_CLUSTER_NAME)\", machineType=\"$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)\", \ - zone=\"$(GCP_CLUSTER_ZONE)\", project=\"$(GCP_PROJECT)\", \ - initialNodeCount=\"$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)\"}"' + cd $(mount_path)/examples/terraform-submodules/gke-local && terraform apply -auto-approve -var project="$(GCP_PROJECT)" -var name="$(GCP_CLUSTER_NAME)"' $(MAKE) gcloud-auth-cluster # Creates a cluster and install current version of Agones controller @@ -53,13 +50,14 @@ ifndef GCP_PROJECT $(eval GCP_PROJECT=$(shell sh -c "gcloud config get-value project 2> /dev/null")) endif $(DOCKER_RUN) bash -c ' \ - cd $(mount_path)/install/terraform && terraform apply -auto-approve -var agones_version="$(VERSION)" -var image_registry="$(REGISTRY)" \ + cd $(mount_path)/examples/terraform-submodules/gke-local && terraform apply -auto-approve -var agones_version="$(VERSION)" -var image_registry="$(REGISTRY)" \ -var pull_policy="$(IMAGE_PULL_POLICY)" \ -var always_pull_sidecar="$(ALWAYS_PULL_SIDECAR)" \ -var image_pull_secret="$(IMAGE_PULL_SECRET)" \ -var ping_service_type="$(PING_SERVICE_TYPE)" \ -var crd_cleanup="$(CRD_CLEANUP)" \ -var "cluster={name=\"$(GCP_CLUSTER_NAME)\", machineType=\"$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)\", \ + -var values_file="../../../helm/agones/values.yaml" \ zone=\"$(GCP_CLUSTER_ZONE)\", project=\"$(GCP_PROJECT)\", \ initialNodeCount=\"$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)\"}"' $(MAKE) gcloud-auth-cluster diff --git a/examples/terraform-submodules/gke-local/module.tf b/examples/terraform-submodules/gke-local/module.tf index 644e1ec58b..95449d0e03 100644 --- a/examples/terraform-submodules/gke-local/module.tf +++ b/examples/terraform-submodules/gke-local/module.tf @@ -40,6 +40,29 @@ variable "agones_version" { variable "machine_type" { default = "n1-standard-4" } +<<<<<<< HEAD:examples/terraform-submodules/gke-local/module.tf +======= + +variable "name" { + default = "agones-terraform-example" +} + +variable "machine_type" { + default = "n1-standard-4" +} + +variable "values_file" { + default = "" +} + +// Note: This is the number of gameserver nodes. The Agones module will automatically create an additional +// two node pools with 1 node each for "agones-system" and "agones-metrics". +variable "node_count" { + default = "4" +} + +module "gke_cluster" { +>>>>>>> Switch from the normal to module terraform config:examples/terraform-submodules/gke-local/main.tf // Note: This is the number of gameserver nodes. The Agones module will automatically create an additional // two node pools with 1 node each for "agones-system" and "agones-metrics". @@ -74,7 +97,7 @@ module "helm_agones" { source = "../../../install/terraform/modules/helm" agones_version = var.agones_version - values_file = "" + values_file = var.values_file chart = "agones" host = module.gke_cluster.host token = module.gke_cluster.token diff --git a/install/terraform/modules/gke/cluster.tf b/install/terraform/modules/gke/cluster.tf index b0aeabce68..7033ac3c0c 100644 --- a/install/terraform/modules/gke/cluster.tf +++ b/install/terraform/modules/gke/cluster.tf @@ -31,6 +31,7 @@ resource "null_resource" "test-setting-variables" { EOT } } + resource "google_container_cluster" "primary" { name = var.cluster["name"] location = var.cluster["zone"] @@ -139,6 +140,5 @@ resource "google_compute_firewall" "default" { protocol = "udp" ports = [var.ports] } - target_tags = ["game-server"] } From 0737fe3bceba3194574bfef054f8e23368e9c38f Mon Sep 17 00:00:00 2001 From: Alexander Apalikov Date: Fri, 6 Mar 2020 14:20:36 +0300 Subject: [PATCH 2/4] Pass more variables to the module helm. Refactor terraform.mk calls. --- build/includes/terraform.mk | 32 +++++++----- .../terraform/gke}/module.tf | 51 ++++++++++++------- install/terraform/modules/helm/variables.tf | 2 +- 3 files changed, 52 insertions(+), 33 deletions(-) rename {examples/terraform-submodules/gke-local => build/terraform/gke}/module.tf (78%) diff --git a/build/includes/terraform.mk b/build/includes/terraform.mk index a152c4f12b..b80caadbe6 100644 --- a/build/includes/terraform.mk +++ b/build/includes/terraform.mk @@ -16,11 +16,11 @@ terraform-init: terraform-init: $(ensure-build-image) docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) bash -c '\ - cd $(mount_path)/examples/terraform-submodules/gke-local && terraform init && gcloud auth application-default login' + cd $(mount_path)/build/terraform/gke && terraform init && gcloud auth application-default login' terraform-clean: - rm -r ../examples/terraform-submodules/gke-local/.terraform - rm ../examples/terraform-submodules/gke-local/.tfstate* + rm -r ../build/terraform/gke/.terraform || true + rm ../build/terraform/gke/terraform.tfstate* || true # Creates a cluster and install release version of Agones controller # Version could be specified by AGONES_VERSION @@ -33,35 +33,41 @@ gcloud-terraform-cluster: ifndef GCP_PROJECT $(eval GCP_PROJECT=$(shell sh -c "gcloud config get-value project 2> /dev/null")) endif - $(DOCKER_RUN) bash -c 'export TF_VAR_agones_version=$(AGONES_VERSION) && \ - cd $(mount_path)/examples/terraform-submodules/gke-local && terraform apply -auto-approve -var project="$(GCP_PROJECT)" -var name="$(GCP_CLUSTER_NAME)"' - $(MAKE) gcloud-auth-cluster + $(DOCKER_RUN) bash -c 'cd $(mount_path)/build/terraform/gke && \ + terraform apply -auto-approve -var agones_version="$(AGONES_VERSION)" \ + -var name=$(GCP_CLUSTER_NAME) -var machine_type="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \ + -var values_file="" \ + -var zone="$(GCP_CLUSTER_ZONE)" -var project="$(GCP_PROJECT)" \ + -var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)' + GCP_CLUSTER_NAME=$(GCP_CLUSTER_NAME) $(MAKE) gcloud-auth-cluster # Creates a cluster and install current version of Agones controller # Set all necessary variables as `make install` does +# Unifies previous `make gcloud-test-cluster` and `make install` targets gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4 gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4 gcloud-terraform-install: ALWAYS_PULL_SIDECAR := true gcloud-terraform-install: IMAGE_PULL_POLICY := "Always" gcloud-terraform-install: PING_SERVICE_TYPE := "LoadBalancer" gcloud-terraform-install: CRD_CLEANUP := true +gcloud-terraform-install: GCP_CLUSTER_NAME ?= test-cluster gcloud-terraform-install: ifndef GCP_PROJECT $(eval GCP_PROJECT=$(shell sh -c "gcloud config get-value project 2> /dev/null")) endif $(DOCKER_RUN) bash -c ' \ - cd $(mount_path)/examples/terraform-submodules/gke-local && terraform apply -auto-approve -var agones_version="$(VERSION)" -var image_registry="$(REGISTRY)" \ + cd $(mount_path)/build/terraform/gke && terraform apply -auto-approve -var agones_version="$(VERSION)" -var image_registry="$(REGISTRY)" \ -var pull_policy="$(IMAGE_PULL_POLICY)" \ -var always_pull_sidecar="$(ALWAYS_PULL_SIDECAR)" \ -var image_pull_secret="$(IMAGE_PULL_SECRET)" \ -var ping_service_type="$(PING_SERVICE_TYPE)" \ -var crd_cleanup="$(CRD_CLEANUP)" \ - -var "cluster={name=\"$(GCP_CLUSTER_NAME)\", machineType=\"$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)\", \ - -var values_file="../../../helm/agones/values.yaml" \ - zone=\"$(GCP_CLUSTER_ZONE)\", project=\"$(GCP_PROJECT)\", \ - initialNodeCount=\"$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)\"}"' - $(MAKE) gcloud-auth-cluster + -var chart="../../../install/helm/agones/" \ + -var name=$(GCP_CLUSTER_NAME) -var machine_type="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \ + -var zone=$(GCP_CLUSTER_ZONE) -var project=$(GCP_PROJECT) \ + -var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)' + GCP_CLUSTER_NAME=$(GCP_CLUSTER_NAME) $(MAKE) gcloud-auth-cluster gcloud-terraform-destroy-cluster: - $(DOCKER_RUN) bash -c 'cd $(mount_path)/install/terraform && \ + $(DOCKER_RUN) bash -c 'cd $(mount_path)/build/terraform/gke && \ terraform destroy -auto-approve' diff --git a/examples/terraform-submodules/gke-local/module.tf b/build/terraform/gke/module.tf similarity index 78% rename from examples/terraform-submodules/gke-local/module.tf rename to build/terraform/gke/module.tf index 95449d0e03..d17f2b47f5 100644 --- a/examples/terraform-submodules/gke-local/module.tf +++ b/build/terraform/gke/module.tf @@ -28,10 +28,6 @@ variable "project" { default = "" } -variable "name" { - default = "agones-terraform-example" -} - // Install latest version of agones variable "agones_version" { default = "" @@ -40,19 +36,13 @@ variable "agones_version" { variable "machine_type" { default = "n1-standard-4" } -<<<<<<< HEAD:examples/terraform-submodules/gke-local/module.tf -======= variable "name" { - default = "agones-terraform-example" -} - -variable "machine_type" { - default = "n1-standard-4" + default = "agones-tf-cluster" } variable "values_file" { - default = "" + default = "../../../install/helm/agones/values.yaml" } // Note: This is the number of gameserver nodes. The Agones module will automatically create an additional @@ -60,14 +50,16 @@ variable "values_file" { variable "node_count" { default = "4" } +variable "chart" { + default = "agones" +} -module "gke_cluster" { ->>>>>>> Switch from the normal to module terraform config:examples/terraform-submodules/gke-local/main.tf +variable "crd_cleanup" { + default = "true" +} -// Note: This is the number of gameserver nodes. The Agones module will automatically create an additional -// two node pools with 1 node each for "agones-system" and "agones-metrics". -variable "node_count" { - default = "4" +variable "ping_service_type" { + default = "LoadBalancer" } variable "zone" { @@ -80,6 +72,23 @@ variable "network" { description = "The name of the VPC network to attach the cluster and firewall rule to" } +variable "pull_policy" { + default = "Always" +} + +variable "image_registry" { + default = "gcr.io/agones-images" +} + +variable "always_pull_sidecar" { + default = "true" +} + +variable "image_pull_secret" { + default = "" +} + + module "gke_cluster" { source = "../../../install/terraform/modules/gke" @@ -98,10 +107,14 @@ module "helm_agones" { agones_version = var.agones_version values_file = var.values_file - chart = "agones" + chart = var.chart host = module.gke_cluster.host token = module.gke_cluster.token cluster_ca_certificate = module.gke_cluster.cluster_ca_certificate + image_registry = var.image_registry + image_pull_secret = var.image_pull_secret + crd_cleanup = var.crd_cleanup + ping_service_type = var.ping_service_type } output "host" { diff --git a/install/terraform/modules/helm/variables.tf b/install/terraform/modules/helm/variables.tf index 7127c48622..e58d4a4256 100644 --- a/install/terraform/modules/helm/variables.tf +++ b/install/terraform/modules/helm/variables.tf @@ -62,5 +62,5 @@ variable "ping_service_type" { } variable "values_file" { - default = "../../../helm/agones/values.yaml" + default = "" } From b8063d50874e478b0db97728d69ea38f1ba22379 Mon Sep 17 00:00:00 2001 From: Alexander Apalikov Date: Mon, 23 Mar 2020 13:29:13 +0300 Subject: [PATCH 3/4] Remove GKE cluster with a small delay after Helm Remove Helm Release first and wait for 60 seconds so that forwarding rule and firewall rules would be cleaned up properly by the GCE controller --- build/includes/terraform.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/includes/terraform.mk b/build/includes/terraform.mk index b80caadbe6..f0ee89accd 100644 --- a/build/includes/terraform.mk +++ b/build/includes/terraform.mk @@ -70,4 +70,4 @@ endif gcloud-terraform-destroy-cluster: $(DOCKER_RUN) bash -c 'cd $(mount_path)/build/terraform/gke && \ - terraform destroy -auto-approve' + terraform destroy -target module.helm_agones.helm_release.agones -auto-approve && sleep 60 && terraform destroy -auto-approve' From 51f0933c8a9f23e8412833a942d586d8b128fc3f Mon Sep 17 00:00:00 2001 From: Alexander Apalikov Date: Thu, 26 Mar 2020 15:44:58 +0300 Subject: [PATCH 4/4] Separate Terraform cluster by name - add variable Now clusters created using gcloud and terraform would have different names. --- build/includes/terraform.mk | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/build/includes/terraform.mk b/build/includes/terraform.mk index f0ee89accd..517af8ce12 100644 --- a/build/includes/terraform.mk +++ b/build/includes/terraform.mk @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# The GKE development cluster name +GCP_TF_CLUSTER_NAME ?= agones-tf-cluster + ### Deploy cluster with Terraform terraform-init: terraform-init: $(ensure-build-image) @@ -27,19 +30,19 @@ terraform-clean: gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4 gcloud-terraform-cluster: AGONES_VERSION ?= '' -gcloud-terraform-cluster: GCP_CLUSTER_NAME ?= test-cluster +gcloud-terraform-cluster: GCP_TF_CLUSTER_NAME ?= agones-tf-cluster gcloud-terraform-cluster: $(ensure-build-image) gcloud-terraform-cluster: ifndef GCP_PROJECT $(eval GCP_PROJECT=$(shell sh -c "gcloud config get-value project 2> /dev/null")) endif $(DOCKER_RUN) bash -c 'cd $(mount_path)/build/terraform/gke && \ - terraform apply -auto-approve -var agones_version="$(AGONES_VERSION)" \ - -var name=$(GCP_CLUSTER_NAME) -var machine_type="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \ + terraform apply -auto-approve -var agones_version="$(AGONES_VERSION)" \ + -var name=$(GCP_TF_CLUSTER_NAME) -var machine_type="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \ -var values_file="" \ -var zone="$(GCP_CLUSTER_ZONE)" -var project="$(GCP_PROJECT)" \ -var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)' - GCP_CLUSTER_NAME=$(GCP_CLUSTER_NAME) $(MAKE) gcloud-auth-cluster + GCP_CLUSTER_NAME=$(GCP_TF_CLUSTER_NAME) $(MAKE) gcloud-auth-cluster # Creates a cluster and install current version of Agones controller # Set all necessary variables as `make install` does @@ -50,7 +53,7 @@ gcloud-terraform-install: ALWAYS_PULL_SIDECAR := true gcloud-terraform-install: IMAGE_PULL_POLICY := "Always" gcloud-terraform-install: PING_SERVICE_TYPE := "LoadBalancer" gcloud-terraform-install: CRD_CLEANUP := true -gcloud-terraform-install: GCP_CLUSTER_NAME ?= test-cluster +gcloud-terraform-install: GCP_TF_CLUSTER_NAME ?= agones-tf-cluster gcloud-terraform-install: ifndef GCP_PROJECT $(eval GCP_PROJECT=$(shell sh -c "gcloud config get-value project 2> /dev/null")) @@ -63,10 +66,10 @@ endif -var ping_service_type="$(PING_SERVICE_TYPE)" \ -var crd_cleanup="$(CRD_CLEANUP)" \ -var chart="../../../install/helm/agones/" \ - -var name=$(GCP_CLUSTER_NAME) -var machine_type="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \ + -var name=$(GCP_TF_CLUSTER_NAME) -var machine_type="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \ -var zone=$(GCP_CLUSTER_ZONE) -var project=$(GCP_PROJECT) \ -var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)' - GCP_CLUSTER_NAME=$(GCP_CLUSTER_NAME) $(MAKE) gcloud-auth-cluster + GCP_CLUSTER_NAME=$(GCP_TF_CLUSTER_NAME) $(MAKE) gcloud-auth-cluster gcloud-terraform-destroy-cluster: $(DOCKER_RUN) bash -c 'cd $(mount_path)/build/terraform/gke && \