From ae918b62c4fa702d2c01d7ebb2c58baf1fc77339 Mon Sep 17 00:00:00 2001 From: Alexander Apalikov Date: Thu, 8 Aug 2019 12:24:47 +0300 Subject: [PATCH] Use terraform-aws-eks module This way of configuring EKS cluster is the most popular, so we will use this github source for deployment. There is an issue with UDP ping service deploying on AWS, disabling it for initial version. Closes #966. --- build/modules/eks/eks.tf | 109 ++++++++++++------ build/modules/eks/outputs.tf | 43 +++++-- build/modules/eks/variables.tf | 45 ++++++++ build/modules/helm/helm.tf | 7 +- build/modules/helm/variables.tf | 4 + examples/terraform-submodules/eks/module.tf | 67 +++++++++++ .../content/en/docs/Installation/terraform.md | 77 ++++++++++++- 7 files changed, 304 insertions(+), 48 deletions(-) create mode 100644 examples/terraform-submodules/eks/module.tf diff --git a/build/modules/eks/eks.tf b/build/modules/eks/eks.tf index 8fa361fc5b..696cdcde0e 100644 --- a/build/modules/eks/eks.tf +++ b/build/modules/eks/eks.tf @@ -12,53 +12,94 @@ # See the License for the specific language governing permissions and # limitations under the License. + +terraform { + required_version = ">= 0.12.0" +} + provider "aws" { - version = ">= 2.11" + version = "~> 2.32" region = var.region } +data "aws_availability_zones" "available" { +} + +resource "aws_security_group" "worker_group_mgmt_one" { + name_prefix = "worker_group_mgmt_one" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + + cidr_blocks = [ + "10.0.0.0/8", + ] + } + ingress { + from_port = 7000 + to_port = 8000 + protocol = "udp" -resource "aws_vpc" "example" { - cidr_block = "10.0.0.0/16" - enable_dns_hostnames = true - enable_dns_support = true - tags = "${ - map( - "Name", "terraform-eks", - "kubernetes.io/cluster/example", "shared", - ) - }" + cidr_blocks = [ + "0.0.0.0/0", + ] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } } -resource "aws_vpc" "main" { -cidr_block = "10.0.0.0/16" -enable_dns_hostnames = true -enable_dns_support = true -tags = "${ - map( - "Name", "terraform-eks", - "kubernetes.io/cluster/example", "shared", - ) -}" +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "2.17.0" + + name = "test-vpc-lt" + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_dns_hostnames = false + + tags = { + "kubernetes.io/cluster/${var.cluster_name}" = "shared" + } } -# -# main EKS terraform resource definition -# -resource "aws_eks_cluster" "main" { - name = "${var.cluster_name}" - worker_groups = [ +module "eks" { + source = "git::github.com/terraform-aws-modules/terraform-aws-eks.git?ref=c9986f5e01c785875cb1e9cfa21ba195ef1bbab7" + cluster_name = "${var.cluster_name}" + subnets = module.vpc.public_subnets + vpc_id = module.vpc.vpc_id + cluster_version = "1.12" + + worker_groups_launch_template = [ { - name = "worker-group-1" - instance_type = "t2.micro" - asg_desired_capacity = 5 + name = "default" + instance_type = "${var.machine_type}" + asg_desired_capacity = 3 additional_security_group_ids = [aws_security_group.worker_group_mgmt_one.id] + public_ip = true }, + // Node Pools with taints for metrics and system { - name = "worker-group-2" - instance_type = "t2.micro" - additional_security_group_ids = [aws_security_group.worker_group_mgmt_two.id] - asg_desired_capacity = 5 + name = "agones-system" + instance_type = "${var.machine_type}" + asg_desired_capacity = 1 + kubelet_extra_args = "--node-labels=agones.dev/agones-system=true --register-with-taints=agones.dev/agones-system=true:NoExecute" + public_ip = true }, + { + name = "agones-metrics" + instance_type = "${var.machine_type}" + asg_desired_capacity = 1 + kubelet_extra_args = "--node-labels=agones.dev/agones-metrics=true --register-with-taints=agones.dev/agones-metrics=true:NoExecute" + public_ip = true + } ] } \ No newline at end of file diff --git a/build/modules/eks/outputs.tf b/build/modules/eks/outputs.tf index 824d90e1b7..e9ebafe5bd 100644 --- a/build/modules/eks/outputs.tf +++ b/build/modules/eks/outputs.tf @@ -11,23 +11,52 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +output "cluster_endpoint" { + description = "Endpoint for EKS control plane." + value = module.eks.cluster_endpoint +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane." + value = module.eks.cluster_security_group_id +} + +output "kubectl_config" { + description = "kubectl config as generated by the module." + value = module.eks.kubeconfig +} + +output "config_map_aws_auth" { + description = "A kubernetes configuration to authenticate to this EKS cluster." + value = module.eks.config_map_aws_auth +} + +output "region" { + description = "AWS region." + value = var.region +} + + output "cluster_ca_certificate" { - value = "${base64decode(aws_eks_cluster.main.kube_config.0.cluster_ca_certificate)}" + value = "${base64decode( module.eks.cluster_certificate_authority_data)}" } +output "host" { + depends_on = ["module.eks"] + value = "${ module.eks.cluster_endpoint}" +} +/* output "client_certificate" { - value = "${aws_eks_cluster.main.kube_config.0.client_certificate}" + value = "${ module.eks.kubeconfig.client_certificate}" } output "kube_config" { - value = "${aws_eks_cluster.main.kube_config_raw}" + value = "${ module.eks.kubeconfig}" } -output "host" { - value = "${aws_eks_cluster.main.kube_config.0.host}" -} output "token" { - value = "${aws_eks_cluster.main.kube_config.0.password}" + value = "${ module.eks.kubeconfig.password}" } +*/ \ No newline at end of file diff --git a/build/modules/eks/variables.tf b/build/modules/eks/variables.tf index a2db5cb15e..9760298086 100644 --- a/build/modules/eks/variables.tf +++ b/build/modules/eks/variables.tf @@ -19,3 +19,48 @@ variable "cluster_name" { variable "region" { default = "us-west-2" } + +variable "machine_type" { + default = "t2.large" +} + +variable "map_accounts" { + description = "Additional AWS account numbers to add to the aws-auth configmap." + type = list(string) + + default = [ + "777777777777", + "888888888888", + ] +} + +variable "map_roles" { + description = "Additional IAM roles to add to the aws-auth configmap." + type = list(map(string)) + + default = [ + { + role_arn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + group = "system:masters" + }, + ] +} + +variable "map_users" { + description = "Additional IAM users to add to the aws-auth configmap." + type = list(map(string)) + + default = [ + { + user_arn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + group = "system:masters" + }, + { + user_arn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + group = "system:masters" + }, + ] +} \ No newline at end of file diff --git a/build/modules/helm/helm.tf b/build/modules/helm/helm.tf index 03f840b44f..6df619f906 100644 --- a/build/modules/helm/helm.tf +++ b/build/modules/helm/helm.tf @@ -124,10 +124,15 @@ resource "helm_release" "agones" { } set { - name = " agones.ping.http.serviceType" + name = "agones.ping.http.serviceType" value = "${var.ping_service_type}" } + set { + name = "agones.ping.udp.expose" + value ="${var.udp_expose}" + } + set { name = "agones.ping.udp.serviceType" value = "${var.ping_service_type}" diff --git a/build/modules/helm/variables.tf b/build/modules/helm/variables.tf index 9b0bb7bfc0..f0c4f58f6e 100644 --- a/build/modules/helm/variables.tf +++ b/build/modules/helm/variables.tf @@ -22,6 +22,10 @@ variable "agones_version" { default = "" } +variable "udp_expose" { + default = "true" +} + variable "host" {} variable "token" {} diff --git a/examples/terraform-submodules/eks/module.tf b/examples/terraform-submodules/eks/module.tf new file mode 100644 index 0000000000..a4af2a6a8d --- /dev/null +++ b/examples/terraform-submodules/eks/module.tf @@ -0,0 +1,67 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// Run: +// terraform apply [-var agones_version="1.0.0"] + +// Install latest version of agones +variable "agones_version" { + default = "1.0.0" +} +variable "cluster_name" { + default = "agones-cluster" +} + +variable "region" { + default = "us-west-2" +} + +provider "aws" { + version = "~> 2.31" + + region = var.region +} + +variable "machine_type" { default = "t2.large" } + +module "eks_cluster" { + source = "git::https://github.com/googleforgames/agones.git//build/modules/eks/?ref=master" + + machine_type = "${var.machine_type}" + cluster_name = "${var.cluster_name}" +} + +data "aws_eks_cluster_auth" "example" { + name = "${var.cluster_name}" +} + +module "helm_agones" { + source = "git::https://github.com/googleforgames/agones.git//build/modules/helm/?ref=master" + + udp_expose = "false" + agones_version = "${var.agones_version}" + values_file = "" + chart = "agones" + host = "${module.eks_cluster.host}" + token = "${data.aws_eks_cluster_auth.example.token}" + cluster_ca_certificate = "${module.eks_cluster.cluster_ca_certificate}" +} + +output "host" { + value = "${module.eks_cluster.host}" +} +output "cluster_ca_certificate" { + value = "${module.eks_cluster.cluster_ca_certificate}" +} diff --git a/site/content/en/docs/Installation/terraform.md b/site/content/en/docs/Installation/terraform.md index 142333cef4..26d05661af 100644 --- a/site/content/en/docs/Installation/terraform.md +++ b/site/content/en/docs/Installation/terraform.md @@ -9,14 +9,15 @@ description: > ## Prerequisites -- Terraform v0.12.3 +- Terraform v0.12 - [Helm](https://docs.helm.sh/helm/) package manager 2.10.0+ -- Access to the the Kubernetes hosting provider you are using (e.g. `gcloud` or `az` utility installed) +- Access to the the Kubernetes hosting provider you are using (e.g. `gcloud` +{{% feature publishVersion="1.1.0" %}}, `awscli`{{% /feature %}} or `az` utility installed) - Git # Installing the Agones as Terraform submodule on Google Kubernetes Engine -You can use Terraform to provision your GKE cluster and install agones on it using Helm Terraform provider. +You can use Terraform to provision your GKE cluster and install Agones on it using Helm Terraform provider. First step would be to enable `Kubernetes Engine API`. From the Cloud Console, navigate to APIs & Services > Dashboard, then click `Enable APIs and Services`. Type `kubernetes` in the search box, and you should find the Kubernetes Engine API. Click Enable. @@ -38,7 +39,7 @@ The example of submodule configuration could be found here: Configurable parameters and their meaning: - password - if not specified basic Auth would be disabled in GKE cluster -- agones_version - which version of agones to install +- agones_version - which version of Agones to install - project - your Google Cloud Project ID - machine_type - primary cluster machine type ( default is "n1-standard-4") - node_count - count of nodes in primary Node Pool. Defaults to "4". @@ -73,7 +74,7 @@ Fetching cluster endpoint and auth data. kubeconfig entry generated for test-cluster. ``` -Check that you have access to kubernetes cluster: +Check that you have an access to kubernetes cluster: ``` kubectl get nodes ``` @@ -118,7 +119,7 @@ Once you created all resources on AKS you can get the credentials so that you ca az aks get-credentials --resource-group agonesRG --name test-cluster ``` -Check that you have access to kubernetes cluster: +Check that you have an access to kubernetes cluster: ``` kubectl get nodes ``` @@ -132,3 +133,67 @@ terraform destroy ## Reference Details on how you can authenticate your AKS terraform provider using official [instructions](https://www.terraform.io/docs/providers/azurerm/auth/service_principal_client_secret.html) + + +{{% feature publishVersion="1.1.0" %}} +# Installing the Agones as Terraform submodule on AWS EKS + +You can use Terraform to provision your Amazon EKS (Elastic Kubernetes Service) cluster and install Agones on it using Helm Terraform provider. + +The example of EKS submodule config file could be found here: + {{< ghlink href="examples/terraform-submodules/eks/module.tf" >}}Terraform configuration with Agones submodule{{< /ghlink >}} + +Copy `module.tf` file into a separate folder. + +Configure your AWS CLI tool [CLI configure](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html): +``` +aws configure +``` + +Configure your terraform: +``` +terraform init +``` + +By editing `modules.tf` you can change some parameters as you need, for example EC2 instance type. Note that the maximum number of instances in the workers group is limited by 3. + +Now you can deploy Agones on EKS: +``` +terraform apply +``` + +After deploying the cluster with Agones, you can get or update your kubeconfig using next command: +``` +aws eks --region us-west-2 update-kubeconfig --name agones-cluster +``` + +With the following output: +``` +Added new context arn:aws:eks:us-west-2:601646756426:cluster/agones-cluster to /Users/alexander.apalikov/.kube/config +``` + +Switch `kubectl` context to recently created one: +``` +kubectl config use-context arn:aws:eks:us-west-2:601646756426:cluster/agones-cluster +``` + +Check that you have an access to kubernetes cluster: +``` +kubectl get nodes +``` + +## Uninstall the Agones and delete EKS cluster + +Run the following command to delete all Terraform provisioned resources: +``` +terraform destroy -target module.eks_cluster.module.eks --auto-approve +terraform destroy +``` + +> Note: there is a known issue with AWS Terraform provider: +https://github.com/terraform-providers/terraform-provider-aws/issues/9101 +> So `terraform destroy` could not succeed: +> `Error: Error waiting for internet gateway (igw-0c37628c5687d2d50) to detach: timeout while waiting for state to become 'detached' (last state: 'detaching', timeout: 15m0s)` +> In that case you should manually delete the provisioned resources using AWS Console. + +{{% /feature %}} \ No newline at end of file