From 162cf4c061954d995403bc4a0cfd7c696a132d55 Mon Sep 17 00:00:00 2001 From: Jacob Lerche Date: Wed, 8 May 2019 20:13:46 -0700 Subject: [PATCH] AWS EKS tutorial change to new terraform script (#463) * Extracts certain values to variables * Adds TF variables for AWS EKS tutorial, generalizes userdata.sh script * Adds some small corrections * Modifies aws-eks-tutorial to use the deploy/aws terraform script * Removes comments from variables.tf * Adds step in tutorial introduction * Adds explanation why scaling out is done with terraform * Adds warning about instance types breaking user data setup * Changes tutorial instace types of monitor and tidb to not use local ssd * Updates tutorial docs to reflect instances in tutorial var file --- deploy/aws/README.md | 2 +- deploy/aws/aws-tutorial.tfvars | 11 + deploy/aws/main.tf | 12 +- deploy/aws/userdata.sh | 35 +++ deploy/aws/variables.tf | 22 ++ docs/aws-eks-tutorial.md | 426 +++++---------------------------- 6 files changed, 141 insertions(+), 367 deletions(-) create mode 100644 deploy/aws/aws-tutorial.tfvars create mode 100644 deploy/aws/userdata.sh diff --git a/deploy/aws/README.md b/deploy/aws/README.md index b20dd90b85..f232fc3284 100644 --- a/deploy/aws/README.md +++ b/deploy/aws/README.md @@ -23,7 +23,7 @@ The default setup will create a new VPC and a t2.micro instance as bastion machi ``` shell $ git clone https://github.com/pingcap/tidb-operator -$ cd tidb-operator/cloud/aws +$ cd tidb-operator/deploy/aws $ terraform init $ terraform apply ``` diff --git a/deploy/aws/aws-tutorial.tfvars b/deploy/aws/aws-tutorial.tfvars new file mode 100644 index 0000000000..beba5bdc9c --- /dev/null +++ b/deploy/aws/aws-tutorial.tfvars @@ -0,0 +1,11 @@ +pd_instance_type = "c5d.large" +tikv_instance_type = "c5d.large" +tidb_instance_type = "c4.large" +monitor_instance_type = "c5.large" + +pd_count = 1 +tikv_count = 1 +tidb_count = 1 + +cluster_name = "aws_tutorial" +tikv_root_volume_size = "50" \ No newline at end of file diff --git a/deploy/aws/main.tf b/deploy/aws/main.tf index cb48cf0d6a..f5687f5b04 100644 --- a/deploy/aws/main.tf +++ b/deploy/aws/main.tf @@ -99,31 +99,31 @@ module "eks" { key_name = "${module.key-pair.key_name}" # WARNING: if you change instance type, you must also modify the corresponding disk mounting in pd-userdata.sh script # instance_type = "c5d.xlarge" # 4c, 8G, 100G NVMe SSD - instance_type = "m5d.xlarge" # 4c, 16G, 150G NVMe SSD + instance_type = "${var.pd_instance_type}" # m5d.xlarge 4c, 16G, 150G NVMe SSD root_volume_size = "50" # rest NVMe disk for PD data public_ip = false kubelet_extra_args = "--register-with-taints=dedicated=pd:NoSchedule --node-labels=dedicated=pd" asg_desired_capacity = "${var.pd_count}" asg_max_size = "${var.pd_count + 2}" - additional_userdata = "${file("pd-userdata.sh")}" + additional_userdata = "${file("userdata.sh")}" }, { # tikv name = "tikv_worker_group" key_name = "${module.key-pair.key_name}" # WARNING: if you change instance type, you must also modify the corresponding disk mounting in tikv-userdata.sh script - instance_type = "i3.2xlarge" # 8c, 61G, 1.9T NVMe SSD + instance_type = "${var.tikv_instance_type}" # i3.2xlarge 8c, 61G, 1.9T NVMe SSD root_volume_type = "gp2" root_volume_size = "100" public_ip = false kubelet_extra_args = "--register-with-taints=dedicated=tikv:NoSchedule --node-labels=dedicated=tikv" asg_desired_capacity = "${var.tikv_count}" asg_max_size = "${var.tikv_count + 2}" - additional_userdata = "${file("tikv-userdata.sh")}" + additional_userdata = "${file("userdata.sh")}" }, { # tidb name = "tidb_worker_group" key_name = "${module.key-pair.key_name}" - instance_type = "c4.4xlarge" # 16c, 30G + instance_type = "${var.tidb_instance_type}" # c4.4xlarge 16c, 30G root_volume_type = "gp2" root_volume_size = "100" public_ip = false @@ -134,7 +134,7 @@ module "eks" { { # monitor name = "monitor_worker_group" key_name = "${module.key-pair.key_name}" - instance_type = "c5.xlarge" # 4c, 8G + instance_type = "${var.monitor_instance_type}" # c5.xlarge 4c, 8G root_volume_type = "gp2" root_volume_size = "100" public_ip = false diff --git a/deploy/aws/userdata.sh b/deploy/aws/userdata.sh new file mode 100644 index 0000000000..123ba40add --- /dev/null +++ b/deploy/aws/userdata.sh @@ -0,0 +1,35 @@ +# set ulimits +cat < /etc/security/limits.d/99-tidb.conf +root soft nofile 1000000 +root hard nofile 1000000 +root soft core unlimited +root soft stack 10240 +EOF +# config docker ulimit +cp /usr/lib/systemd/system/docker.service /etc/systemd/system/docker.service +sed -i 's/LimitNOFILE=infinity/LimitNOFILE=1048576/' /etc/systemd/system/docker.service +sed -i 's/LimitNPROC=infinity/LimitNPROC=1048576/' /etc/systemd/system/docker.service +systemctl daemon-reload +systemctl restart docker + +# format and mount nvme disk +if grep nvme0n1 /etc/fstab || grep nvme1n1 /etc/fstab; then + echo "disk already mounted" +else + if mkfs -t ext4 /dev/nvme1n1 ; then + + mkdir -p /mnt/disks/ssd1 + cat <> /etc/fstab +/dev/nvme1n1 /mnt/disks/ssd1 ext4 defaults,nofail,noatime,nodelalloc 0 2 +EOF + mount -a + else + mkfs -t ext4 /dev/nvme0n1 + mkdir -p /mnt/disks/ssd1 + cat <> /etc/fstab +/dev/nvme0n1 /mnt/disks/ssd1 ext4 defaults,nofail,noatime,nodelalloc 0 2 +EOF + mount -a + fi +fi + diff --git a/deploy/aws/variables.tf b/deploy/aws/variables.tf index a612819a2d..4896870335 100644 --- a/deploy/aws/variables.tf +++ b/deploy/aws/variables.tf @@ -83,3 +83,25 @@ variable "tikv_count" { variable "tidb_count" { default = 2 } + +// Be careful about changing the instance types, it may break the user data and local volume setup +variable "pd_instance_type" { + default = "m5d.xlarge" +} + +variable "tikv_instance_type" { + default = "i3.2xlarge" +} + +variable "tidb_instance_type" { + default = "c4.4xlarge" +} + +variable "monitor_instance_type" { + default = "c5.xlarge" +} + +variable "tikv_root_volume_size" { + default = "100" +} + diff --git a/docs/aws-eks-tutorial.md b/docs/aws-eks-tutorial.md index 53179b0609..9097170ed2 100644 --- a/docs/aws-eks-tutorial.md +++ b/docs/aws-eks-tutorial.md @@ -6,19 +6,23 @@ category: operations # Deploy TiDB, a distributed MySQL compatible database, on Kubernetes via AWS EKS -## Introduction +## Requirements: +* [awscli](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) >= 1.16.73 +* [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) >= 1.11 +* [helm](https://github.com/helm/helm/blob/master/docs/install.md#installing-the-helm-client) >= 2.9.0 +* [jq](https://stedolan.github.io/jq/download/) +* [aws-iam-authenticator](https://github.com/kubernetes-sigs/aws-iam-authenticator#4-set-up-kubectl-to-use-authentication-tokens-provided-by-aws-iam-authenticator-for-kubernetes) +* [terraform](https://www.terraform.io/downloads.html) -This tutorial is designed to be run locally with tools like [AWS Command Line Interface](https://aws.amazon.com/cli/) and [Terraform](https://www.terraform.io/). The Terraform code will ultilize a relatively new AWS service called [Amazon Elastic Container Service for Kubernetes (Amazon EKS)](https://aws.amazon.com/eks). +## Introduction -This guide is for running Tidb cluster in a testing Kubernetes environment. The following steps will use certain unsecure configurations. Do not just follow this guide only to build your production db environment. +This tutorial is designed to be a self-contained deployment of a Kubernetes cluster on AWS EKS with a running TiDB installattion managed by the TiDB Kubernetes operator. It takes you through these steps: -- Launching a new 3-node Kubernetes cluster via Terraform code provisioning with AWS EKS (optional) -- Installing the Helm package manager for Kubernetes -- Deploying the TiDB Operator -- Deploying your first TiDB cluster +- Standing up a Kubernetes cluster with TiDB running inside - Connecting to TiDB +- Scale out the cluster - Shutting down down the Kubernetes cluster > Warning: Following this guide will create objects in your AWS account that will cost you money against your AWS bill. @@ -39,7 +43,7 @@ For simplicity you can just assign `AdministratorAccess` to the group this user belongs to. With more detailed permissions, you will have to be sure you also have `AmazonEKSClusterPolicy` and `AmazonEKSServicePolicy` for this user. -Then generate a pair of access keys and keep them safe locally. Now we can continue about using Terraform to provision a Kubernetes on AWS. +Then generate a pair of access keys and keep them safe locally. ## A bit more about Terraform @@ -69,25 +73,7 @@ unzip terraform* sudo mv terraform /usr/local/bin/ ``` -### Setting up your terraform config - -1. Before continuing, make sure you have create a new user (other than the - root user of your AWS account) in IAM, giving it enough permissions. - For simplicity you can just assign `AdministratorAccess` to the group this user - belongs to. With more detailed permissions, you will have to be sure you also have - `AmazonEKSClusterPolicy` and `AmazonEKSServicePolicy` for this user. -1. Use editor to open file `~/.aws/credentials` and edit it with your AWS keys associated with the user mentioned above such as: - ```ini - [default] - aws_access_key_id = XXX - aws_secret_access_key = XXX - ``` - -### (Optional) Verify Terraform setup - -You can confirm that Terraform is installed correctly by deploying the most simple infrastructure provision. From the Terraform manual: - -Prepare an example file +At this point, let us make sure that `awscli` and `terraform` are properly configured. Here is a simple example that will provision a `t2.micro` for us ```tf # example.tf @@ -115,373 +101,91 @@ Note that Terraform will automatically search for saved API credentials (for exa In our next step, we will be deploying infrastructure based on the Terraform EKS tutorial. -## About starting AWS EKS with Terraform - -Steps to provision AWS EKS: - -1. Provision an EKS cluster with IAM roles, Security Groups and VPC -2. Deploy worker nodes with Launch Configuration, Autoscaling Group, IAM Roles and Security Groups -3. Connect to EKS - -More detailed steps can follow [here](https://github.com/wardviaene/terraform-course/tree/master/eks-demo) and [here](https://github.com/terraform-providers/terraform-provider-aws/tree/master/examples/eks-getting-started), more a perhaps more detailed version [here](https://github.com/liufuyang/terraform-course/tree/master/eks-demo) (which is based on the first one). - -For now we will follow the configs from the [last link](https://github.com/liufuyang/terraform-course/tree/master/eks-demo). -As it should be more easier and has more related info. - ---- - -## Launch a 3-node Kubernetes cluster - -### Clone a Terraform code repo for AWS EKS - -See https://www.terraform.io/docs/providers/aws/guides/eks-getting-started.html for full guide. - -This step requires cloning a git repository which contains Terraform templates. - -Firstly clone [this repo](https://github.com/liufuyang/terraform-course/tree/master/eks-demo) and go to folder `eks-demo`: - -```sh -git clone https://github.com/liufuyang/terraform-course.git -cd terraform-course/eks-demo -``` - -> Note: the following guide assumes you are at the terraform file folder `eks-demo` from the repo mentioned above - -At the end of this guide you will be asked to come back to this `tidb-operator` folder -to continue the Tidb demo. - -### Download kubectl - -Download kubectl, the command line tool for Kubernetes - -```sh -# For macOS -curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/darwin/amd64/kubectl - -chmod +x kubectl -ln -s $(pwd)/kubectl /usr/local/bin/kubectl -``` - -```sh -# For linux -curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl - -chmod +x kubectl -ln -s $(pwd)/kubectl /usr/local/bin/kubectl -``` - -### Download the aws-iam-authenticator - -A tool to use AWS IAM credentials to authenticate to a Kubernetes cluster. - -```sh -# For macOS -wget -O heptio-authenticator-aws https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.3.0/heptio-authenticator-aws_0.3.0_darwin_amd64 - -chmod +x heptio-authenticator-aws -ln -s $(pwd)/heptio-authenticator-aws /usr/local/bin/heptio-authenticator-aws -``` - -```sh -# For linux -wget -O heptio-authenticator-aws https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.3.0/heptio-authenticator-aws_0.3.0_linux_amd64 - -chmod +x heptio-authenticator-aws -ln -s $(pwd)/heptio-authenticator-aws /usr/local/bin/heptio-authenticator-aws -``` - -### Modify providers.tf - -Choose your region. EKS is not available in every region, use the Region Table to check whether your region is supported: https://aws.amazon.com/about-aws/global-infrastructure/regional-product-services/ - -Make changes in providers.tf accordingly (region, optionally profile) - -### Terraform apply - -Start AWS EKS service and create a fully operational Kubernetes environment on your -AWS cloud. +## Running the Terraform script +The Terraform script is located in `./deploy/aws`. After changing directory, we can run the same commands as with our test example ```sh terraform init -terraform apply # This step may take more than 10 minutes to finish. Just wait patiently. -``` - -### Configure kubectl - -Export the configuration of the EKS cluster to `.config`. This will allow the `kubectl` command line client to access our newly deployed cluster. - -```sh -mkdir .config - -terraform output kubeconfig > .config/ekskubeconfig -# Save output in ~/.kube/config and then use the following env prarm - -export KUBECONFIG=$KUBECONFIG:$(pwd)/.config/ekskubeconfig -# Remember to run the above command again with the right folders when you open a new shell window later for TiDB related works. -``` - -### Configure config-map-auth-aws - -Setup a ConfigMap on Kubernetes so the node instances can have proper IAM roles. - -```sh -terraform output config-map-aws-auth > .config/config-map-aws-auth.yaml -# save output in config-map-aws-auth.yaml - -kubectl apply -f .config/config-map-aws-auth.yaml -``` - -### See service and nodes coming up - -Your `kubectl` command line client should now be installed and configured. Check the installed services and kubernetes nodes: - -```sh -kubectl get svc - -kubectl get nodes +terraform apply -var-file=aws-tutorial.tfvars ``` +Here, we are passing in a file that specifies values for certain variables to the `terraform apply` command. -More info about getting started with Amazon EKS[here](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) +The provisioning of all the infrastructure will take several minutes. We are creating: -### Deploy Kubernetes Dashboard +* 1 VPC +* 1 t2.micro as a bastion machine +* 1 internal ELB +* 1 c5d.large for PD pods +* 1 c5d.large for TiKV pods +* 1 c4.large for TiDB pods +* 1 c5.large for monitoring related pods -#### Step 1 (Optional): Deploy the Dashboard - -To allow an easy way of monitoring what's happening on your Kubernetes -cluster, it is very helpful to have a `kubernetes-dashboard` installed. -And you can then easily monitor every pod or service of your Kubernetes cluster in browser. +When everything has been successfully created, you will see something like this: ```sh -kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml - -kubectl apply -f https://raw.githubusercontent.com/kubernetes/heapster/master/deploy/kube-config/influxdb/heapster.yaml +Apply complete! Resources: 67 added, 0 changed, 0 destroyed. -kubectl apply -f https://raw.githubusercontent.com/kubernetes/heapster/master/deploy/kube-config/influxdb/influxdb.yaml -``` - -#### Step 2: Create an eks-admin Service Account and Cluster Role Binding +Outputs: -```sh -kubectl apply -f eks-admin-service-account.yaml -kubectl apply -f eks-admin-cluster-role-binding.yaml +bastion_ip = [ + 3.14.255.194 +] +eks_endpoint = https://8B49E8619835B8C79BD383B542161819.sk1.us-east-2.eks.amazonaws.com +eks_version = 1.12 +monitor_endpoint = http://a37987df9710211e9b48c0ae40bc8d7b-1847612729.us-east-2.elb.amazonaws.com:3000 +region = us-east-2 +tidb_dns = internal-a37a17c22710211e9b48c0ae40bc8d7b-1891023212.us-east-2.elb.amazonaws.com +tidb_port = 4000 +tidb_version = v2.1.8 ``` -#### Step 3: Connect to the Dashboard - -Now you can monitoring your Kubernetes cluster in browser: +> *NOTE*: Be careful about changing instance types for PD and TiKV worker groups as they rely on local SSD. Doing so may break user-data and local volume setup. +### Connecting to TiDB +Access to TiDB is gated behind the bastion machine. First ssh into it and then use the mysql client ```sh -# output a token for logging in kubernetes-dashboard -kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep eks-admin | awk '{print $1}') - -kubectl proxy +ssh -i credentials/k8s-prod-aws_tutorial.pem ec2-user@ +mysql -h -P -u root ``` -Then open http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/ +### Interacting with the Kubernetes cluster -And use the token output from previous command to login. +It is possible to interact with the cluster via `kubectl` and `helm` with the kubeconfig file that is created `credentials/kubeconfig_aws_tutorial`. -More info see [here](https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html). +```sh +# By specifying --kubeconfig +kubectl --kubeconfig credentials/kubeconfig_aws_tutorial get po -n tidb +helm --kubeconfig credentials/kubeconfig_aws_tutorial ls -#### Step 4: Create an AWS storage class for your Amazon EKS cluster - -Amazon EKS clusters are not created with any storage classes. You must define storage classes for your cluster to use and you should define a default storage class for your persistent volume claims. For the demo purpose, -we simply use the default `gp2` storage class. - -`gp2` is a general purpose SSD volume that balances price and performance for a wide variety of workloads. - -```sh -kubectl create -f gp2-storage-class.yaml -kubectl get storageclass -``` - -More info to see: -[here](https://docs.aws.amazon.com/eks/latest/userguide/storage-classes.html) , -[here](https://kubernetes.io/docs/concepts/storage/storage-classes/#aws) -and -[here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) - ---- - -Congratulations, now you have a Kubernetes cloud running on AWS. - -Then from this stage, you should be able to follow the [Tidb-Operator guide](https://github.com/pingcap/tidb-operator/blob/master/docs/google-kubernetes-tutorial.md) from section `Install Helm`, `Deploy TiDB Operator` and so on. - -Or just continue with our guide here, simply do: - -```sh -curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh -chmod 700 get_helm.sh && ./get_helm.sh -``` - -> Now we will be working under folder `tidb-operator` folder of `pingcap/tidb-operator` repo. - -```sh -# clone repo, if you haven't done it -git clone https://github.com/pingcap/tidb-operator.git -cd tidb-operator -``` - -Please note that if you have opened a new shell or terminal since you have -worked previously worked in the directory `eks-demo`, make sure you have the same `KUBECONFIG` setup again in the current shell, -otherwise `kubectl` command will not be able to -connect onto your newly created cluster. - -### Deploy tidb-operator - -TiDB Operator manages TiDB clusters on Kubernetes and automates tasks related to operating a TiDB cluster. It makes TiDB a truly cloud-native database. - -Uncomment the `scheduler.kubeSchedulerImage` in `values.yaml`, set it to the same as your kubernetes cluster version. - -If you already have helm installed, you can continue here to deploy tidb-operator: - -```sh -# within directory tidb-operator of repo https://github.com/pingcap/tidb-operator - -kubectl create serviceaccount tiller --namespace kube-system -kubectl apply -f ./manifests/tiller-rbac.yaml -helm init --service-account tiller --upgrade - - -# verify: -kubectl get pods --namespace kube-system | grep tiller - -# Deploy TiDB operator: - -kubectl apply -f ./manifests/crd.yaml -kubectl apply -f ./manifests/gp2-storage.yaml -helm install ./charts/tidb-operator -n tidb-admin --namespace=tidb-admin - -# verify: -kubectl get pods --namespace tidb-admin -o wide -``` - -### Adjust max-open-files - -See issue: -https://github.com/pingcap/tidb-operator/issues/91 - -Basically in un-comment all the `max-open-files` settings -in file `charts/tidb-cluster/templates/tikv-configmap.yaml` -and set everyone as: - -```yaml -max-open-files = 1024 -``` - -### Deploy your first TiDB cluster - -```sh -# Deploy your first TiDB cluster - -helm install ./charts/tidb-cluster -n demo --namespace=tidb --set pd.storageClassName=gp2,tikv.storageClassName=gp2 - -# Or if something goes wrong later and you want to update the deployment, use command: -# helm upgrade demo ./charts/tidb-cluster --namespace=tidb --set pd.storageClassName=gp2,tikv.storageClassName=gp2 - -# verify and wait until tidb-initializer pod status becomes completed: -kubectl get pods --namespace tidb -o wide -``` - -Then keep watching output of: - -```sh -watch "kubectl get svc -n tidb" -``` - -When you see `demo-tidb` appear, you can `Control + C` to stop watching. Then the service is ready to connect to! - -```sh -kubectl run -n tidb mysql-client --rm -i --tty --image mysql -- mysql -P 4000 -u root -h $(kubectl get svc demo-tidb -n tidb -o jsonpath='{.spec.clusterIP}') -p -``` - -Or you can forward ports and run a mysql client locally: - -```sh -kubectl -n tidb port-forward demo-tidb-0 4000:4000 &>/tmp/port-forward.log & -mysql -h 127.0.0.1 -u root -P 4000 --default-character-set=utf8 -p -``` - -Try out some mysql commands: - -```sh -select tidb_version(); -``` - -It works! :tada: - -If you did not specify a password in helm, set one now: - -```sh -SET PASSWORD FOR 'root'@'%' = -``` - - -### Monitoring with Grafana - -TiDB cluster by default ships with a Grafana monitoring page which -is very useful for DevOps. You can view it as well now. -Simply do a kubectl port-forward on the demo-monitor pod. - -```sh -# Note: you will check your cluster and use the actual demo-monitor pod's name instead of the example blow: -kubectl port-forward demo-monitor-5bc85fdb7f-qwl5h 3000 & +# or setting KUBECONFIG environment variable +export KUBECONFIG=$PWD/credentials/kubeconfig_aws_tutorial +kubectl get po -n tidb +helm ls ``` -Then visit [localhost:3000](localhost:3000) and login with default -username and password as `admin`, `admin`. - -(Optional) Or use a LoadBalancer and visit the site with public IP. -However this method is not recommended for demo here as it will create -extra resources in our AWS VPC, making the later `terraform destroy` -command cannot clean up all the resources. +### Viewing the Grafana dashboards -If you know what you are doing, then continue. Otherwise use the -`kubectl port-forward` command mentioned above. +Now that we know how to interact with the cluster, we can port-forward the Grafana service locally -```sh -# setting monitor.server.type = LoadBalancer in charts/tidb-cluster/values.yaml -# then do helm update again to update the tidb release -# then check loadbalancer external ip via command -kubectl get svc -n tidb +```bash +kubectl port-forward svc/tidb-cluster-grafana 3000:3000 -n tidb &>/dev/null & ``` -### Perhaps load some data to test your TiDB +We can now point a browser to `localhost:3000` and view the dashboards. -A good way of testing database is to use the `TPC-H` Benchmark. -One can easily generate some data and load into your TiDB via -[PingCap's `tidb-bench` repo](https://github.com/pingcap/tidb-bench/tree/master/tpch) -Please refer to the [README](https://github.com/pingcap/tidb-bench/tree/master/tpch) of the `tidb-bench` -to see how to easily generate some test data and load into TiDB. -And test TiDB with SQL queries in the [`queries` folder](https://github.com/pingcap/tidb-bench/tree/master/tpch/queries) +### Scale TiDB cluster -For example, with our 3 node EKS Kubernetes cluster created above, -with TPC-H data generated with scale factor 1 (local data file around 1GB), -the TiDB (V2.0) should be able to finish the first TPC-H query with 15 seconds. +To scale out TiDB cluster, modify `tikv_count` or `tidb_count` in `aws-tutorial.tfvars` to your desired count, and then run `terraform apply -var-file=aws-tutorial.tfvars`. -### Scale out the TiDB cluster - -With a single command we can easily scale out the TiDB cluster. To scale out TiKV: - -```sh -helm upgrade demo charts/tidb-cluster --set pd.storageClassName=gp2,tikv.storageClassName=gp2,tikv.replicas=5,tidb.replicas=3 -``` - -Now the number of TiKV pods is increased from the default 3 to 5. You can check it with: - -```sh -kubectl get po -n tidb -``` - -We have noticed with a 5 node EKS Kubernetes cluster, 1G TPC-H data, -the first TPC-H query can be finished within 10 seconds. +> *Note*: Currently, scaling in is not supported since we cannot determine which node to scale. Scaling out needs a few minutes to complete, you can watch the scaling out by `watch kubectl --kubeconfig credentials/kubeconfig_aws_tutorial get po -n tidb` +> *Note*: There are taints and tolerations in place such that only a single pod will be scheduled per node. The count is also passed onto helm via terraform. For this reason attempting to scale out pods via helm or `kubectl scale` will not work as expected. --- ## Destroy -At the end of the demo, please make sure all the resources created by Kubernetes are removed (LoadBalancers, Security groups), so you get a +At the end of the tutorial, please make sure all the resources created by Kubernetes are removed (LoadBalancers, Security groups), so you get a big bill from AWS. Simply run: @@ -490,4 +194,6 @@ Simply run: terraform destroy ``` -(Do this command at the end to clean up, you don't have to don't do it now!) +(Do this command at the end to clean up, you don't have to do it now!) + +> **NOTE:** You have to manually delete the EBS volumes after running `terraform destroy` if you don't need the data on the volumes any more.