From ca0a52bfeb8a3dbb86326863b34825405cec04a5 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 9 May 2019 15:26:57 +0800 Subject: [PATCH 01/18] support affinity --- .../tidb-cluster/templates/tidb-cluster.yaml | 21 +-- charts/tidb-cluster/values.yaml | 97 +++++++++++--- pkg/apis/pingcap.com/v1alpha1/types.go | 38 +++--- .../v1alpha1/zz_generated.deepcopy.go | 30 ++--- pkg/manager/member/pd_member_manager.go | 7 +- pkg/manager/member/tidb_member_manager.go | 7 +- pkg/manager/member/tikv_member_manager.go | 7 +- pkg/util/util.go | 120 ------------------ 8 files changed, 117 insertions(+), 210 deletions(-) diff --git a/charts/tidb-cluster/templates/tidb-cluster.yaml b/charts/tidb-cluster/templates/tidb-cluster.yaml index a0a91c4dab..f026d55e63 100644 --- a/charts/tidb-cluster/templates/tidb-cluster.yaml +++ b/charts/tidb-cluster/templates/tidb-cluster.yaml @@ -27,11 +27,8 @@ spec: {{- if .Values.pd.resources }} {{ toYaml .Values.pd.resources | indent 4 }} {{- end }} - {{- if .Values.pd.nodeSelector }} - nodeSelector: -{{ toYaml .Values.pd.nodeSelector | indent 6 }} - {{- end }} - nodeSelectorRequired: {{ .Values.nodeSelectorRequired | default true }} + affinity: +{{ toYaml .Values.pd.affinity | indent 6 }} {{- if .Values.pd.tolerations }} tolerations: {{ toYaml .Values.pd.tolerations | indent 4 }} @@ -46,11 +43,8 @@ spec: {{- if .Values.tikv.resources }} {{ toYaml .Values.tikv.resources | indent 4 }} {{- end }} - {{- if .Values.tikv.nodeSelector }} - nodeSelector: -{{ toYaml .Values.tikv.nodeSelector | indent 6 }} - {{- end }} - nodeSelectorRequired: {{ .Values.nodeSelectorRequired | default true }} + affinity: +{{ toYaml .Values.tikv.affinity | indent 6 }} {{- if .Values.tikv.tolerations }} tolerations: {{ toYaml .Values.tikv.tolerations | indent 4 }} @@ -68,11 +62,8 @@ spec: {{- if .Values.tidb.resources }} {{ toYaml .Values.tidb.resources | indent 4 }} {{- end }} - {{- if .Values.tidb.nodeSelector }} - nodeSelector: -{{ toYaml .Values.tidb.nodeSelector | indent 6 }} - {{- end }} - nodeSelectorRequired: {{ .Values.nodeSelectorRequired | default true }} + affinity: +{{ toYaml .Values.tidb.affinity | indent 6 }} {{- if .Values.tidb.tolerations }} tolerations: {{ toYaml .Values.tidb.tolerations | indent 4 }} diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 6d264f0986..0dbe70da4e 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -66,16 +66,68 @@ pd: # cpu: 4000m # memory: 4Gi storage: 1Gi - # nodeSelector is used for scheduling pod, - # if nodeSelectorRequired is true, all the following labels must be matched - nodeSelector: {} - # kind: pd - # # zone is comma separated availability zone list - # zone: cn-bj1-01,cn-bj1-02 - # # region is comma separated region list - # region: cn-bj1 - # Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints. - # refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration + + ## affinity defines pd scheduling rules,it's default settings is empty. + ## please read the affinity document before set your scheduling rule: + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + ## The following is typical example of affinity settings: + ## The PodAntiAffinity setting of the example keeps PD pods does not co-locate on a topology node as far as possible to improve the high availability of PD on Kubernetes. + ## The NodeAffinity setting of the example ensure that the PD pods can only be scheduled to nodes with label:[type="pd"], + # affinity: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # # this term work when the nodes have the label named region + # - weight: 10 + # podAffinityTerm: + # labelSelector: + # matchLabels: + # app.kubernetes.io/instance: + # app.kubernetes.io/component: "pd" + # topologyKey: "region" + # namespaces: + # - + # # this term work when the nodes have the label named zone + # - weight: 20 + # podAffinityTerm: + # labelSelector: + # matchLabels: + # app.kubernetes.io/instance: + # app.kubernetes.io/component: "pd" + # topologyKey: "zone" + # namespaces: + # - + # # this term work when the nodes have the label named rack + # - weight: 40 + # podAffinityTerm: + # labelSelector: + # matchLabels: + # app.kubernetes.io/instance: + # app.kubernetes.io/component: "pd" + # topologyKey: "rack" + # namespaces: + # - + # # this term work when the nodes have the label named kubernetes.io/hostname + # - weight: 80 + # podAffinityTerm: + # labelSelector: + # matchLabels: + # app.kubernetes.io/instance: + # app.kubernetes.io/component: "pd" + # topologyKey: "kubernetes.io/hostname" + # namespaces: + # - + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: "kind" + # operator: In + # values: + # - "pd" + + ## Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints. + ## refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration tolerations: [] # - key: node-role # operator: Equal @@ -107,10 +159,14 @@ tikv: # cpu: 12000m # memory: 24Gi storage: 10Gi - nodeSelector: {} - # kind: tikv - # zone: cn-bj1-01,cn-bj1-02 - # region: cn-bj1 + + ## affinity defines tikv scheduling rules,affinity default settings is empty. + ## please read the affinity document before set your scheduling rule: + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints. + ## refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration tolerations: [] # - key: node-role # operator: Equal @@ -166,10 +222,15 @@ tidb: requests: {} # cpu: 12000m # memory: 12Gi - nodeSelector: {} - # kind: tidb - # zone: cn-bj1-01,cn-bj1-02 - # region: cn-bj1 + + + ## affinity defines tikv scheduling rules,affinity default settings is empty. + ## please read the affinity document before set your scheduling rule: + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints. + ## refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration tolerations: [] # - key: node-role # operator: Equal diff --git a/pkg/apis/pingcap.com/v1alpha1/types.go b/pkg/apis/pingcap.com/v1alpha1/types.go index 458543df41..28579695a4 100644 --- a/pkg/apis/pingcap.com/v1alpha1/types.go +++ b/pkg/apis/pingcap.com/v1alpha1/types.go @@ -15,10 +15,9 @@ package v1alpha1 import ( apps "k8s.io/api/apps/v1beta1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - - corev1 "k8s.io/api/core/v1" ) const ( @@ -109,25 +108,23 @@ type TidbClusterStatus struct { // PDSpec contains details of PD member type PDSpec struct { ContainerSpec - Replicas int32 `json:"replicas"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - NodeSelectorRequired bool `json:"nodeSelectorRequired,omitempty"` - StorageClassName string `json:"storageClassName,omitempty"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + Replicas int32 `json:"replicas"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + StorageClassName string `json:"storageClassName,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` } // TiDBSpec contains details of PD member type TiDBSpec struct { ContainerSpec - Replicas int32 `json:"replicas"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - NodeSelectorRequired bool `json:"nodeSelectorRequired,omitempty"` - StorageClassName string `json:"storageClassName,omitempty"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - BinlogEnabled bool `json:"binlogEnabled,omitempty"` - MaxFailoverCount int32 `json:"maxFailoverCount,omitempty"` - SeparateSlowLog bool `json:"separateSlowLog,omitempty"` - SlowLogTailer TiDBSlowLogTailerSpec `json:"slowLogTailer,omitempty"` + Replicas int32 `json:"replicas"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + StorageClassName string `json:"storageClassName,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + BinlogEnabled bool `json:"binlogEnabled,omitempty"` + MaxFailoverCount int32 `json:"maxFailoverCount,omitempty"` + SeparateSlowLog bool `json:"separateSlowLog,omitempty"` + SlowLogTailer TiDBSlowLogTailerSpec `json:"slowLogTailer,omitempty"` } // TiDBSlowLogTailerSpec represents an optional log tailer sidecar with TiDB @@ -138,11 +135,10 @@ type TiDBSlowLogTailerSpec struct { // TiKVSpec contains details of PD member type TiKVSpec struct { ContainerSpec - Replicas int32 `json:"replicas"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - NodeSelectorRequired bool `json:"nodeSelectorRequired,omitempty"` - StorageClassName string `json:"storageClassName,omitempty"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + Replicas int32 `json:"replicas"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + StorageClassName string `json:"storageClassName,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` } // TiKVPromGatewaySpec runs as a sidecar with TiKVSpec diff --git a/pkg/apis/pingcap.com/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap.com/v1alpha1/zz_generated.deepcopy.go index 9ae44e85c5..4b828246dc 100644 --- a/pkg/apis/pingcap.com/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap.com/v1alpha1/zz_generated.deepcopy.go @@ -89,12 +89,10 @@ func (in *PDMember) DeepCopy() *PDMember { func (in *PDSpec) DeepCopyInto(out *PDSpec) { *out = *in in.ContainerSpec.DeepCopyInto(&out.ContainerSpec) - if in.NodeSelector != nil { - in, out := &in.NodeSelector, &out.NodeSelector - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations @@ -238,12 +236,10 @@ func (in *TiDBSlowLogTailerSpec) DeepCopy() *TiDBSlowLogTailerSpec { func (in *TiDBSpec) DeepCopyInto(out *TiDBSpec) { *out = *in in.ContainerSpec.DeepCopyInto(&out.ContainerSpec) - if in.NodeSelector != nil { - in, out := &in.NodeSelector, &out.NodeSelector - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations @@ -338,12 +334,10 @@ func (in *TiKVPromGatewaySpec) DeepCopy() *TiKVPromGatewaySpec { func (in *TiKVSpec) DeepCopyInto(out *TiKVSpec) { *out = *in in.ContainerSpec.DeepCopyInto(&out.ContainerSpec) - if in.NodeSelector != nil { - in, out := &in.NodeSelector, &out.NodeSelector - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index 8d5b0797b9..f9d51f70a7 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -472,12 +472,7 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) }, Spec: corev1.PodSpec{ SchedulerName: tc.Spec.SchedulerName, - Affinity: util.AffinityForNodeSelector( - ns, - tc.Spec.PD.NodeSelectorRequired, - label.New().Instance(instanceName).PD(), - tc.Spec.PD.NodeSelector, - ), + Affinity: tc.Spec.PD.Affinity, Containers: []corev1.Container{ { Name: v1alpha1.PDMemberType.String(), diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 4a4eb51e76..5706b9fde8 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -343,12 +343,7 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust }, Spec: corev1.PodSpec{ SchedulerName: tc.Spec.SchedulerName, - Affinity: util.AffinityForNodeSelector( - ns, - tc.Spec.TiDB.NodeSelectorRequired, - label.New().Instance(instanceName).TiDB(), - tc.Spec.TiDB.NodeSelector, - ), + Affinity: tc.Spec.TiDB.Affinity, Containers: containers, RestartPolicy: corev1.RestartPolicyAlways, Tolerations: tc.Spec.TiDB.Tolerations, diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 8fe442a2bf..aaab1ea88e 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -332,12 +332,7 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) }, Spec: corev1.PodSpec{ SchedulerName: tc.Spec.SchedulerName, - Affinity: util.AffinityForNodeSelector( - ns, - tc.Spec.TiKV.NodeSelectorRequired, - tikvLabel, - tc.Spec.TiKV.NodeSelector, - ), + Affinity: tc.Spec.TiKV.Affinity, Containers: []corev1.Container{ { Name: v1alpha1.TiKVMemberType.String(), diff --git a/pkg/util/util.go b/pkg/util/util.go index acb5a03c0f..ba196f362c 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -15,7 +15,6 @@ package util import ( "fmt" - "sort" "strconv" "strings" @@ -23,127 +22,8 @@ import ( "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/pkg/kubelet/apis" ) -var ( - // weight is in range 1-100 - topologySchedulingWeight = map[string]int32{ - "region": 10, - "zone": 20, - "rack": 40, - apis.LabelHostname: 80, - } -) - -// AntiAffinityForPod creates a PodAntiAffinity with antiLabels -func AntiAffinityForPod(namespace string, antiLabels map[string]string) *corev1.PodAntiAffinity { - keys := []string{} - for key := range topologySchedulingWeight { - keys = append(keys, key) - } - sort.Strings(keys) // we must use sorted selector, otherwise affinity may vary causing new statefulset generated and pod recreated - terms := []corev1.WeightedPodAffinityTerm{} - for _, key := range keys { - term := corev1.WeightedPodAffinityTerm{ - Weight: topologySchedulingWeight[key], - PodAffinityTerm: corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{MatchLabels: antiLabels}, - TopologyKey: key, - Namespaces: []string{namespace}, - }, - } - terms = append(terms, term) - } - return &corev1.PodAntiAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: terms, - } -} - -// AffinityForNodeSelector creates an Affinity for NodeSelector -// Externally we use NodeSelector for simplicity, -// while internally we convert it to affinity which can express complex scheduling rules -func AffinityForNodeSelector(namespace string, required bool, antiLabels, selector map[string]string) *corev1.Affinity { - if selector == nil { - return nil - } - affinity := &corev1.Affinity{} - if antiLabels != nil { - affinity.PodAntiAffinity = AntiAffinityForPod(namespace, antiLabels) - } - - keys := []string{} - for key := range selector { - keys = append(keys, key) - } - sort.Strings(keys) // we must use sorted selector, otherwise affinity may vary causing new statefulset generated and pod recreated - - requiredTerms := []corev1.NodeSelectorTerm{} - if required { // all nodeSelectors are required - var exps []corev1.NodeSelectorRequirement - for _, key := range keys { - requirement := corev1.NodeSelectorRequirement{ - Key: key, - Operator: corev1.NodeSelectorOpIn, - Values: strings.Split(selector[key], ","), - } - // NodeSelectorRequirement in the same MatchExpressions are ANDed otherwise ORed - exps = append(exps, requirement) - } - requiredTerms = append(requiredTerms, corev1.NodeSelectorTerm{MatchExpressions: exps}) - affinity.NodeAffinity = &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: requiredTerms, - }, - } - return affinity - } - - preferredTerms := []corev1.PreferredSchedulingTerm{} - exps := []corev1.NodeSelectorRequirement{} - for _, key := range keys { - if selector[key] == "" { - continue - } - values := strings.Split(selector[key], ",") - // region,zone,rack,host are preferred labels, others are must match labels - if weight, ok := topologySchedulingWeight[key]; ok { - t := corev1.PreferredSchedulingTerm{ - Weight: weight, - Preference: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: key, - Operator: corev1.NodeSelectorOpIn, - Values: values, - }, - }, - }, - } - preferredTerms = append(preferredTerms, t) - } else { - requirement := corev1.NodeSelectorRequirement{ - Key: key, - Operator: corev1.NodeSelectorOpIn, - Values: values, - } - // NodeSelectorRequirement in the same MatchExpressions are ANDed otherwise ORed - exps = append(exps, requirement) - } - } - requiredTerms = append(requiredTerms, corev1.NodeSelectorTerm{MatchExpressions: exps}) - - affinity.NodeAffinity = &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: requiredTerms, - }, - PreferredDuringSchedulingIgnoredDuringExecution: preferredTerms, - } - - return affinity -} - // ResourceRequirement creates ResourceRequirements for MemberSpec // Optionally pass in a default value func ResourceRequirement(spec v1alpha1.ContainerSpec, defaultRequests ...corev1.ResourceRequirements) corev1.ResourceRequirements { From 2a3acc3cd6ceec1ed48b73cb6f882c66a8c245cb Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Fri, 10 May 2019 20:18:39 +0800 Subject: [PATCH 02/18] change user guide --- .../templates/config/_pd-config.tpl | 2 +- docs/operation-guide.md | 61 ++++++++++++++++--- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/charts/tidb-cluster/templates/config/_pd-config.tpl b/charts/tidb-cluster/templates/config/_pd-config.tpl index daea702054..ea5d044f5a 100644 --- a/charts/tidb-cluster/templates/config/_pd-config.tpl +++ b/charts/tidb-cluster/templates/config/_pd-config.tpl @@ -82,7 +82,7 @@ max-replicas = {{ .Values.pd.maxReplicas }} # The placement priorities is implied by the order of label keys. # For example, ["zone", "rack"] means that we should place replicas to # different zones first, then to different racks if we don't have enough zones. -location-labels = ["zone", "rack", "host"] +location-labels = ["region", "zone", "rack", "host"] [label-property] # Do not assign region leaders to stores that have these tags. diff --git a/docs/operation-guide.md b/docs/operation-guide.md index 8ab0fc53f2..a30bd20ab9 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -11,21 +11,68 @@ $ namespace="tidb" > **Note:** The rest of the document will use `values.yaml` to reference `charts/tidb-cluster/values.yaml` +## Configuration + +TiDB Operator use `values.yaml` as TiDB cluster configuration file. It provides the default basic configuration which you can use directly for quick deployment, but if you have specific configuration requirements or for production deployment, you need to manually modify the variables in the `value.yaml` file. + +* Resource setting + + * CPU & Memory + + The default deployment doesn't set CPU and memory requests or limits for any of the pods, these settings can make TiDB cluster run on a small Kubernetes cluster like DinD or the default GKE cluster for testing. But for production deployment, you would likely to adjust the cpu, memory and storage resources according to the [recommendations](https://github.com/pingcap/docs/blob/master/op-guide/recommendation.md). + The resource limits should be equal or bigger than the resource requests, it is suggested to set limit and request equal to get [`Guaranteed` QoS]( https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/#create-a-pod-that-gets-assigned-a-qos-class-of-guaranteed). + + * Storage + + The variables `pd.storageClassName` and `tikv.storageClassName` in `values.yaml` are used to set `StorageClass` of pd and tikv,their default setting are `local-storage` with minimal size. + If you don't want to use the default `StorageClass` or your Kubernetes cluster does not support `local-storage` class, please execute the following command to find an available `StorageClass` and select the ones you want to provide to TiDB cluster. + + ```shell + $ kubectl get sc + ``` + +* HA setting + + TiDB cluster is a distributed database. Its high availability means that when any physical node failed, not only to ensure TiDB server is available, but also ensure the data is complete and available. + + How to guarantee high availability of TiDB cluster work on Kubernetes? + + We mainly solve the problem from the scheduling of services and data. + + * HA guarantee of TiDB server + + TiDB Operator provides a external scheduler to guarantee PD/TiKV/TiDB pods HA on host level. TiDB Cluster have set the external scheduler as default scheduler, you will find the setting in the variable `schedulerName` of `values.yaml`. + + In the other hand use `PodAntiAffinity` term of `affinity` to ensure HA on the other topology levels (e.g. rack, zone, region). + refer to the doc: [pod affnity & anti affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity-beta-feature), moreover `values.yaml` also provides a typical HA setting example in the comments of `pd.affinity`. + + * HA guarantee of data + + HA of data is guaranteed by TiDB Cluster itself. The only work Operator needs to do is that collects topology info from specific labels of Kubernetes nodes where TiKV Pod runs on and then PD will schedule data replicas auto according to the topology info. + Cause currently TiDB Operator can only recognize some specific labels, so you can only set nodes topology info with the following particular labels + + * `region`: region where node is located + * `zone`: zone where node is located + * `rack`: rack where node is located + * `kubernetes.io/hostname`: hostname of the node + + you can label topology info to nodes of Kubernetes cluster use the following command + ```shell + # The labels are optional + $ kubectl label node region= zone= rack= kubernetes.io/hostname= + ``` + +For other settings, the variables in `values.yaml` are self-explanatory with comments. You can modify them according to your need before installing the charts. + ## Deploy TiDB cluster -After TiDB Operator and Helm are deployed correctly, TiDB cluster can be deployed using following command: +After TiDB Operator and Helm are deployed correctly and configuration completed, TiDB cluster can be deployed using following command: ```shell $ helm install charts/tidb-cluster --name=${releaseName} --namespace=${namespace} $ kubectl get po -n ${namespace} -l app.kubernetes.io/instance=${releaseName} ``` -The default deployment doesn't set CPU and memory requests or limits for any of the pods, and the storage used is `local-storage` with minimal size. These settings can make TiDB cluster run on a small Kubernetes cluster like DinD or the default GKE cluster for testing. But for production deployment, you would likely to adjust the cpu, memory and storage resources according to the [recommendations](https://github.com/pingcap/docs/blob/master/op-guide/recommendation.md). - -The resource limits should be equal or bigger than the resource requests, it is suggested to set limit and request equal to get [`Guaranteed` QoS]( https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/#create-a-pod-that-gets-assigned-a-qos-class-of-guaranteed). - -For other settings, the variables in `values.yaml` are self-explanatory with comments. You can modify them according to your need before installing the charts. - ## Access TiDB cluster By default TiDB service is exposed using [`NodePort`](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport). You can modify it to `ClusterIP` which will disable access from outside of the cluster. Or modify it to [`LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) if the underlining Kubernetes supports this kind of service. From c8d533bdea80760706b8539c20d0e15d21927c1d Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Tue, 14 May 2019 20:08:53 +0800 Subject: [PATCH 03/18] fix doc --- charts/tidb-cluster/values.yaml | 2 +- docs/operation-guide.md | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index e2975d19f9..4a44053d17 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -72,7 +72,7 @@ pd: ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} ## The following is typical example of affinity settings: - ## The PodAntiAffinity setting of the example keeps PD pods does not co-locate on a topology node as far as possible to improve the high availability of PD on Kubernetes. + ## The PodAntiAffinity setting of the example keeps PD pods does not co-locate on a topology node as far as possible to improve the disaster tolerance of PD on Kubernetes. ## The NodeAffinity setting of the example ensure that the PD pods can only be scheduled to nodes with label:[type="pd"], # affinity: # podAntiAffinity: diff --git a/docs/operation-guide.md b/docs/operation-guide.md index a30bd20ab9..4961ddc50d 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -31,24 +31,24 @@ TiDB Operator use `values.yaml` as TiDB cluster configuration file. It provides $ kubectl get sc ``` -* HA setting +* Disaster Tolerance setting - TiDB cluster is a distributed database. Its high availability means that when any physical node failed, not only to ensure TiDB server is available, but also ensure the data is complete and available. + TiDB cluster is a distributed database. Its Disaster Tolerance means that when any physical node failed, not only to ensure TiDB server is available, but also ensure the data is complete and available. - How to guarantee high availability of TiDB cluster work on Kubernetes? + How to guarantee Disaster Tolerance of TiDB cluster on Kubernetes? We mainly solve the problem from the scheduling of services and data. - * HA guarantee of TiDB server + * Disaster Tolerance of TiDB server - TiDB Operator provides a external scheduler to guarantee PD/TiKV/TiDB pods HA on host level. TiDB Cluster have set the external scheduler as default scheduler, you will find the setting in the variable `schedulerName` of `values.yaml`. + TiDB Operator provides a external scheduler to guarantee PD/TiKV/TiDB server disaster tolerance on host level. TiDB Cluster have set the external scheduler as default scheduler, you will find the setting in the variable `schedulerName` of `values.yaml`. - In the other hand use `PodAntiAffinity` term of `affinity` to ensure HA on the other topology levels (e.g. rack, zone, region). - refer to the doc: [pod affnity & anti affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity-beta-feature), moreover `values.yaml` also provides a typical HA setting example in the comments of `pd.affinity`. + In the other hand use `PodAntiAffinity` term of `affinity` to ensure disaster tolerance on the other topology levels (e.g. rack, zone, region). + refer to the doc: [pod affnity & anti affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity-beta-feature), moreover `values.yaml` also provides a typical disaster tolerance setting example in the comments of `pd.affinity`. - * HA guarantee of data + * Disaster Tolerance of data - HA of data is guaranteed by TiDB Cluster itself. The only work Operator needs to do is that collects topology info from specific labels of Kubernetes nodes where TiKV Pod runs on and then PD will schedule data replicas auto according to the topology info. + Disaster tolerance of data is guaranteed by TiDB Cluster itself. The only work Operator needs to do is that collects topology info from specific labels of Kubernetes nodes where TiKV Pod runs on and then PD will schedule data replicas auto according to the topology info. Cause currently TiDB Operator can only recognize some specific labels, so you can only set nodes topology info with the following particular labels * `region`: region where node is located @@ -56,7 +56,7 @@ TiDB Operator use `values.yaml` as TiDB cluster configuration file. It provides * `rack`: rack where node is located * `kubernetes.io/hostname`: hostname of the node - you can label topology info to nodes of Kubernetes cluster use the following command + you need label topology info to nodes of Kubernetes cluster use the following command ```shell # The labels are optional $ kubectl label node region= zone= rack= kubernetes.io/hostname= From 34be5bdeae0107790d645f08cf8f97c52c0d0c2b Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Tue, 14 May 2019 20:17:37 +0800 Subject: [PATCH 04/18] remove useless test --- pkg/util/utils_test.go | 196 ----------------------------------------- 1 file changed, 196 deletions(-) diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go index 2163883401..77101bd5d6 100644 --- a/pkg/util/utils_test.go +++ b/pkg/util/utils_test.go @@ -20,204 +20,8 @@ import ( "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/pkg/kubelet/apis" ) -func TestAffinityForNodeSelector(t *testing.T) { - g := NewGomegaWithT(t) - type testcase struct { - name string - required bool - antiLabels map[string]string - selector map[string]string - expectFn func(*GomegaWithT, *corev1.Affinity) - } - - antiLabels := map[string]string{"region": "region1", "zone": "zone1", "rack": "rack1", apis.LabelHostname: "host1"} - testFn := func(test *testcase, t *testing.T) { - t.Log(test.name) - test.expectFn(g, AffinityForNodeSelector(metav1.NamespaceDefault, test.required, test.antiLabels, test.selector)) - } - - tests := []testcase{ - { - name: "selector is nil", - required: false, - antiLabels: nil, - selector: nil, - expectFn: func(g *GomegaWithT, affinity *corev1.Affinity) { - g.Expect(affinity).To(BeNil()) - }, - }, - { - name: "required, antiLabels is nil", - required: true, - antiLabels: nil, - selector: map[string]string{"a": "a1,a2,a3", "b": "b1"}, - expectFn: func(g *GomegaWithT, affinity *corev1.Affinity) { - affi := &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "a", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"a1", "a2", "a3"}, - }, - { - Key: "b", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"b1"}, - }, - }, - }, - }, - }, - }, - } - g.Expect(affinity).To(Equal(affi)) - }, - }, - { - name: "required, antiLabels is not nil", - required: true, - antiLabels: antiLabels, - selector: map[string]string{"a": "a1,a2,a3", "b": "b1"}, - expectFn: func(g *GomegaWithT, affinity *corev1.Affinity) { - affi := &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "a", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"a1", "a2", "a3"}, - }, - { - Key: "b", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"b1"}, - }, - }, - }, - }, - }, - }, - PodAntiAffinity: &corev1.PodAntiAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ - { - Weight: 80, - PodAffinityTerm: corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{MatchLabels: antiLabels}, - TopologyKey: apis.LabelHostname, - Namespaces: []string{metav1.NamespaceDefault}, - }, - }, - { - Weight: 40, - PodAffinityTerm: corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{MatchLabels: antiLabels}, - TopologyKey: "rack", - Namespaces: []string{metav1.NamespaceDefault}, - }, - }, - { - Weight: 10, - PodAffinityTerm: corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{MatchLabels: antiLabels}, - TopologyKey: "region", - Namespaces: []string{metav1.NamespaceDefault}, - }, - }, - { - Weight: 20, - PodAffinityTerm: corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{MatchLabels: antiLabels}, - TopologyKey: "zone", - Namespaces: []string{metav1.NamespaceDefault}, - }, - }, - }, - }, - } - g.Expect(affinity).To(Equal(affi)) - }, - }, - { - name: "not required", - required: false, - antiLabels: nil, - selector: map[string]string{ - "region": "region1", - "zone": "zone1,zone2", - "rack": "", - "a": "a1,a2,a3", - "b": "b1", - }, - expectFn: func(g *GomegaWithT, affinity *corev1.Affinity) { - affi := &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "a", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"a1", "a2", "a3"}, - }, - { - Key: "b", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"b1"}, - }, - }, - }, - }, - }, - PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ - { - Weight: 10, - Preference: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "region", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"region1"}, - }, - }, - }, - }, - { - Weight: 20, - Preference: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "zone", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"zone1", "zone2"}, - }, - }, - }, - }, - }, - }, - } - g.Expect(affinity).To(Equal(affi)) - }, - }, - } - - for i := range tests { - testFn(&tests[i], t) - } -} - func TestResourceRequirement(t *testing.T) { g := NewGomegaWithT(t) type testcase struct { From 69e09d52a1162b79c6039bb8a350f2bf7c6fce39 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Wed, 15 May 2019 16:20:55 +0800 Subject: [PATCH 05/18] fix comment --- charts/tidb-cluster/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 4a44053d17..e07b238e70 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -82,7 +82,7 @@ pd: # podAffinityTerm: # labelSelector: # matchLabels: - # app.kubernetes.io/instance: + # app.kubernetes.io/instance: # app.kubernetes.io/component: "pd" # topologyKey: "region" # namespaces: @@ -92,7 +92,7 @@ pd: # podAffinityTerm: # labelSelector: # matchLabels: - # app.kubernetes.io/instance: + # app.kubernetes.io/instance: # app.kubernetes.io/component: "pd" # topologyKey: "zone" # namespaces: @@ -102,7 +102,7 @@ pd: # podAffinityTerm: # labelSelector: # matchLabels: - # app.kubernetes.io/instance: + # app.kubernetes.io/instance: # app.kubernetes.io/component: "pd" # topologyKey: "rack" # namespaces: @@ -112,7 +112,7 @@ pd: # podAffinityTerm: # labelSelector: # matchLabels: - # app.kubernetes.io/instance: + # app.kubernetes.io/instance: # app.kubernetes.io/component: "pd" # topologyKey: "kubernetes.io/hostname" # namespaces: From 3e83fe637b423f86eb9725d99ebe7db4485a912a Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Wed, 15 May 2019 19:46:25 +0800 Subject: [PATCH 06/18] Update charts/tidb-cluster/values.yaml Co-Authored-By: weekface --- charts/tidb-cluster/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index e07b238e70..15d360256b 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -86,7 +86,7 @@ pd: # app.kubernetes.io/component: "pd" # topologyKey: "region" # namespaces: - # - + # - # # this term work when the nodes have the label named zone # - weight: 20 # podAffinityTerm: From 882555f4ee98fd601ece5bfaec9fc694bb4b5b16 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 16 May 2019 16:00:37 +0800 Subject: [PATCH 07/18] address comment --- charts/tidb-cluster/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 15d360256b..55a9b32a0f 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -96,7 +96,7 @@ pd: # app.kubernetes.io/component: "pd" # topologyKey: "zone" # namespaces: - # - + # - # # this term work when the nodes have the label named rack # - weight: 40 # podAffinityTerm: @@ -106,7 +106,7 @@ pd: # app.kubernetes.io/component: "pd" # topologyKey: "rack" # namespaces: - # - + # - # # this term work when the nodes have the label named kubernetes.io/hostname # - weight: 80 # podAffinityTerm: @@ -116,7 +116,7 @@ pd: # app.kubernetes.io/component: "pd" # topologyKey: "kubernetes.io/hostname" # namespaces: - # - + # - # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: # nodeSelectorTerms: From da2d9d54cde9f3ce63f48cff5a7dce3d78f4d4f9 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 16 May 2019 16:31:09 +0800 Subject: [PATCH 08/18] Update docs/operation-guide.md Co-Authored-By: weekface --- docs/operation-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operation-guide.md b/docs/operation-guide.md index 4961ddc50d..b5eae883c6 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -13,7 +13,7 @@ $ namespace="tidb" ## Configuration -TiDB Operator use `values.yaml` as TiDB cluster configuration file. It provides the default basic configuration which you can use directly for quick deployment, but if you have specific configuration requirements or for production deployment, you need to manually modify the variables in the `value.yaml` file. +TiDB Operator uses `values.yaml` as TiDB cluster configuration file. It provides the default basic configuration which you can use directly for quick deployment, but if you have specific configuration requirements or for production deployment, you need to manually modify the variables in the `value.yaml` file. * Resource setting From 388a40481bbacc6fe6a6070874eaa1c2019d9488 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 16 May 2019 16:32:28 +0800 Subject: [PATCH 09/18] Update docs/operation-guide.md Co-Authored-By: weekface --- docs/operation-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operation-guide.md b/docs/operation-guide.md index b5eae883c6..a45d4d7cb3 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -24,7 +24,7 @@ TiDB Operator uses `values.yaml` as TiDB cluster configuration file. It provides * Storage - The variables `pd.storageClassName` and `tikv.storageClassName` in `values.yaml` are used to set `StorageClass` of pd and tikv,their default setting are `local-storage` with minimal size. + The variables `pd.storageClassName` and `tikv.storageClassName` in `values.yaml` are used to set `StorageClass` of PD and TiKV,their default setting are `local-storage` with minimal size. If you don't want to use the default `StorageClass` or your Kubernetes cluster does not support `local-storage` class, please execute the following command to find an available `StorageClass` and select the ones you want to provide to TiDB cluster. ```shell From dfcdda709c3c1ffdc225004268b8580aced55bc0 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 16 May 2019 16:32:51 +0800 Subject: [PATCH 10/18] Update docs/operation-guide.md Co-Authored-By: weekface --- docs/operation-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operation-guide.md b/docs/operation-guide.md index a45d4d7cb3..47d8090d92 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -33,7 +33,7 @@ TiDB Operator uses `values.yaml` as TiDB cluster configuration file. It provides * Disaster Tolerance setting - TiDB cluster is a distributed database. Its Disaster Tolerance means that when any physical node failed, not only to ensure TiDB server is available, but also ensure the data is complete and available. + TiDB is a distributed database. Its disaster tolerance means that when any physical node failed, not only to ensure TiDB server is available, but also ensure the data is complete and available. How to guarantee Disaster Tolerance of TiDB cluster on Kubernetes? From 972fa46035e04204a2aabe4c128f9d7d51a44877 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 16 May 2019 16:33:05 +0800 Subject: [PATCH 11/18] Update docs/operation-guide.md Co-Authored-By: weekface --- docs/operation-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operation-guide.md b/docs/operation-guide.md index 47d8090d92..607c775c31 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -39,7 +39,7 @@ TiDB Operator uses `values.yaml` as TiDB cluster configuration file. It provides We mainly solve the problem from the scheduling of services and data. - * Disaster Tolerance of TiDB server + * Disaster Tolerance of TiDB instance TiDB Operator provides a external scheduler to guarantee PD/TiKV/TiDB server disaster tolerance on host level. TiDB Cluster have set the external scheduler as default scheduler, you will find the setting in the variable `schedulerName` of `values.yaml`. From 29c8355f2336519bec32b3f435d05521644f3221 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 16 May 2019 16:33:23 +0800 Subject: [PATCH 12/18] Update docs/operation-guide.md Co-Authored-By: weekface --- docs/operation-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operation-guide.md b/docs/operation-guide.md index 607c775c31..63601654f9 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -41,7 +41,7 @@ TiDB Operator uses `values.yaml` as TiDB cluster configuration file. It provides * Disaster Tolerance of TiDB instance - TiDB Operator provides a external scheduler to guarantee PD/TiKV/TiDB server disaster tolerance on host level. TiDB Cluster have set the external scheduler as default scheduler, you will find the setting in the variable `schedulerName` of `values.yaml`. + TiDB Operator provides an extended scheduler to guarantee PD/TiKV/TiDB instance disaster tolerance on host level. TiDB Cluster has set the extended scheduler as default scheduler, you will find the setting in the variable `schedulerName` of `values.yaml`. In the other hand use `PodAntiAffinity` term of `affinity` to ensure disaster tolerance on the other topology levels (e.g. rack, zone, region). refer to the doc: [pod affnity & anti affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity-beta-feature), moreover `values.yaml` also provides a typical disaster tolerance setting example in the comments of `pd.affinity`. From 7c1e04dbb47da67b815bb6dcfb43189eef16bb7c Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 16 May 2019 16:33:38 +0800 Subject: [PATCH 13/18] Update docs/operation-guide.md Co-Authored-By: weekface --- docs/operation-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operation-guide.md b/docs/operation-guide.md index 63601654f9..dec414b878 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -49,7 +49,7 @@ TiDB Operator uses `values.yaml` as TiDB cluster configuration file. It provides * Disaster Tolerance of data Disaster tolerance of data is guaranteed by TiDB Cluster itself. The only work Operator needs to do is that collects topology info from specific labels of Kubernetes nodes where TiKV Pod runs on and then PD will schedule data replicas auto according to the topology info. - Cause currently TiDB Operator can only recognize some specific labels, so you can only set nodes topology info with the following particular labels + Because current TiDB Operator can only recognize some specific labels, so you can only set nodes topology info with the following particular labels * `region`: region where node is located * `zone`: zone where node is located From 63ce0d3b31d84a2a541be5451d03e019556cf142 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 16 May 2019 16:33:47 +0800 Subject: [PATCH 14/18] Update docs/operation-guide.md Co-Authored-By: weekface --- docs/operation-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operation-guide.md b/docs/operation-guide.md index dec414b878..ea21e4a87e 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -58,7 +58,7 @@ TiDB Operator uses `values.yaml` as TiDB cluster configuration file. It provides you need label topology info to nodes of Kubernetes cluster use the following command ```shell - # The labels are optional + # Not all tags are required $ kubectl label node region= zone= rack= kubernetes.io/hostname= ``` From 8c0e1c2ad915bf9b840dbe3227fa7045fd5cde37 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 16 May 2019 23:44:13 +0800 Subject: [PATCH 15/18] add nodeSelector --- .../tidb-cluster/templates/tidb-cluster.yaml | 6 ++++++ charts/tidb-cluster/values.yaml | 12 +++++++++++ pkg/apis/pingcap.com/v1alpha1/types.go | 3 +++ .../v1alpha1/zz_generated.deepcopy.go | 21 +++++++++++++++++++ pkg/manager/member/pd_member_manager.go | 1 + pkg/manager/member/tidb_member_manager.go | 1 + pkg/manager/member/tikv_member_manager.go | 1 + 7 files changed, 45 insertions(+) diff --git a/charts/tidb-cluster/templates/tidb-cluster.yaml b/charts/tidb-cluster/templates/tidb-cluster.yaml index 2c41d7f2b9..be95fca5a9 100644 --- a/charts/tidb-cluster/templates/tidb-cluster.yaml +++ b/charts/tidb-cluster/templates/tidb-cluster.yaml @@ -35,6 +35,8 @@ spec: {{- end }} affinity: {{ toYaml .Values.pd.affinity | indent 6 }} + nodeSelector: +{{ toYaml .Values.pd.nodeSelector | indent 6 }} {{- if .Values.pd.tolerations }} tolerations: {{ toYaml .Values.pd.tolerations | indent 4 }} @@ -51,6 +53,8 @@ spec: {{- end }} affinity: {{ toYaml .Values.tikv.affinity | indent 6 }} + nodeSelector: +{{ toYaml .Values.tikv.nodeSelector | indent 6 }} {{- if .Values.tikv.tolerations }} tolerations: {{ toYaml .Values.tikv.tolerations | indent 4 }} @@ -64,6 +68,8 @@ spec: {{- end }} affinity: {{ toYaml .Values.tidb.affinity | indent 6 }} + nodeSelector: +{{ toYaml .Values.tidb.nodeSelector | indent 6 }} {{- if .Values.tidb.tolerations }} tolerations: {{ toYaml .Values.tidb.tolerations | indent 4 }} diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index a3b16334d1..848a4f6441 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -133,6 +133,10 @@ pd: # values: # - "pd" + ## nodeSelector ensure pods only assigning to nodes which have each of the indicated key-value pairs as labels + ## ref:https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + nodeSelector: {} + ## Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints. ## refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration tolerations: [] @@ -174,6 +178,10 @@ tikv: ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} + ## nodeSelector ensure pods only assigning to nodes which have each of the indicated key-value pairs as labels + ## ref:https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + nodeSelector: {} + ## Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints. ## refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration tolerations: [] @@ -257,6 +265,10 @@ tidb: ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} + ## nodeSelector ensure pods only assigning to nodes which have each of the indicated key-value pairs as labels + ## ref:https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + nodeSelector: {} + ## Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints. ## refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration tolerations: [] diff --git a/pkg/apis/pingcap.com/v1alpha1/types.go b/pkg/apis/pingcap.com/v1alpha1/types.go index 37fcdc2ff6..a4de9aec93 100644 --- a/pkg/apis/pingcap.com/v1alpha1/types.go +++ b/pkg/apis/pingcap.com/v1alpha1/types.go @@ -108,6 +108,7 @@ type PDSpec struct { ContainerSpec Replicas int32 `json:"replicas"` Affinity *corev1.Affinity `json:"affinity,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` StorageClassName string `json:"storageClassName,omitempty"` Tolerations []corev1.Toleration `json:"tolerations,omitempty"` } @@ -117,6 +118,7 @@ type TiDBSpec struct { ContainerSpec Replicas int32 `json:"replicas"` Affinity *corev1.Affinity `json:"affinity,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` StorageClassName string `json:"storageClassName,omitempty"` Tolerations []corev1.Toleration `json:"tolerations,omitempty"` BinlogEnabled bool `json:"binlogEnabled,omitempty"` @@ -135,6 +137,7 @@ type TiKVSpec struct { ContainerSpec Replicas int32 `json:"replicas"` Affinity *corev1.Affinity `json:"affinity,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` StorageClassName string `json:"storageClassName,omitempty"` Tolerations []corev1.Toleration `json:"tolerations,omitempty"` } diff --git a/pkg/apis/pingcap.com/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap.com/v1alpha1/zz_generated.deepcopy.go index 4b828246dc..e32c9e13ee 100644 --- a/pkg/apis/pingcap.com/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap.com/v1alpha1/zz_generated.deepcopy.go @@ -94,6 +94,13 @@ func (in *PDSpec) DeepCopyInto(out *PDSpec) { *out = new(v1.Affinity) (*in).DeepCopyInto(*out) } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]v1.Toleration, len(*in)) @@ -241,6 +248,13 @@ func (in *TiDBSpec) DeepCopyInto(out *TiDBSpec) { *out = new(v1.Affinity) (*in).DeepCopyInto(*out) } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]v1.Toleration, len(*in)) @@ -339,6 +353,13 @@ func (in *TiKVSpec) DeepCopyInto(out *TiKVSpec) { *out = new(v1.Affinity) (*in).DeepCopyInto(*out) } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]v1.Toleration, len(*in)) diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index a1a09c689d..8f08fd528c 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -477,6 +477,7 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) Spec: corev1.PodSpec{ SchedulerName: tc.Spec.SchedulerName, Affinity: tc.Spec.PD.Affinity, + NodeSelector: tc.Spec.PD.NodeSelector, Containers: []corev1.Container{ { Name: v1alpha1.PDMemberType.String(), diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index d4c5bde818..8cf7227ee2 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -347,6 +347,7 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust Spec: corev1.PodSpec{ SchedulerName: tc.Spec.SchedulerName, Affinity: tc.Spec.TiDB.Affinity, + NodeSelector: tc.Spec.TiDB.NodeSelector, Containers: containers, RestartPolicy: corev1.RestartPolicyAlways, Tolerations: tc.Spec.TiDB.Tolerations, diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 5cdbe7741f..dd2ff9b7ab 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -336,6 +336,7 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) Spec: corev1.PodSpec{ SchedulerName: tc.Spec.SchedulerName, Affinity: tc.Spec.TiKV.Affinity, + NodeSelector: tc.Spec.TiKV.NodeSelector, Containers: []corev1.Container{ { Name: v1alpha1.TiKVMemberType.String(), From 55cd66b5ee85a25b611097ef0501aec7b051796d Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Mon, 20 May 2019 10:49:44 +0800 Subject: [PATCH 16/18] Update docs/operation-guide.md Co-Authored-By: Tennix --- docs/operation-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operation-guide.md b/docs/operation-guide.md index 00a704bf1c..a8232daa4d 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -13,7 +13,7 @@ $ namespace="tidb" ## Configuration -TiDB Operator uses `values.yaml` as TiDB cluster configuration file. It provides the default basic configuration which you can use directly for quick deployment, but if you have specific configuration requirements or for production deployment, you need to manually modify the variables in the `value.yaml` file. +TiDB Operator uses `values.yaml` as TiDB cluster configuration file. It provides the default basic configuration which you can use directly for quick deployment, but if you have specific configuration requirements or for production deployment, you need to manually modify the variables in the `values.yaml` file. * Resource setting From 2682ffa4f69c5b898a34655b1a15985f342f5677 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Mon, 20 May 2019 11:56:05 +0800 Subject: [PATCH 17/18] fix doc error --- docs/operation-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operation-guide.md b/docs/operation-guide.md index a8232daa4d..c7a2d87198 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -19,7 +19,7 @@ TiDB Operator uses `values.yaml` as TiDB cluster configuration file. It provides * CPU & Memory - The default deployment doesn't set CPU and memory requests or limits for any of the pods, these settings can make TiDB cluster run on a small Kubernetes cluster like DinD or the default GKE cluster for testing. But for production deployment, you would likely to adjust the cpu, memory and storage resources according to the [recommendations](https://github.com/pingcap/docs/blob/master/op-guide/recommendation.md). + The default deployment doesn't set CPU and memory requests or limits for any of the pods, these settings can make TiDB cluster run on a small Kubernetes cluster like DinD or the default GKE cluster for testing. But for production deployment, you would likely to adjust the cpu, memory and storage resources according to the [recommendations](https://pingcap.com/docs/dev/how-to/deploy/hardware-recommendations/#software-and-hardware-recommendations). The resource limits should be equal or bigger than the resource requests, it is suggested to set limit and request equal to get [`Guaranteed` QoS]( https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/#create-a-pod-that-gets-assigned-a-qos-class-of-guaranteed). * Storage From 29327c1084e24507bbe4231d3d06fda701aaee0a Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Wed, 22 May 2019 16:39:15 +0800 Subject: [PATCH 18/18] add blank line --- docs/operation-guide.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/operation-guide.md b/docs/operation-guide.md index dc418efdb9..6a46c81211 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -20,11 +20,13 @@ TiDB Operator uses `values.yaml` as TiDB cluster configuration file. It provides * CPU & Memory The default deployment doesn't set CPU and memory requests or limits for any of the pods, these settings can make TiDB cluster run on a small Kubernetes cluster like DinD or the default GKE cluster for testing. But for production deployment, you would likely to adjust the cpu, memory and storage resources according to the [recommendations](https://pingcap.com/docs/dev/how-to/deploy/hardware-recommendations/#software-and-hardware-recommendations). + The resource limits should be equal or bigger than the resource requests, it is suggested to set limit and request equal to get [`Guaranteed` QoS]( https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/#create-a-pod-that-gets-assigned-a-qos-class-of-guaranteed). * Storage The variables `pd.storageClassName` and `tikv.storageClassName` in `values.yaml` are used to set `StorageClass` of PD and TiKV,their default setting are `local-storage` with minimal size. + If you don't want to use the default `StorageClass` or your Kubernetes cluster does not support `local-storage` class, please execute the following command to find an available `StorageClass` and select the ones you want to provide to TiDB cluster. ```shell