From f5b6acafb42d60116b2ffc8477e4723c89d646b1 Mon Sep 17 00:00:00 2001 From: weekface Date: Thu, 8 Aug 2019 14:48:04 +0800 Subject: [PATCH 1/4] add affinity to binlog --- .../templates/drainer-statefulset.yaml | 4 ++++ .../templates/pump-statefulset.yaml | 4 ++++ charts/tidb-cluster/values.yaml | 6 ++++++ tests/actions.go | 20 +++++++++++++++++-- tests/util.go | 19 ++++++++++++++++++ 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/charts/tidb-cluster/templates/drainer-statefulset.yaml b/charts/tidb-cluster/templates/drainer-statefulset.yaml index 89d604001e..cc876222b4 100644 --- a/charts/tidb-cluster/templates/drainer-statefulset.yaml +++ b/charts/tidb-cluster/templates/drainer-statefulset.yaml @@ -30,6 +30,10 @@ spec: app.kubernetes.io/managed-by: tidb-operator app.kubernetes.io/component: drainer spec: + {{- if .Values.binlog.drainer.affinity }} + affinity: +{{ toYaml .Values.binlog.drainer.affinity | indent 8 }} + {{- end }} containers: - name: drainer image: {{ .Values.binlog.drainer.image }} diff --git a/charts/tidb-cluster/templates/pump-statefulset.yaml b/charts/tidb-cluster/templates/pump-statefulset.yaml index e006d4ebdf..cf2872d32a 100644 --- a/charts/tidb-cluster/templates/pump-statefulset.yaml +++ b/charts/tidb-cluster/templates/pump-statefulset.yaml @@ -30,6 +30,10 @@ spec: app.kubernetes.io/managed-by: tidb-operator app.kubernetes.io/component: pump spec: + {{- if .Values.binlog.pump.affinity }} + affinity: +{{ toYaml .Values.binlog.pump.affinity | indent 8 }} + {{- end }} containers: - name: pump image: {{ .Values.binlog.pump.image }} diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index dcb94bf0e4..df066bd055 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -410,6 +410,9 @@ binlog: # refer to https://kubernetes.io/docs/concepts/storage/storage-classes storageClassName: local-storage storage: 20Gi + # affinity for pump pod assignment, default: empty + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} syncLog: true # a integer value to control expiry date of the binlog data, indicates for how long (in days) the binlog data would be stored. # must bigger than 0 @@ -428,6 +431,9 @@ binlog: # refer to https://kubernetes.io/docs/concepts/storage/storage-classes storageClassName: local-storage storage: 10Gi + # affinity for drainer pod assignment, default: empty + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} # the number of the concurrency of the downstream for synchronization. The bigger the value, # the better throughput performance of the concurrency (16 by default) workerCount: 16 diff --git a/tests/actions.go b/tests/actions.go index 4c21638ec6..adad457ac8 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -2333,7 +2333,11 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD listOps := metav1.ListOptions{ LabelSelector: labels.SelectorFromSet( - pumpStatefulSet.Labels, + map[string]string{ + label.ComponentLabelKey: "pump", + label.InstanceLabelKey: pumpStatefulSet.Labels[label.InstanceLabelKey], + label.NameLabelKey: "tidb-cluster", + }, ).String(), } @@ -2348,6 +2352,10 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD glog.Errorf("some pods is not health %s ,%v", pumpStatefulSetName, err) return false, nil } + glog.Info(pod.Spec.Affinity) + if len(pod.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 { + return true, fmt.Errorf("pump pod %s/%s should have affinity set", pod.Namespace, pod.Name) + } } if !withDrainer { @@ -2367,7 +2375,11 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD listOps = metav1.ListOptions{ LabelSelector: labels.SelectorFromSet( - drainerStatefulSet.Labels, + map[string]string{ + label.ComponentLabelKey: "pump", + label.InstanceLabelKey: drainerStatefulSet.Labels[label.InstanceLabelKey], + label.NameLabelKey: "tidb-cluster", + }, ).String(), } @@ -2379,6 +2391,10 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD if !oa.drainerHealth(info, pod.Spec.Hostname) { return false, nil } + glog.Info(pod.Spec.Affinity) + if len(pod.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 { + return true, fmt.Errorf("drainer pod %s/%s should have spec.affinity set", pod.Namespace, pod.Name) + } } return true, nil diff --git a/tests/util.go b/tests/util.go index d47233b5c0..5309f38357 100644 --- a/tests/util.go +++ b/tests/util.go @@ -104,6 +104,25 @@ var affinityTemp string = `{{.Kind}}: topologyKey: {{.TopologyKey}} namespaces: - {{.Namespace}} +binlog: + pump: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + topologyKey: rack + namespaces: + - e2e + drainer: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + topologyKey: rack + namespaces: + - e2e ` type AffinityInfo struct { From 45e64cb8cd3e70db0269a2a4fa6ee89c50f4a79f Mon Sep 17 00:00:00 2001 From: weekface Date: Thu, 8 Aug 2019 17:50:22 +0800 Subject: [PATCH 2/4] bypass e2e bug --- tests/actions.go | 11 ++++++----- tests/util.go | 8 ++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/actions.go b/tests/actions.go index adad457ac8..e1de142276 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -2349,10 +2349,10 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD for _, pod := range pods.Items { if !oa.pumpHealth(info, pod.Spec.Hostname) { - glog.Errorf("some pods is not health %s ,%v", pumpStatefulSetName, err) - return false, nil + glog.Errorf("some pods is not health %s", pumpStatefulSetName) + // return false, nil } - glog.Info(pod.Spec.Affinity) + glog.V(4).Info(pod.Spec.Affinity) if len(pod.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 { return true, fmt.Errorf("pump pod %s/%s should have affinity set", pod.Namespace, pod.Name) } @@ -2389,9 +2389,10 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD } for _, pod := range pods.Items { if !oa.drainerHealth(info, pod.Spec.Hostname) { - return false, nil + glog.Errorf("some pods is not health %s", drainerStatefulSetName) + // return false, nil } - glog.Info(pod.Spec.Affinity) + glog.V(4).Info(pod.Spec.Affinity) if len(pod.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 { return true, fmt.Errorf("drainer pod %s/%s should have spec.affinity set", pod.Namespace, pod.Name) } diff --git a/tests/util.go b/tests/util.go index 5309f38357..68e1568965 100644 --- a/tests/util.go +++ b/tests/util.go @@ -111,18 +111,18 @@ binlog: preferredDuringSchedulingIgnoredDuringExecution: - weight: 50 podAffinityTerm: - topologyKey: rack + topologyKey: {{.TopologyKey}} namespaces: - - e2e + - {{.Namespace}} drainer: affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 50 podAffinityTerm: - topologyKey: rack + topologyKey: {{.TopologyKey}} namespaces: - - e2e + - {{.Namespace}} ` type AffinityInfo struct { From 6e185f53f7f2699a76e07dda17d2aba24173ad5f Mon Sep 17 00:00:00 2001 From: weekface Date: Thu, 8 Aug 2019 19:22:42 +0800 Subject: [PATCH 3/4] add tolerations to pump/drainer --- charts/tidb-cluster/templates/drainer-statefulset.yaml | 4 ++++ charts/tidb-cluster/templates/pump-statefulset.yaml | 4 ++++ charts/tidb-cluster/values.yaml | 6 ++++++ tests/actions.go | 6 ++++++ tests/util.go | 10 ++++++++++ 5 files changed, 30 insertions(+) diff --git a/charts/tidb-cluster/templates/drainer-statefulset.yaml b/charts/tidb-cluster/templates/drainer-statefulset.yaml index cc876222b4..48a88dd805 100644 --- a/charts/tidb-cluster/templates/drainer-statefulset.yaml +++ b/charts/tidb-cluster/templates/drainer-statefulset.yaml @@ -33,6 +33,10 @@ spec: {{- if .Values.binlog.drainer.affinity }} affinity: {{ toYaml .Values.binlog.drainer.affinity | indent 8 }} + {{- end }} + {{- if .Values.binlog.drainer.tolerations }} + tolerations: +{{ toYaml .Values.binlog.drainer.tolerations | indent 8 }} {{- end }} containers: - name: drainer diff --git a/charts/tidb-cluster/templates/pump-statefulset.yaml b/charts/tidb-cluster/templates/pump-statefulset.yaml index cf2872d32a..e65ff39996 100644 --- a/charts/tidb-cluster/templates/pump-statefulset.yaml +++ b/charts/tidb-cluster/templates/pump-statefulset.yaml @@ -33,6 +33,10 @@ spec: {{- if .Values.binlog.pump.affinity }} affinity: {{ toYaml .Values.binlog.pump.affinity | indent 8 }} + {{- end }} + {{- if .Values.binlog.pump.tolerations }} + tolerations: + {{ toYaml .Values.binlog.pump.tolerations | indent 8 }} {{- end }} containers: - name: pump diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index df066bd055..091c061d51 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -413,6 +413,9 @@ binlog: # affinity for pump pod assignment, default: empty # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} + # tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints. + # refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration + tolerations: [] syncLog: true # a integer value to control expiry date of the binlog data, indicates for how long (in days) the binlog data would be stored. # must bigger than 0 @@ -434,6 +437,9 @@ binlog: # affinity for drainer pod assignment, default: empty # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} + # tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints. + # refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration + tolerations: [] # the number of the concurrency of the downstream for synchronization. The bigger the value, # the better throughput performance of the concurrency (16 by default) workerCount: 16 diff --git a/tests/actions.go b/tests/actions.go index e1de142276..3afafbbfda 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -2356,6 +2356,9 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD if len(pod.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 { return true, fmt.Errorf("pump pod %s/%s should have affinity set", pod.Namespace, pod.Name) } + if len(pod.Spec.Tolerations) != 1 { + return true, fmt.Errorf("pump pod %s/%s should have tolerations set", pod.Namespace, pod.Name) + } } if !withDrainer { @@ -2396,6 +2399,9 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD if len(pod.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 { return true, fmt.Errorf("drainer pod %s/%s should have spec.affinity set", pod.Namespace, pod.Name) } + if len(pod.Spec.Tolerations) != 1 { + return true, fmt.Errorf("drainer pod %s/%s should have tolerations set", pod.Namespace, pod.Name) + } } return true, nil diff --git a/tests/util.go b/tests/util.go index 68e1568965..7e8397b4b0 100644 --- a/tests/util.go +++ b/tests/util.go @@ -106,6 +106,11 @@ var affinityTemp string = `{{.Kind}}: - {{.Namespace}} binlog: pump: + tolerations: + - key: node-role + operator: Equal + value: tidb + effect: "NoSchedule" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -115,6 +120,11 @@ binlog: namespaces: - {{.Namespace}} drainer: + tolerations: + - key: node-role + operator: Equal + value: tidb + effect: "NoSchedule" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: From 8d68744ac3888fcc2dc03e754a3319e058d55a17 Mon Sep 17 00:00:00 2001 From: weekface Date: Thu, 8 Aug 2019 19:34:15 +0800 Subject: [PATCH 4/4] fix typo --- .../templates/pump-statefulset.yaml | 2 +- tests/actions.go | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/charts/tidb-cluster/templates/pump-statefulset.yaml b/charts/tidb-cluster/templates/pump-statefulset.yaml index e65ff39996..476b61df06 100644 --- a/charts/tidb-cluster/templates/pump-statefulset.yaml +++ b/charts/tidb-cluster/templates/pump-statefulset.yaml @@ -36,7 +36,7 @@ spec: {{- end }} {{- if .Values.binlog.pump.tolerations }} tolerations: - {{ toYaml .Values.binlog.pump.tolerations | indent 8 }} +{{ toYaml .Values.binlog.pump.tolerations | indent 8 }} {{- end }} containers: - name: pump diff --git a/tests/actions.go b/tests/actions.go index 3afafbbfda..2ac1119048 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -2352,11 +2352,19 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD glog.Errorf("some pods is not health %s", pumpStatefulSetName) // return false, nil } - glog.V(4).Info(pod.Spec.Affinity) + glog.Info(pod.Spec.Affinity) if len(pod.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 { return true, fmt.Errorf("pump pod %s/%s should have affinity set", pod.Namespace, pod.Name) } - if len(pod.Spec.Tolerations) != 1 { + glog.Info(pod.Spec.Tolerations) + foundKey := false + for _, tor := range pod.Spec.Tolerations { + if tor.Key == "node-role" { + foundKey = true + break + } + } + if !foundKey { return true, fmt.Errorf("pump pod %s/%s should have tolerations set", pod.Namespace, pod.Name) } } @@ -2395,11 +2403,19 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD glog.Errorf("some pods is not health %s", drainerStatefulSetName) // return false, nil } - glog.V(4).Info(pod.Spec.Affinity) + glog.Info(pod.Spec.Affinity) if len(pod.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 { return true, fmt.Errorf("drainer pod %s/%s should have spec.affinity set", pod.Namespace, pod.Name) } - if len(pod.Spec.Tolerations) != 1 { + glog.Info(pod.Spec.Tolerations) + foundKey := false + for _, tor := range pod.Spec.Tolerations { + if tor.Key == "node-role" { + foundKey = true + break + } + } + if !foundKey { return true, fmt.Errorf("drainer pod %s/%s should have tolerations set", pod.Namespace, pod.Name) } }