From 74cf0402753f0fd7953a23b263087fd02ce9d60d Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Wed, 31 Jul 2019 12:11:37 -0700 Subject: [PATCH 1/3] scheduler: inject a failure by pod label This can be useful for testing scheduling failures. You can for example add it to monitor-deployment along with changing the scheduler to the custom scheduler. --- pkg/label/label.go | 3 +++ pkg/scheduler/scheduler.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/pkg/label/label.go b/pkg/label/label.go index 1c1099437c..04d1be2904 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -65,6 +65,9 @@ const ( TiDBLabelVal string = "tidb" // TiKVLabelVal is TiKV label value TiKVLabelVal string = "tikv" + // FailTiDBSchedulerLabelKey is for injecting a failure into the TiDB custom scheduler + // A pod with this label will produce an error when scheduled. + FailTiDBSchedulerLabelKey string = "tidb.pingcap.com/fail-scheduler" ) // Label is the label field in metadata diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 0ecacbe170..0c093ef5db 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -14,6 +14,7 @@ package scheduler import ( + "fmt" "github.com/golang/glog" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/features" @@ -79,6 +80,10 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E podName := pod.GetName() kubeNodes := args.Nodes.Items + if _, ok := pod.Labels[label.FailTiDBSchedulerLabelKey]; ok { + return nil, FailureError{PodName: pod.Name} + } + var instanceName string var exist bool if instanceName, exist = pod.Labels[label.InstanceLabelKey]; !exist { @@ -118,6 +123,15 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E }, nil } +// FailureError is returne when the FailTiDBSchedulerLabelKey is seen +type FailureError struct { + PodName string +} + +func (ferr FailureError) Error() string { + return fmt.Sprintf("pod %s had an intentional failure injected", ferr.PodName) +} + // We don't pass `prioritizeVerb` to kubernetes scheduler extender's config file, this method will not be called. func (s *scheduler) Priority(args *schedulerapiv1.ExtenderArgs) (schedulerapiv1.HostPriorityList, error) { result := schedulerapiv1.HostPriorityList{} From 1a004124d3dc05f9bf9236bf0a8afb7d8097c94b Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Fri, 2 Aug 2019 06:01:36 -0500 Subject: [PATCH 2/3] Fix typo Co-Authored-By: Tennix --- pkg/scheduler/scheduler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 0c093ef5db..a5b5c74b03 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -123,7 +123,7 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E }, nil } -// FailureError is returne when the FailTiDBSchedulerLabelKey is seen +// FailureError is returned when the FailTiDBSchedulerLabelKey is seen type FailureError struct { PodName string } From cd6ee7559f67a9285c3f2ea50434149748c1b35f Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Fri, 2 Aug 2019 04:19:02 -0700 Subject: [PATCH 3/3] switch from a label to an annotation --- pkg/label/label.go | 6 +++--- pkg/scheduler/scheduler.go | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/label/label.go b/pkg/label/label.go index 04d1be2904..38136bbb7f 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -45,6 +45,9 @@ const ( StoreIDLabelKey string = "tidb.pingcap.com/store-id" // MemberIDLabelKey is member id label key MemberIDLabelKey string = "tidb.pingcap.com/member-id" + // AnnFailTiDBScheduler is for injecting a failure into the TiDB custom scheduler + // A pod with this annotation will produce an error when scheduled. + AnnFailTiDBScheduler string = "tidb.pingcap.com/fail-scheduler" // AnnPodNameKey is pod name annotation key used in PV/PVC for synchronizing tidb cluster meta info AnnPodNameKey string = "tidb.pingcap.com/pod-name" // AnnPVCDeferDeleting is pvc defer deletion annotation key used in PVC for defer deleting PVC @@ -65,9 +68,6 @@ const ( TiDBLabelVal string = "tidb" // TiKVLabelVal is TiKV label value TiKVLabelVal string = "tikv" - // FailTiDBSchedulerLabelKey is for injecting a failure into the TiDB custom scheduler - // A pod with this label will produce an error when scheduled. - FailTiDBSchedulerLabelKey string = "tidb.pingcap.com/fail-scheduler" ) // Label is the label field in metadata diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 0c093ef5db..187f49c911 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -80,8 +80,10 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E podName := pod.GetName() kubeNodes := args.Nodes.Items - if _, ok := pod.Labels[label.FailTiDBSchedulerLabelKey]; ok { - return nil, FailureError{PodName: pod.Name} + if pod.Annotations != nil { + if _, ok := pod.Annotations[label.AnnFailTiDBScheduler]; ok { + return nil, FailureError{PodName: pod.Name} + } } var instanceName string