Add stop kube-scheduler/kube-controller-manager test cases

tests/actions.go

@@ -156,7 +156,9 @@ type OperatorActions interface {
 	CheckTidbClustersAvailableOrDie(infos []*TidbClusterConfig)
 	CheckOneEtcdDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig, faultNode string)
 	CheckOneApiserverDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig, faultNode string)
-	CheckKubeProxyDownOrDie(clusters []*TidbClusterConfig)
+	CheckKubeProxyDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig)
+	CheckKubeSchedulerDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig)
+	CheckKubeControllerManagerDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig)
 	RegisterWebHookAndService(context *apimachinery.CertContext, info *OperatorConfig) error
 	RegisterWebHookAndServiceOrDie(context *apimachinery.CertContext, info *OperatorConfig)
 	CleanWebHookAndService(info *OperatorConfig) error

tests/cmd/stability/main.go

@@ -339,10 +339,36 @@ func run() {
 
 	// stop all kube-proxy and k8s/operator/tidbcluster is available
 	fta.StopKubeProxyOrDie()
-	oa.CheckKubeProxyDownOrDie(allClusters)
+	oa.CheckKubeProxyDownOrDie(operatorCfg, allClusters)
 	fta.StartKubeProxyOrDie()
 
-	//clean temp dirs when stability success
+	// stop all kube-scheduler pods
+	for _, physicalNode := range cfg.APIServers {
+		for _, vNode := range physicalNode.Nodes {
+			fta.StopKubeSchedulerOrDie(vNode)
+		}
+	}
+	oa.CheckKubeSchedulerDownOrDie(operatorCfg, allClusters)
+	for _, physicalNode := range cfg.APIServers {
+		for _, vNode := range physicalNode.Nodes {
+			fta.StartKubeSchedulerOrDie(vNode)
+		}
+	}
+
+	// stop all kube-controller-manager pods
+	for _, physicalNode := range cfg.APIServers {
+		for _, vNode := range physicalNode.Nodes {
+			fta.StopKubeControllerManagerOrDie(vNode)
+		}
+	}
+	oa.CheckKubeControllerManagerDownOrDie(operatorCfg, allClusters)
+	for _, physicalNode := range cfg.APIServers {
+		for _, vNode := range physicalNode.Nodes {
+			fta.StartKubeControllerManagerOrDie(vNode)
+		}
+	}
+
+	// clean temp dirs when stability success
 	err := cfg.CleanTempDirs()
 	if err != nil {
 		glog.Errorf("failed to clean temp dirs, this error can be ignored.")

tests/failover.go

@@ -457,7 +457,7 @@ func (oa *operatorActions) CheckOneEtcdDownOrDie(operatorConfig *OperatorConfig,
 	})
 }
 
-func (oa *operatorActions) CheckKubeProxyDownOrDie(clusters []*TidbClusterConfig) {
+func (oa *operatorActions) CheckKubeProxyDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig) {
 	glog.Infof("checking k8s/tidbCluster status when kube-proxy down")
 
 	KeepOrDie(3*time.Second, 10*time.Minute, func() error {
@@ -467,6 +467,63 @@ func (oa *operatorActions) CheckKubeProxyDownOrDie(clusters []*TidbClusterConfig
 
 		}
 		glog.V(4).Infof("k8s cluster is available.")
+
+		err = oa.CheckOperatorAvailable(operatorConfig)
+		if err != nil {
+			return err
+		}
+		glog.V(4).Infof("tidb operator is available.")
+
+		err = oa.CheckTidbClustersAvailable(clusters)
+		if err != nil {
+			return err
+		}
+		glog.V(4).Infof("all clusters are available.")
+		return nil
+	})
+}
+
+func (oa *operatorActions) CheckKubeSchedulerDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig) {
+	glog.Infof("verify kube-scheduler is not avaiavble")
+
+	if err := waitForComponentStatus(oa.kubeCli, "scheduler", corev1.ComponentHealthy, corev1.ConditionFalse); err != nil {
+		slack.NotifyAndPanic(fmt.Errorf("failed to stop kube-scheduler: %v", err))
+	}
+
+	glog.Infof("checking operator/tidbCluster status when kube-scheduler is not available")
+
+	KeepOrDie(3*time.Second, 10*time.Minute, func() error {
+		err := oa.CheckOperatorAvailable(operatorConfig)
+		if err != nil {
+			return err
+		}
+		glog.V(4).Infof("tidb operator is available.")
+
+		err = oa.CheckTidbClustersAvailable(clusters)
+		if err != nil {
+			return err
+		}
+		glog.V(4).Infof("all clusters are available.")
+		return nil
+	})
+}
+
+func (oa *operatorActions) CheckKubeControllerManagerDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig) {
+	glog.Infof("verify kube-controller-manager is not avaiavble")
+
+	if err := waitForComponentStatus(oa.kubeCli, "controller-manager", corev1.ComponentHealthy, corev1.ConditionFalse); err != nil {
+		slack.NotifyAndPanic(fmt.Errorf("failed to stop kube-controller-manager: %v", err))
+	}
+
+	glog.Infof("checking operator/tidbCluster status when kube-controller-manager is not available")
+
+	KeepOrDie(3*time.Second, 10*time.Minute, func() error {
+		err := oa.CheckOperatorAvailable(operatorConfig)
+		if err != nil {
+			return err
+		}
+		glog.V(4).Infof("tidb operator is available.")
+
 		err = oa.CheckTidbClustersAvailable(clusters)
 		if err != nil {
 			return err

tests/fault.go

@@ -44,9 +44,13 @@ type FaultTriggerActions interface {
 	StartKubeAPIServer(node string) error
 	StartKubeAPIServerOrDie(node string)
 	StopKubeControllerManager(node string) error
+	StopKubeControllerManagerOrDie(node string)
 	StartKubeControllerManager(node string) error
-	StopKubeScheduler(node string) error
+	StartKubeControllerManagerOrDie(node string)
 	StartKubeScheduler(node string) error
+	StartKubeSchedulerOrDie(node string)
+	StopKubeScheduler(node string) error
+	StopKubeSchedulerOrDie(node string)
 	StopKubeProxy() error
 	StopKubeProxyOrDie()
 	StartKubeProxy() error
@@ -373,21 +377,49 @@ func (fa *faultTriggerActions) StopKubeScheduler(node string) error {
 	return fa.serviceAction(node, manager.KubeSchedulerService, stopAction)
 }
 
+// StopKubeScheduler stops the kube-scheduler service or dies.
+func (fa *faultTriggerActions) StopKubeSchedulerOrDie(node string) {
+	if err := fa.serviceAction(node, manager.KubeSchedulerService, stopAction); err != nil {
+		slack.NotifyAndPanic(err)
+	}
+}
+
 // StartKubeScheduler starts the kube-scheduler service.
 func (fa *faultTriggerActions) StartKubeScheduler(node string) error {
 	return fa.serviceAction(node, manager.KubeSchedulerService, startAction)
 }
 
+// StartKubeScheduler starts the kube-scheduler service or dies
+func (fa *faultTriggerActions) StartKubeSchedulerOrDie(node string) {
+	if err := fa.serviceAction(node, manager.KubeSchedulerService, startAction); err != nil {
+		slack.NotifyAndPanic(err)
+	}
+}
+
 // StopKubeControllerManager stops the kube-controller-manager service.
 func (fa *faultTriggerActions) StopKubeControllerManager(node string) error {
 	return fa.serviceAction(node, manager.KubeControllerManagerService, stopAction)
 }
 
+// StopKubeControllerManager stops the kube-controller-manager service or dies
+func (fa *faultTriggerActions) StopKubeControllerManagerOrDie(node string) {
+	if err := fa.serviceAction(node, manager.KubeControllerManagerService, stopAction); err != nil {
+		slack.NotifyAndPanic(err)
+	}
+}
+
 // StartKubeControllerManager starts the kube-controller-manager service.
 func (fa *faultTriggerActions) StartKubeControllerManager(node string) error {
 	return fa.serviceAction(node, manager.KubeControllerManagerService, startAction)
 }
 
+// StartKubeControllerManager starts the kube-controller-manager service or dies.
+func (fa *faultTriggerActions) StartKubeControllerManagerOrDie(node string) {
+	if err := fa.serviceAction(node, manager.KubeControllerManagerService, startAction); err != nil {
+		slack.NotifyAndPanic(err)
+	}
+}
+
 // StopKubeAPIServer stops the apiserver service.
 func (fa *faultTriggerActions) StopKubeAPIServer(node string) error {
 	return fa.serviceAction(node, manager.KubeAPIServerService, stopAction)

tests/pkg/fault-trigger/client/client.go

@@ -33,14 +33,14 @@ type Client interface {
 	StartKubeAPIServer() error
 	// StopKubeAPIServer stops the apiserver service
 	StopKubeAPIServer() error
+	// StartKubeScheduler starts the kube-scheduler service
 	StartKubeScheduler() error
 	// StopKubeScheduler stops the kube-scheduler service
 	StopKubeScheduler() error
 	// StartKubeControllerManager starts the kube-controller-manager service
 	StartKubeControllerManager() error
 	// StopKubeControllerManager stops the kube-controller-manager service
 	StopKubeControllerManager() error
-	// StartKubeScheduler starts the kube-scheduler service
 }
 
 // client is used to communicate with the fault-trigger

tests/util.go

@@ -153,3 +153,20 @@ func waitForPodNotFoundInNamespace(c kubernetes.Interface, podName, ns string, t
 		return false, nil
 	})
 }
+
+func waitForComponentStatus(c kubernetes.Interface, component string, statusType corev1.ComponentConditionType, status corev1.ConditionStatus) error {
+	return wait.PollImmediate(time.Second, 5*time.Minute, func() (bool, error) {
+		componentStatus, err := c.CoreV1().ComponentStatuses().Get(component, metav1.GetOptions{})
+		if err != nil {
+			return true, err // stop wait with error
+		}
+		found := false
+		for _, condition := range componentStatus.Conditions {
+			if condition.Type == statusType && condition.Status == status {
+				found = true
+				break
+			}
+		}
+		return found, nil
+	})
+}