From 50486234bc954f78e7951ed0e02f47d8a0486b9b Mon Sep 17 00:00:00 2001 From: zyguan Date: Tue, 12 Mar 2019 12:19:53 +0800 Subject: [PATCH 1/3] impl scale & upgrade cases --- tests/actions.go | 59 ++++++++++++++++++++++++++++++++++++++++--- tests/cluster_info.go | 50 ++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 tests/cluster_info.go diff --git a/tests/actions.go b/tests/actions.go index 6636337c22..15ddadbccf 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -22,11 +22,13 @@ import ( "time" "github.com/golang/glog" + "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/label" + "k8s.io/api/apps/v1beta1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -93,6 +95,8 @@ type operatorActions struct { pdControl controller.PDControlInterface } +var _ = OperatorActions(&operatorActions{}) + type OperatorInfo struct { Namespace string ReleaseName string @@ -112,7 +116,7 @@ type TidbClusterInfo struct { StorageClassName string Password string RecordCount string - InsertBetchSize string + InsertBatchSize string Resources map[string]string Args map[string]string } @@ -319,8 +323,32 @@ func (oa *operatorActions) StopInsertDataTo(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterInfo) error { return nil } +func chartPath(name string, tag string) string { + return "/charts/" + tag + "/" + name +} + +func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error { + cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", + info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString()) + glog.Info("[SCALE] " + cmd) + res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil { + return errors.Wrapf(err, "failed to scale tidb cluster: %s", string(res)) + } + return nil +} + +func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterInfo) error { + cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", + info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString()) + glog.Info("[UPGRADE] " + cmd) + res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil { + return errors.Wrapf(err, "failed to upgrade tidb cluster: %s", string(res)) + } + return nil +} + func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterInfo) error { return nil } func (oa *operatorActions) CleanAdHocBackup(info *TidbClusterInfo) error { return nil } func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterInfo) error { return nil } @@ -333,6 +361,16 @@ func (oa *operatorActions) Restore(from *TidbClusterInfo, jobName string, to *Ti func (oa *operatorActions) DeployMonitor(info *TidbClusterInfo) error { return nil } func (oa *operatorActions) CleanMonitor(info *TidbClusterInfo) error { return nil } +func getComponentContainer(set *v1beta1.StatefulSet) (corev1.Container, bool) { + name := set.Labels["app.kubernetes.io/component"] + for _, c := range set.Spec.Template.Spec.Containers { + if c.Name == name { + return c, true + } + } + return corev1.Container{}, false +} + func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, error) { tcName := tc.GetName() ns := tc.GetNamespace() @@ -370,6 +408,11 @@ func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, err ns, pdSetName, pdSet.Status.ReadyReplicas, pdSet.Status.Replicas) return false, nil } + if c, ok := getComponentContainer(pdSet); !ok || tc.Spec.PD.Image != c.Image { + glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=pd].image(%s) != %s", + ns, pdSetName, c.Image, tc.Spec.PD.Image) + return false, nil + } for _, member := range tc.Status.PD.Members { if !member.Health { @@ -430,6 +473,11 @@ func (oa *operatorActions) tikvMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e ns, tikvSetName, tikvSet.Status.ReadyReplicas, tikvSet.Status.Replicas) return false, nil } + if c, ok := getComponentContainer(tikvSet); !ok || tc.Spec.TiKV.Image != c.Image { + glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tikv].image(%s) != %s", + ns, tikvSetName, c.Image, tc.Spec.TiKV.Image) + return false, nil + } for _, store := range tc.Status.TiKV.Stores { if store.State != v1alpha1.TiKVStateUp { @@ -479,6 +527,11 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e ns, tidbSetName, tidbSet.Status.ReadyReplicas, tidbSet.Status.Replicas) return false, nil } + if c, ok := getComponentContainer(tidbSet); !ok || tc.Spec.TiDB.Image != c.Image { + glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tikv].image(%s) != %s", + ns, tidbSetName, c.Image, tc.Spec.TiDB.Image) + return false, nil + } _, err = oa.kubeCli.CoreV1().Services(ns).Get(tidbSetName, metav1.GetOptions{}) if err != nil { diff --git a/tests/cluster_info.go b/tests/cluster_info.go new file mode 100644 index 0000000000..332be46759 --- /dev/null +++ b/tests/cluster_info.go @@ -0,0 +1,50 @@ +package tests + +import "strconv" + +func (tc *TidbClusterInfo) set(name string, value string) (string, bool) { + // NOTE: not thread-safe, maybe make info struct immutable + if tc.Args == nil { + tc.Args = make(map[string]string) + } + origVal, ok := tc.Args[name] + tc.Args[name] = value + return origVal, ok +} + +func (tc *TidbClusterInfo) ScalePD(replicas uint) *TidbClusterInfo { + tc.set("pd.replicas", strconv.Itoa(int(replicas))) + return tc +} + +func (tc *TidbClusterInfo) ScaleTiKV(replicas uint) *TidbClusterInfo { + tc.set("tikv.replicas", strconv.Itoa(int(replicas))) + return tc +} + +func (tc *TidbClusterInfo) ScaleTiDB(replicas uint) *TidbClusterInfo { + tc.set("tidb.replicas", strconv.Itoa(int(replicas))) + return tc +} + +func (tc *TidbClusterInfo) UpgradePD(image string) *TidbClusterInfo { + tc.PDImage = image + return tc +} + +func (tc *TidbClusterInfo) UpgradeTiKV(image string) *TidbClusterInfo { + tc.TiKVImage = image + return tc +} + +func (tc *TidbClusterInfo) UpgradeTiDB(image string) *TidbClusterInfo { + tc.TiDBImage = image + return tc +} + +func (tc *TidbClusterInfo) UpgradeAll(tag string) *TidbClusterInfo { + return tc. + UpgradePD("pingcap/pd:" + tag). + UpgradeTiKV("pingcap/tikv:" + tag). + UpgradeTiDB("pingcap/tidb:" + tag) +} From f633bbaea9b555c968df1e9f96da35fefb644a6b Mon Sep 17 00:00:00 2001 From: zyguan Date: Tue, 12 Mar 2019 14:08:37 +0800 Subject: [PATCH 2/3] address the comment https://github.com/pingcap/tidb-operator/pull/309#discussion_r264525407 --- tests/actions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/actions.go b/tests/actions.go index 15ddadbccf..1c332c9ab5 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -362,7 +362,7 @@ func (oa *operatorActions) DeployMonitor(info *TidbClusterInfo) error { return n func (oa *operatorActions) CleanMonitor(info *TidbClusterInfo) error { return nil } func getComponentContainer(set *v1beta1.StatefulSet) (corev1.Container, bool) { - name := set.Labels["app.kubernetes.io/component"] + name := set.Labels[label.ComponentLabelKey] for _, c := range set.Spec.Template.Spec.Containers { if c.Name == name { return c, true From 2bffb300d80ec75549a1096fd4434af698831b10 Mon Sep 17 00:00:00 2001 From: zyguan Date: Thu, 14 Mar 2019 19:37:14 +0800 Subject: [PATCH 3/3] add cases to main for example --- tests/actions.go | 4 ++- tests/cmd/e2e/main.go | 80 +++++++++++++++++++++++++++++-------------- 2 files changed, 57 insertions(+), 27 deletions(-) diff --git a/tests/actions.go b/tests/actions.go index 8da73133c7..deca0e8fed 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -339,6 +339,7 @@ func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterInfo) error { return true, nil }); err != nil { + glog.Infof("check tidb cluster status failed: %s", err.Error()) return fmt.Errorf("failed to waiting for tidbcluster %s/%s ready in 10 minutes", ns, tcName) } @@ -621,8 +622,9 @@ outerLoop: for _, pod := range podList.Items { podName := pod.GetName() if pod.Labels[label.ClusterIDLabelKey] != clusterID { - return false, fmt.Errorf("tidbcluster %s/%s's pod %s's label %s not equals %s ", + glog.Infof("tidbcluster %s/%s's pod %s's label %s not equals %s ", ns, tcName, podName, label.ClusterIDLabelKey, clusterID) + return false, nil } component := pod.Labels[label.ComponentLabelKey] diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index d88878ad59..5a0939fa3a 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -24,6 +24,12 @@ import ( "k8s.io/client-go/rest" ) +func perror(err error) { + if err != nil { + glog.Fatal(err) + } +} + func main() { flag.Parse() logs.InitLogs() @@ -52,12 +58,8 @@ func main() { SchedulerImage: "gcr.io/google-containers/hyperkube:v1.12.1", LogLevel: "2", } - if err := oa.CleanOperator(operatorInfo); err != nil { - glog.Fatal(err) - } - if err := oa.DeployOperator(operatorInfo); err != nil { - glog.Fatal(err) - } + perror(oa.CleanOperator(operatorInfo)) + perror(oa.DeployOperator(operatorInfo)) clusterInfo := &tests.TidbClusterInfo{ Namespace: "tidb", @@ -68,18 +70,35 @@ func main() { TiDBImage: "pingcap/tidb:v2.1.3", StorageClassName: "local-storage", Password: "admin", - Args: map[string]string{}, - } - if err := oa.CleanTidbCluster(clusterInfo); err != nil { - glog.Fatal(err) - } - if err := oa.DeployTidbCluster(clusterInfo); err != nil { - glog.Fatal(err) - } - if err := oa.CheckTidbClusterStatus(clusterInfo); err != nil { - glog.Fatal(err) + Resources: map[string]string{ + "pd.resources.limits.cpu": "1000m", + "pd.resources.limits.memory": "2Gi", + "pd.resources.requests.cpu": "200m", + "pd.resources.requests.memory": "1Gi", + "tikv.resources.limits.cpu": "2000m", + "tikv.resources.limits.memory": "4Gi", + "tikv.resources.requests.cpu": "1000m", + "tikv.resources.requests.memory": "2Gi", + "tidb.resources.limits.cpu": "2000m", + "tidb.resources.limits.memory": "4Gi", + "tidb.resources.requests.cpu": "500m", + "tidb.resources.requests.memory": "1Gi", + }, + Args: map[string]string{}, } + perror(oa.CleanTidbCluster(clusterInfo)) + perror(oa.DeployTidbCluster(clusterInfo)) + perror(oa.CheckTidbClusterStatus(clusterInfo)) + + clusterInfo = clusterInfo.ScaleTiDB(3) + perror(oa.ScaleTidbCluster(clusterInfo)) + perror(oa.CheckTidbClusterStatus(clusterInfo)) + + clusterInfo = clusterInfo.UpgradeAll("v2.1.4") + perror(oa.UpgradeTidbCluster(clusterInfo)) + perror(oa.CheckTidbClusterStatus(clusterInfo)) + restoreClusterInfo := &tests.TidbClusterInfo{ Namespace: "tidb", ClusterName: "demo2", @@ -89,16 +108,25 @@ func main() { TiDBImage: "pingcap/tidb:v2.1.3", StorageClassName: "local-storage", Password: "admin", - Args: map[string]string{}, + Resources: map[string]string{ + "pd.resources.limits.cpu": "1000m", + "pd.resources.limits.memory": "2Gi", + "pd.resources.requests.cpu": "200m", + "pd.resources.requests.memory": "1Gi", + "tikv.resources.limits.cpu": "2000m", + "tikv.resources.limits.memory": "4Gi", + "tikv.resources.requests.cpu": "1000m", + "tikv.resources.requests.memory": "2Gi", + "tidb.resources.limits.cpu": "2000m", + "tidb.resources.limits.memory": "4Gi", + "tidb.resources.requests.cpu": "500m", + "tidb.resources.requests.memory": "1Gi", + }, + Args: map[string]string{}, } - if err := oa.CleanTidbCluster(restoreClusterInfo); err != nil { - glog.Fatal(err) - } - if err := oa.DeployTidbCluster(restoreClusterInfo); err != nil { - glog.Fatal(err) - } - if err := oa.CheckTidbClusterStatus(restoreClusterInfo); err != nil { - glog.Fatal(err) - } + perror(oa.CleanTidbCluster(restoreClusterInfo)) + perror(oa.DeployTidbCluster(restoreClusterInfo)) + perror(oa.CheckTidbClusterStatus(restoreClusterInfo)) + }