Skip to content

Commit

Permalink
fix bakcup test bugs (pingcap#335)
Browse files Browse the repository at this point in the history 
* fix-main
  • Loading branch information
@xiaojingchen @weekface
xiaojingchen authored and weekface committed Mar 21, 2019
1 parent 6e36dca commit f99bfdf
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 35 deletions.
62 changes: 42 additions & 20 deletions tests/actions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,28 @@ type TidbClusterInfo struct {
BackupSecretName string
}

func (tc *TidbClusterInfo) HelmSetString(m map[string]string) string {
func (tc *TidbClusterInfo) BackupHelmSetString(m map[string]string) string {

set := map[string]string{
"clusterName": tc.ClusterName,
"secretName": tc.BackupSecretName,
}

for k, v := range tc.Args {
set[k] = v
}
for k, v := range m {
set[k] = v
}

arr := make([]string, 0, len(set))
for k, v := range set {
arr = append(arr, fmt.Sprintf("%s=%s", k, v))
}
return strings.Join(arr, ",")
}

func (tc *TidbClusterInfo) TidbClusterHelmSetString(m map[string]string) string {

set := map[string]string{
"clusterName": tc.ClusterName,
Expand All @@ -150,7 +171,6 @@ func (tc *TidbClusterInfo) HelmSetString(m map[string]string) string {
"tidb.passwordSecretName": tc.InitSecretName,
"tidb.initSql": tc.InitSql,
"monitor.create": strconv.FormatBool(tc.Monitor),
"secretName": tc.BackupSecretName,
}

for k, v := range tc.Resources {
Expand Down Expand Up @@ -237,7 +257,7 @@ func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error {
}

cmd := fmt.Sprintf("helm install /charts/%s/tidb-cluster --name %s --namespace %s --set-string %s",
info.OperatorTag, info.ClusterName, info.Namespace, info.HelmSetString(nil))
info.OperatorTag, info.ClusterName, info.Namespace, info.TidbClusterHelmSetString(nil))
if res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil {
return fmt.Errorf("failed to deploy tidbcluster: %s/%s, %v, %s",
info.Namespace, info.ClusterName, err, string(res))
Expand Down Expand Up @@ -405,7 +425,7 @@ func chartPath(name string, tag string) string {

func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error {
cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString(nil))
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.TidbClusterHelmSetString(nil))
glog.Info("[SCALE] " + cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand All @@ -416,7 +436,7 @@ func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error {

func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterInfo) error {
cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString(nil))
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.TidbClusterHelmSetString(nil))
glog.Info("[UPGRADE] " + cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -1056,7 +1076,7 @@ func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterInfo) error {
"storage.size": "10Gi",
}

setString := info.HelmSetString(sets)
setString := info.BackupHelmSetString(sets)

fullbackupName := fmt.Sprintf("%s-backup", info.ClusterName)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s",
Expand Down Expand Up @@ -1112,11 +1132,11 @@ func (oa *operatorActions) Restore(from *TidbClusterInfo, to *TidbClusterInfo) e
"storage.size": "10Gi",
}

setString := to.HelmSetString(sets)
setString := to.BackupHelmSetString(sets)

restoreName := fmt.Sprintf("%s-restore", from.ClusterName)
cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-backup --set-string %s",
restoreName, to.OperatorTag, setString)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s",
restoreName, to.Namespace, to.OperatorTag, setString)
glog.Infof("install restore [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -1257,15 +1277,16 @@ func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterInfo) error {

cron := fmt.Sprintf("'*/1 * * * *'")
sets := map[string]string{
"clusterName": info.ClusterName,
"scheduledBackup.create": "true",
"scheduledBackup.user": "root",
"scheduledBackup.password": info.Password,
"scheduledBackup.schedule": cron,
"scheduledBackup.storage": "10Gi",
"clusterName": info.ClusterName,
"scheduledBackup.create": "true",
"scheduledBackup.user": "root",
"scheduledBackup.password": info.Password,
"scheduledBackup.schedule": cron,
"scheduledBackup.storage": "10Gi",
"scheduledBackup.secretName": info.BackupSecretName,
}

setString := info.HelmSetString(sets)
setString := info.TidbClusterHelmSetString(sets)

cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s",
info.ClusterName, info.OperatorTag, setString)
Expand Down Expand Up @@ -1333,7 +1354,7 @@ func (oa *operatorActions) CheckScheduledBackup(info *TidbClusterInfo) error {
return fmt.Errorf("failed to get backup dir: %v", err)
}

if len(dirs) != 3 {
if len(dirs) <= 2 {
return fmt.Errorf("scheduler job failed!")
}

Expand All @@ -1356,6 +1377,7 @@ func getParentUIDFromJob(j batchv1.Job) (types.UID, bool) {
}

func (oa *operatorActions) getBackupDir(info *TidbClusterInfo) ([]string, error) {
scheduledPvcName := fmt.Sprintf("%s-scheduled-backup", info.ClusterName)
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: getBackupDirPodName,
Expand All @@ -1380,7 +1402,7 @@ func (oa *operatorActions) getBackupDir(info *TidbClusterInfo) ([]string, error)
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: info.BackupPVC,
ClaimName: scheduledPvcName,
},
},
},
Expand Down Expand Up @@ -1433,7 +1455,7 @@ func (oa *operatorActions) getBackupDir(info *TidbClusterInfo) ([]string, error)
}

dirs := strings.Split(string(res), "\n")
glog.Infof("dirs in pod info name [%s] dir name [%s]", info.BackupPVC, strings.Join(dirs, ","))
glog.Infof("dirs in pod info name [%s] dir name [%s]", scheduledPvcName, strings.Join(dirs, ","))
return dirs, nil
}

Expand All @@ -1456,7 +1478,7 @@ func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterInfo, to *Ti
"binlog.drainer.mysql.port": "4000",
}

setString := from.HelmSetString(sets)
setString := from.TidbClusterHelmSetString(sets)

cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s",
from.ClusterName, from.OperatorTag, setString)
Expand Down
2 changes: 0 additions & 2 deletions tests/backup/backupcase.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,6 @@ func (bc *BackupCase) Run() error {
return err
}

bc.srcCluster.BackupPVC = "demo-scheduled-backup"

err = bc.operator.DeployScheduledBackup(bc.srcCluster)
if err != nil {
glog.Errorf("cluster:[%s] scheduler happen error: %v", bc.srcCluster.ClusterName, err)
Expand Down
31 changes: 18 additions & 13 deletions tests/cmd/e2e/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,14 @@ import (
"net/http"
_ "net/http/pprof"

"github.com/pingcap/tidb-operator/tests/pkg/workload"
"github.com/pingcap/tidb-operator/tests/pkg/workload/ddl"

"github.com/golang/glog"
"github.com/jinzhu/copier"
"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
"github.com/pingcap/tidb-operator/tests"
"github.com/pingcap/tidb-operator/tests/backup"
"github.com/pingcap/tidb-operator/tests/pkg/workload"
"github.com/pingcap/tidb-operator/tests/pkg/workload/ddl"
"k8s.io/apiserver/pkg/util/logs"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
Expand Down Expand Up @@ -77,10 +78,12 @@ func main() {
// create database and table and insert a column for test backup and restore
initSql := `"create database record;use record;create table test(t char(32))"`

name1 := "e2e-cluster1"
name2 := "e2e-cluster2"
clusterInfos := []*tests.TidbClusterInfo{
{
Namespace: "e2e-cluster1",
ClusterName: "e2e-cluster1",
Namespace: name1,
ClusterName: name1,
OperatorTag: operatorTag,
PDImage: fmt.Sprintf("pingcap/pd:%s", beginTidbVersion),
TiKVImage: fmt.Sprintf("pingcap/tikv:%s", beginTidbVersion),
Expand All @@ -89,9 +92,9 @@ func main() {
Password: "admin",
InitSql: initSql,
UserName: "root",
InitSecretName: "demo-set-secret",
BackupSecretName: "demo-backup-secret",
BackupPVC: "test-backup",
InitSecretName: fmt.Sprintf("%s-set-secret", name1),
BackupSecretName: fmt.Sprintf("%s-backup-secret", name1),
BackupPVC: "backup-pvc",
Resources: map[string]string{
"pd.resources.limits.cpu": "1000m",
"pd.resources.limits.memory": "2Gi",
Expand All @@ -110,8 +113,8 @@ func main() {
Monitor: true,
},
{
Namespace: "e2e-cluster2",
ClusterName: "e2e-cluster2",
Namespace: name2,
ClusterName: name2,
OperatorTag: "master",
PDImage: fmt.Sprintf("pingcap/pd:%s", beginTidbVersion),
TiKVImage: fmt.Sprintf("pingcap/tikv:%s", beginTidbVersion),
Expand All @@ -120,9 +123,9 @@ func main() {
Password: "admin",
InitSql: initSql,
UserName: "root",
InitSecretName: "demo-set-secret",
BackupSecretName: "demo-backup-secret",
BackupPVC: "test-backup",
InitSecretName: fmt.Sprintf("%s-set-secret", name2),
BackupSecretName: fmt.Sprintf("%s-backup-secret", name2),
BackupPVC: "backup-pvc",
Resources: map[string]string{
"pd.resources.limits.cpu": "1000m",
"pd.resources.limits.memory": "2Gi",
Expand Down Expand Up @@ -252,7 +255,9 @@ func main() {
backupClusterInfo := clusterInfos[0]
restoreClusterInfo := &tests.TidbClusterInfo{}
copier.Copy(restoreClusterInfo, backupClusterInfo)
restoreClusterInfo.ClusterName = restoreClusterInfo.ClusterName + "-restore"
restoreClusterInfo.ClusterName = restoreClusterInfo.ClusterName + "-other"
restoreClusterInfo.InitSecretName = fmt.Sprintf("%s-set-secret", restoreClusterInfo.ClusterName)
restoreClusterInfo.BackupSecretName = fmt.Sprintf("%s-backup-secret", restoreClusterInfo.ClusterName)

if err = oa.CleanTidbCluster(restoreClusterInfo); err != nil {
glog.Fatal(err)
Expand Down

0 comments on commit f99bfdf

Please sign in to comment.