diff --git a/docs/stability-test-cookbook.md b/docs/stability-test-cookbook.md index e163811d0b..612dfa10b4 100644 --- a/docs/stability-test-cookbook.md +++ b/docs/stability-test-cookbook.md @@ -22,7 +22,7 @@ Deploy & witness flow can be tedious when developing stability-test, this docume ```shell $ telepresence --new-deployment ${POD_NAME} $ go build -o stability ./tests/cmd/stability/main.go -$ ./stability --operator-repo-dir=${ABITRARY_EMPTY_DIR_TO_CLONE_OPERATOR_REPO} --kubeconfig=${YOUR_KUBE_CONFIG_PATH} +$ ./stability --kubeconfig=${YOUR_KUBE_CONFIG_PATH} ``` ### Explained @@ -34,17 +34,7 @@ Generally we have three problems to solve: * if `KUBECONFIG` env variable set, use it * try loading `InClusterConfig()` so you have to specify the `kubeconfig` path by either command line option or env variable if you want to test locally. -2. **Privilege issue**: If you don't want to or cannot run stability test with root privilege, change the working dir or create it in advance: - * git repo dir can be overridden by option `--git-repo-dir=xxxx`, but helm dir must be created manually. -```shell -# helm dir -$ mkdir /charts -$ chmod 777 /charts -# git repo dir if you don't set command line option -$ mkdir /tidb-operator -$ chmod 777 /tidb-operator -``` -3. **DNS and network issue**: Two-way proxy using Telepresence. We cannot resolve cluster dns name and access cluster ip easily, `telepresence` helps with that, it creates a proxy pod in the cluster and open a vpn connection to kubernetes cluster via this pod. Just run ([full documentations](https://www.telepresence.io/reference/install)): +2. **DNS and network issue**: Two-way proxy using Telepresence. We cannot resolve cluster dns name and access cluster ip easily, `telepresence` helps with that, it creates a proxy pod in the cluster and open a vpn connection to kubernetes cluster via this pod. Just run ([full documentations](https://www.telepresence.io/reference/install)): ```shell $ brew cask install osxfuse $ brew install datawire/blackbird/telepresence diff --git a/tests/actions.go b/tests/actions.go index 7869fe5e30..8fbcd63573 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -22,6 +22,7 @@ import ( "net/url" "os" "os/exec" + "path/filepath" "strconv" "strings" "time" @@ -64,11 +65,14 @@ func NewOperatorActions(cli versioned.Interface, kubeCli kubernetes.Interface, c } const ( - DefaultPollTimeout time.Duration = 10 * time.Minute - DefaultPollInterval time.Duration = 1 * time.Minute - getBackupDirPodName = "get-backup-dir" - grafanaUsername = "admin" - grafanaPassword = "admin" + DefaultPollTimeout time.Duration = 10 * time.Minute + DefaultPollInterval time.Duration = 1 * time.Minute + getBackupDirPodName = "get-backup-dir" + grafanaUsername = "admin" + grafanaPassword = "admin" + operartorChartName = "tidb-operator" + tidbClusterChartName = "tidb-cluster" + backupChartName = "tidb-backup" ) type OperatorActions interface { @@ -250,11 +254,11 @@ func (oa *operatorActions) DeployOperator(info *OperatorConfig) error { } } - cmd := fmt.Sprintf(`helm install /charts/%s/tidb-operator \ + cmd := fmt.Sprintf(`helm install %s \ --name %s \ --namespace %s \ --set-string %s`, - info.Tag, + oa.operatorChartPath(info.Tag), info.ReleaseName, info.Namespace, info.OperatorHelmSetString(nil)) @@ -299,9 +303,9 @@ func (oa *operatorActions) UpgradeOperator(info *OperatorConfig) error { return err } - cmd := fmt.Sprintf(`helm upgrade %s /charts/%s/tidb-operator + cmd := fmt.Sprintf(`helm upgrade %s %s --set operatorImage=%s`, - info.ReleaseName, info.Tag, + info.ReleaseName, oa.operatorChartPath(info.Tag), info.Image) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -328,8 +332,8 @@ func (oa *operatorActions) DeployTidbCluster(info *TidbClusterConfig) error { return fmt.Errorf("failed to create secret of cluster [%s]: %v", info.ClusterName, err) } - cmd := fmt.Sprintf("helm install /charts/%s/tidb-cluster --name %s --namespace %s --set-string %s", - info.OperatorTag, info.ClusterName, info.Namespace, info.TidbClusterHelmSetString(nil)) + cmd := fmt.Sprintf("helm install %s --name %s --namespace %s --set-string %s", + oa.tidbClusterChartPath(info.OperatorTag), info.ClusterName, info.Namespace, info.TidbClusterHelmSetString(nil)) glog.Infof(cmd) if res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil { return fmt.Errorf("failed to deploy tidbcluster: %s/%s, %v, %s", @@ -518,13 +522,25 @@ func (oa *operatorActions) StopInsertDataTo(info *TidbClusterConfig) { info.blockWriter.Stop() } -func chartPath(name string, tag string) string { - return "/charts/" + tag + "/" + name +func (oa *operatorActions) chartPath(name string, tag string) string { + return filepath.Join(oa.cfg.ChartDir, tag, name) +} + +func (oa *operatorActions) operatorChartPath(tag string) string { + return oa.chartPath(operartorChartName, tag) +} + +func (oa *operatorActions) tidbClusterChartPath(tag string) string { + return oa.chartPath(tidbClusterChartName, tag) +} + +func (oa *operatorActions) backupChartPath(tag string) string { + return oa.chartPath(backupChartName, tag) } func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterConfig) error { cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", - info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.TidbClusterHelmSetString(nil)) + info.ClusterName, oa.tidbClusterChartPath(info.OperatorTag), info.TidbClusterHelmSetString(nil)) glog.Info("[SCALE] " + cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -598,7 +614,7 @@ func (oa *operatorActions) CheckScaledCorrectly(info *TidbClusterConfig, podUIDs func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterConfig) error { cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", - info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.TidbClusterHelmSetString(nil)) + info.ClusterName, oa.tidbClusterChartPath(info.OperatorTag), info.TidbClusterHelmSetString(nil)) glog.Info("[UPGRADE] " + cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -1356,11 +1372,11 @@ func (oa *operatorActions) checkoutTag(tagName string) error { cmd := fmt.Sprintf(`cd %s && git stash -u && git checkout %s && - mkdir -p /charts/%s && - cp -rf charts/tidb-operator /charts/%s/tidb-operator && - cp -rf charts/tidb-cluster /charts/%s/tidb-cluster && - cp -rf charts/tidb-backup /charts/%s/tidb-backup`, - oa.cfg.OperatorRepoDir, tagName, tagName, tagName, tagName, tagName) + mkdir -p %s && + cp -rf charts/tidb-operator %s && + cp -rf charts/tidb-cluster %s && + cp -rf charts/tidb-backup %s`, + oa.cfg.OperatorRepoDir, tagName, filepath.Join(oa.cfg.ChartDir, tagName), oa.operatorChartPath(tagName), oa.tidbClusterChartPath(tagName), oa.backupChartPath(tagName)) glog.Info(cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -1384,8 +1400,8 @@ func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterConfig) error { setString := info.BackupHelmSetString(sets) fullbackupName := fmt.Sprintf("%s-backup", info.ClusterName) - cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s", - fullbackupName, info.Namespace, info.OperatorTag, setString) + cmd := fmt.Sprintf("helm install -n %s --namespace %s %s --set-string %s", + fullbackupName, info.Namespace, oa.backupChartPath(info.OperatorTag), setString) glog.Infof("install adhoc deployment [%s]", cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -1435,8 +1451,8 @@ func (oa *operatorActions) Restore(from *TidbClusterConfig, to *TidbClusterConfi setString := to.BackupHelmSetString(sets) restoreName := fmt.Sprintf("%s-restore", from.ClusterName) - cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s", - restoreName, to.Namespace, to.OperatorTag, setString) + cmd := fmt.Sprintf("helm install -n %s --namespace %s %s --set-string %s", + restoreName, to.Namespace, oa.backupChartPath(to.OperatorTag), setString) glog.Infof("install restore [%s]", cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -1581,8 +1597,8 @@ func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterConfig) error setString := info.TidbClusterHelmSetString(sets) - cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s", - info.ClusterName, info.OperatorTag, setString) + cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", + info.ClusterName, oa.tidbClusterChartPath(info.OperatorTag), setString) glog.Infof("scheduled-backup delploy [%s]", cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() @@ -1768,8 +1784,8 @@ func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterConfig, to * setString := from.TidbClusterHelmSetString(sets) - cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s", - from.ClusterName, from.OperatorTag, setString) + cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", + from.ClusterName, oa.tidbClusterChartPath(from.OperatorTag), setString) glog.Infof(cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index 8834ef0472..5c71b1a71f 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -30,14 +30,10 @@ func main() { logs.InitLogs() defer logs.FlushLogs() - conf := tests.NewConfig() - err := conf.Parse() - if err != nil { - glog.Fatalf("failed to parse config: %v", err) - } + conf := tests.ParseConfigOrDie() + conf.ChartDir = "/charts" cli, kubeCli := client.NewCliOrDie() - oa := tests.NewOperatorActions(cli, kubeCli, conf) // start a http server in goruntine @@ -253,4 +249,11 @@ func main() { if err := backupCase.Run(); err != nil { glog.Fatal(err) } + + //clean temp dirs when e2e success + err = conf.CleanTempDirs() + if err != nil { + glog.Errorf("failed to clean temp dirs, this error can be ignored.") + } + glog.Infof("\nFinished.") } diff --git a/tests/cmd/stability/main.go b/tests/cmd/stability/main.go index 4b8ce8cb53..7ebf2ad81e 100644 --- a/tests/cmd/stability/main.go +++ b/tests/cmd/stability/main.go @@ -222,5 +222,10 @@ func main() { // truncate a sst file and check failover oa.TruncateSSTFileThenCheckFailoverOrDie(cluster1, 5*time.Minute) + //clean temp dirs when stability success + err := conf.CleanTempDirs() + if err != nil { + glog.Errorf("failed to clean temp dirs, this error can be ignored.") + } glog.Infof("\nFinished.") } diff --git a/tests/config.go b/tests/config.go index b363b5dc0b..c22445aadc 100644 --- a/tests/config.go +++ b/tests/config.go @@ -4,10 +4,12 @@ import ( "crypto/tls" "flag" "fmt" - "github.com/pingcap/tidb-operator/tests/pkg/blockwriter" "io/ioutil" + "os" "strings" + "github.com/pingcap/tidb-operator/tests/pkg/blockwriter" + "github.com/golang/glog" "gopkg.in/yaml.v2" ) @@ -39,6 +41,8 @@ type Config struct { // For local test OperatorRepoDir string `yaml:"operator_repo_dir" json:"operator_repo_dir"` + // chart dir + ChartDir string `yaml:"chart_dir" json:"chart_dir"` } // Nodes defines a series of nodes that belong to the same physical node. @@ -48,7 +52,7 @@ type Nodes struct { } // NewConfig creates a new config. -func NewConfig() *Config { +func NewConfig() (*Config, error) { cfg := &Config{ BlockWriter: blockwriter.Config{ TableNum: defaultTableNum, @@ -71,11 +75,25 @@ func NewConfig() *Config { flag.StringVar(&cfg.OperatorRepoDir, "operator-repo-dir", "/tidb-operator", "local directory to which tidb-operator cloned") flag.Parse() - return cfg + operatorRepo, err := ioutil.TempDir("", "tidb-operator") + if err != nil { + return nil, err + } + cfg.OperatorRepoDir = operatorRepo + + chartDir, err := ioutil.TempDir("", "charts") + if err != nil { + return nil, err + } + cfg.ChartDir = chartDir + return cfg, nil } func ParseConfigOrDie() *Config { - cfg := NewConfig() + cfg, err := NewConfig() + if err != nil { + panic(err) + } if err := cfg.Parse(); err != nil { panic(err) } @@ -156,3 +174,19 @@ func (c *Config) GetUpgradeTidbVersionsOrDie() []string { return versions } + +func (c *Config) CleanTempDirs() error { + if c.OperatorRepoDir != "" { + err := os.RemoveAll(c.OperatorRepoDir) + if err != nil { + return err + } + } + if c.ChartDir != "" { + err := os.RemoveAll(c.ChartDir) + if err != nil { + return err + } + } + return nil +} diff --git a/tests/fault.go b/tests/fault.go index d1b2862507..319f57fc6d 100644 --- a/tests/fault.go +++ b/tests/fault.go @@ -361,9 +361,6 @@ func getFaultNode(kubeCli kubernetes.Interface) (string, error) { } myNode := getMyNodeName() - if myNode == "" { - return "", fmt.Errorf("get own node name is empty") - } index := rand.Intn(len(nodes.Items)) faultNode := nodes.Items[index].Name