From f3643ebafec97794925e14ac1c0672c3bd96a334 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Wed, 17 Apr 2019 16:42:59 +0800 Subject: [PATCH 1/2] use tempdir --- docs/stability-test-cookbook.md | 14 +------ tests/actions.go | 72 ++++++++++++++++++++------------- tests/cmd/e2e/main.go | 8 +--- tests/config.go | 23 +++++++++-- tests/fault.go | 3 -- 5 files changed, 67 insertions(+), 53 deletions(-) diff --git a/docs/stability-test-cookbook.md b/docs/stability-test-cookbook.md index e163811d0b..612dfa10b4 100644 --- a/docs/stability-test-cookbook.md +++ b/docs/stability-test-cookbook.md @@ -22,7 +22,7 @@ Deploy & witness flow can be tedious when developing stability-test, this docume ```shell $ telepresence --new-deployment ${POD_NAME} $ go build -o stability ./tests/cmd/stability/main.go -$ ./stability --operator-repo-dir=${ABITRARY_EMPTY_DIR_TO_CLONE_OPERATOR_REPO} --kubeconfig=${YOUR_KUBE_CONFIG_PATH} +$ ./stability --kubeconfig=${YOUR_KUBE_CONFIG_PATH} ``` ### Explained @@ -34,17 +34,7 @@ Generally we have three problems to solve: * if `KUBECONFIG` env variable set, use it * try loading `InClusterConfig()` so you have to specify the `kubeconfig` path by either command line option or env variable if you want to test locally. -2. **Privilege issue**: If you don't want to or cannot run stability test with root privilege, change the working dir or create it in advance: - * git repo dir can be overridden by option `--git-repo-dir=xxxx`, but helm dir must be created manually. -```shell -# helm dir -$ mkdir /charts -$ chmod 777 /charts -# git repo dir if you don't set command line option -$ mkdir /tidb-operator -$ chmod 777 /tidb-operator -``` -3. **DNS and network issue**: Two-way proxy using Telepresence. We cannot resolve cluster dns name and access cluster ip easily, `telepresence` helps with that, it creates a proxy pod in the cluster and open a vpn connection to kubernetes cluster via this pod. Just run ([full documentations](https://www.telepresence.io/reference/install)): +2. **DNS and network issue**: Two-way proxy using Telepresence. We cannot resolve cluster dns name and access cluster ip easily, `telepresence` helps with that, it creates a proxy pod in the cluster and open a vpn connection to kubernetes cluster via this pod. Just run ([full documentations](https://www.telepresence.io/reference/install)): ```shell $ brew cask install osxfuse $ brew install datawire/blackbird/telepresence diff --git a/tests/actions.go b/tests/actions.go index 02af71ef7a..a270b48350 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -21,6 +21,7 @@ import ( "net/http" "net/url" "os/exec" + "path/filepath" "strconv" "strings" "time" @@ -66,11 +67,14 @@ func NewOperatorActions(cli versioned.Interface, kubeCli kubernetes.Interface, c } const ( - DefaultPollTimeout time.Duration = 10 * time.Minute - DefaultPollInterval time.Duration = 1 * time.Minute - getBackupDirPodName = "get-backup-dir" - grafanaUsername = "admin" - grafanaPassword = "admin" + DefaultPollTimeout time.Duration = 10 * time.Minute + DefaultPollInterval time.Duration = 1 * time.Minute + getBackupDirPodName = "get-backup-dir" + grafanaUsername = "admin" + grafanaPassword = "admin" + operartorChartName = "tidb-operator" + tidbClusterChartName = "tidb-cluster" + backupChartName = "tidb-backup" ) type OperatorActions interface { @@ -242,11 +246,11 @@ func (oa *operatorActions) DeployOperator(info *OperatorConfig) error { } } - cmd := fmt.Sprintf(`helm install /charts/%s/tidb-operator \ + cmd := fmt.Sprintf(`helm install %s \ --name %s \ --namespace %s \ --set-string %s`, - info.Tag, + oa.operatorChartPath(info.Tag), info.ReleaseName, info.Namespace, info.OperatorHelmSetString(nil)) @@ -284,9 +288,9 @@ func (oa *operatorActions) UpgradeOperator(info *OperatorConfig) error { return err } - cmd := fmt.Sprintf(`helm upgrade %s /charts/%s/tidb-operator + cmd := fmt.Sprintf(`helm upgrade %s %s --set operatorImage=%s`, - info.ReleaseName, info.Tag, + info.ReleaseName, oa.operatorChartPath(info.Tag), info.Image) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -313,8 +317,8 @@ func (oa *operatorActions) DeployTidbCluster(info *TidbClusterConfig) error { return fmt.Errorf("failed to create secret of cluster [%s]: %v", info.ClusterName, err) } - cmd := fmt.Sprintf("helm install /charts/%s/tidb-cluster --name %s --namespace %s --set-string %s", - info.OperatorTag, info.ClusterName, info.Namespace, info.TidbClusterHelmSetString(nil)) + cmd := fmt.Sprintf("helm install %s --name %s --namespace %s --set-string %s", + oa.tidbClusterChartPath(info.OperatorTag), info.ClusterName, info.Namespace, info.TidbClusterHelmSetString(nil)) if res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil { return fmt.Errorf("failed to deploy tidbcluster: %s/%s, %v, %s", info.Namespace, info.ClusterName, err, string(res)) @@ -495,13 +499,25 @@ func (oa *operatorActions) StopInsertDataTo(info *TidbClusterConfig) error { return nil } -func chartPath(name string, tag string) string { - return "/charts/" + tag + "/" + name +func (oa *operatorActions) chartPath(name string, tag string) string { + return filepath.Join(oa.cfg.ChartDir, tag, name) +} + +func (oa *operatorActions) operatorChartPath(tag string) string { + return oa.chartPath(operartorChartName, tag) +} + +func (oa *operatorActions) tidbClusterChartPath(tag string) string { + return oa.chartPath(tidbClusterChartName, tag) +} + +func (oa *operatorActions) backupChartPath(tag string) string { + return oa.chartPath(backupChartName, tag) } func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterConfig) error { cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", - info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.TidbClusterHelmSetString(nil)) + info.ClusterName, oa.tidbClusterChartPath(info.OperatorTag), info.TidbClusterHelmSetString(nil)) glog.Info("[SCALE] " + cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -575,7 +591,7 @@ func (oa *operatorActions) CheckScaledCorrectly(info *TidbClusterConfig, podUIDs func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterConfig) error { cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", - info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.TidbClusterHelmSetString(nil)) + info.ClusterName, oa.tidbClusterChartPath(info.OperatorTag), info.TidbClusterHelmSetString(nil)) glog.Info("[UPGRADE] " + cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -1333,11 +1349,11 @@ func (oa *operatorActions) checkoutTag(tagName string) error { cmd := fmt.Sprintf(`cd %s && git stash -u && git checkout %s && - mkdir -p /charts/%s && - cp -rf charts/tidb-operator /charts/%s/tidb-operator && - cp -rf charts/tidb-cluster /charts/%s/tidb-cluster && - cp -rf charts/tidb-backup /charts/%s/tidb-backup`, - oa.cfg.OperatorRepoDir, tagName, tagName, tagName, tagName, tagName) + mkdir -p %s && + cp -rf charts/tidb-operator %s && + cp -rf charts/tidb-cluster %s && + cp -rf charts/tidb-backup %s`, + oa.cfg.OperatorRepoDir, tagName, filepath.Join(oa.cfg.ChartDir, tagName), oa.operatorChartPath(tagName), oa.tidbClusterChartPath(tagName), oa.backupChartPath(tagName)) glog.Info(cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -1361,8 +1377,8 @@ func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterConfig) error { setString := info.BackupHelmSetString(sets) fullbackupName := fmt.Sprintf("%s-backup", info.ClusterName) - cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s", - fullbackupName, info.Namespace, info.OperatorTag, setString) + cmd := fmt.Sprintf("helm install -n %s --namespace %s %s --set-string %s", + fullbackupName, info.Namespace, oa.backupChartPath(info.OperatorTag), setString) glog.Infof("install adhoc deployment [%s]", cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -1412,8 +1428,8 @@ func (oa *operatorActions) Restore(from *TidbClusterConfig, to *TidbClusterConfi setString := to.BackupHelmSetString(sets) restoreName := fmt.Sprintf("%s-restore", from.ClusterName) - cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s", - restoreName, to.Namespace, to.OperatorTag, setString) + cmd := fmt.Sprintf("helm install -n %s --namespace %s %s --set-string %s", + restoreName, to.Namespace, oa.backupChartPath(to.OperatorTag), setString) glog.Infof("install restore [%s]", cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { @@ -1558,8 +1574,8 @@ func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterConfig) error setString := info.TidbClusterHelmSetString(sets) - cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s", - info.ClusterName, info.OperatorTag, setString) + cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", + info.ClusterName, oa.tidbClusterChartPath(info.OperatorTag), setString) glog.Infof("scheduled-backup delploy [%s]", cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() @@ -1745,8 +1761,8 @@ func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterConfig, to * setString := from.TidbClusterHelmSetString(sets) - cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s", - from.ClusterName, from.OperatorTag, setString) + cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", + from.ClusterName, oa.tidbClusterChartPath(from.OperatorTag), setString) glog.Infof(cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index adc56f23a1..0af78573ca 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -32,14 +32,10 @@ func main() { logs.InitLogs() defer logs.FlushLogs() - conf := tests.NewConfig() - err := conf.Parse() - if err != nil { - glog.Fatalf("failed to parse config: %v", err) - } + conf := tests.ParseConfigOrDie() + conf.ChartDir = "/charts" cli, kubeCli := client.NewCliOrDie() - oa := tests.NewOperatorActions(cli, kubeCli, conf) operatorInfo := &tests.OperatorConfig{ diff --git a/tests/config.go b/tests/config.go index 7af7be0a73..2f2fb3b9d3 100644 --- a/tests/config.go +++ b/tests/config.go @@ -25,6 +25,8 @@ type Config struct { // For local test OperatorRepoDir string `yaml:"operator_repo_dir" json:"operator_repo_dir"` + // chart dir + ChartDir string `yaml:"chart_dir" json:"chart_dir"` } // Nodes defines a series of nodes that belong to the same physical node. @@ -34,7 +36,7 @@ type Nodes struct { } // NewConfig creates a new config. -func NewConfig() *Config { +func NewConfig() (*Config, error) { cfg := &Config{} flag.StringVar(&cfg.configFile, "config", "", "Config file") flag.StringVar(&cfg.LogDir, "log-dir", "/logDir", "log directory") @@ -42,14 +44,27 @@ func NewConfig() *Config { flag.StringVar(&cfg.TidbVersions, "tidb-versions", "v2.1.3,v2.1.4", "tidb versions") flag.StringVar(&cfg.OperatorTag, "operator-tag", "master", "operator tag used to choose charts") flag.StringVar(&cfg.OperatorImage, "operator-image", "pingcap/tidb-operator:latest", "operator image") - flag.StringVar(&cfg.OperatorRepoDir, "operator-repo-dir", "/tidb-operator", "local directory to which tidb-operator cloned") flag.Parse() - return cfg + operatorRepo, err := ioutil.TempDir("", "tidb-operator") + if err != nil { + return nil, err + } + cfg.OperatorRepoDir = operatorRepo + + chartDir, err := ioutil.TempDir("", "charts") + if err != nil { + return nil, err + } + cfg.ChartDir = chartDir + return cfg, nil } func ParseConfigOrDie() *Config { - cfg := NewConfig() + cfg, err := NewConfig() + if err != nil { + panic(err) + } if err := cfg.Parse(); err != nil { panic(err) } diff --git a/tests/fault.go b/tests/fault.go index d1b2862507..319f57fc6d 100644 --- a/tests/fault.go +++ b/tests/fault.go @@ -361,9 +361,6 @@ func getFaultNode(kubeCli kubernetes.Interface) (string, error) { } myNode := getMyNodeName() - if myNode == "" { - return "", fmt.Errorf("get own node name is empty") - } index := rand.Intn(len(nodes.Items)) faultNode := nodes.Items[index].Name From ef00d97e81ee81327ea58b575dc25c39496d74bf Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Wed, 17 Apr 2019 17:31:26 +0800 Subject: [PATCH 2/2] address comment --- tests/cmd/e2e/main.go | 7 +++++++ tests/cmd/stability/main.go | 5 +++++ tests/config.go | 17 +++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index 0af78573ca..414c4e33e7 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -254,4 +254,11 @@ func main() { if err := backupCase.Run(); err != nil { glog.Fatal(err) } + + //clean temp dirs when e2e success + err = conf.CleanTempDirs() + if err != nil { + glog.Errorf("failed to clean temp dirs, this error can be ignored.") + } + glog.Infof("\nFinished.") } diff --git a/tests/cmd/stability/main.go b/tests/cmd/stability/main.go index e9a5733eef..7db5bdc4f9 100644 --- a/tests/cmd/stability/main.go +++ b/tests/cmd/stability/main.go @@ -217,5 +217,10 @@ func main() { // truncate a sst file and check failover oa.TruncateSSTFileThenCheckFailoverOrDie(cluster1, 5*time.Minute) + //clean temp dirs when stability success + err := conf.CleanTempDirs() + if err != nil { + glog.Errorf("failed to clean temp dirs, this error can be ignored.") + } glog.Infof("\nFinished.") } diff --git a/tests/config.go b/tests/config.go index 2f2fb3b9d3..42e1009d73 100644 --- a/tests/config.go +++ b/tests/config.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "io/ioutil" + "os" "strings" "github.com/golang/glog" @@ -135,3 +136,19 @@ func (c *Config) GetUpgradeTidbVersionsOrDie() []string { return versions } + +func (c *Config) CleanTempDirs() error { + if c.OperatorRepoDir != "" { + err := os.RemoveAll(c.OperatorRepoDir) + if err != nil { + return err + } + } + if c.ChartDir != "" { + err := os.RemoveAll(c.ChartDir) + if err != nil { + return err + } + } + return nil +}