Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use ioutil.TempDir() create charts and operator repo's directories #405

Merged
merged 4 commits into from
Apr 19, 2019
Merged

use ioutil.TempDir() create charts and operator repo's directories #405

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f3643eb
use tempdir
xiaojingchen Apr 17, 2019
ef00d97
address comment
xiaojingchen Apr 17, 2019
fb79ca7
Merge branch 'master' into use-tempdir
xiaojingchen Apr 17, 2019
3819b51
Merge branch 'master' into use-tempdir
xiaojingchen Apr 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 2 additions & 12 deletions docs/stability-test-cookbook.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Deploy & witness flow can be tedious when developing stability-test, this docume
```shell
$ telepresence --new-deployment ${POD_NAME}
$ go build -o stability ./tests/cmd/stability/main.go
$ ./stability --operator-repo-dir=${ABITRARY_EMPTY_DIR_TO_CLONE_OPERATOR_REPO} --kubeconfig=${YOUR_KUBE_CONFIG_PATH}
$ ./stability --kubeconfig=${YOUR_KUBE_CONFIG_PATH}
```

### Explained
Expand All @@ -34,17 +34,7 @@ Generally we have three problems to solve:
* if `KUBECONFIG` env variable set, use it
* try loading `InClusterConfig()`
so you have to specify the `kubeconfig` path by either command line option or env variable if you want to test locally.
2. **Privilege issue**: If you don't want to or cannot run stability test with root privilege, change the working dir or create it in advance:
* git repo dir can be overridden by option `--git-repo-dir=xxxx`, but helm dir must be created manually.
```shell
# helm dir
$ mkdir /charts
$ chmod 777 /charts
# git repo dir if you don't set command line option
$ mkdir /tidb-operator
$ chmod 777 /tidb-operator
```
3. **DNS and network issue**: Two-way proxy using Telepresence. We cannot resolve cluster dns name and access cluster ip easily, `telepresence` helps with that, it creates a proxy pod in the cluster and open a vpn connection to kubernetes cluster via this pod. Just run ([full documentations](https://www.telepresence.io/reference/install)):
2. **DNS and network issue**: Two-way proxy using Telepresence. We cannot resolve cluster dns name and access cluster ip easily, `telepresence` helps with that, it creates a proxy pod in the cluster and open a vpn connection to kubernetes cluster via this pod. Just run ([full documentations](https://www.telepresence.io/reference/install)):
```shell
$ brew cask install osxfuse
$ brew install datawire/blackbird/telepresence
Expand Down
72 changes: 44 additions & 28 deletions tests/actions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"net/http"
"net/url"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -66,11 +67,14 @@ func NewOperatorActions(cli versioned.Interface, kubeCli kubernetes.Interface, c
}

const (
DefaultPollTimeout time.Duration = 10 * time.Minute
DefaultPollInterval time.Duration = 1 * time.Minute
getBackupDirPodName = "get-backup-dir"
grafanaUsername = "admin"
grafanaPassword = "admin"
DefaultPollTimeout time.Duration = 10 * time.Minute
DefaultPollInterval time.Duration = 1 * time.Minute
getBackupDirPodName = "get-backup-dir"
grafanaUsername = "admin"
grafanaPassword = "admin"
operartorChartName = "tidb-operator"
tidbClusterChartName = "tidb-cluster"
backupChartName = "tidb-backup"
)

type OperatorActions interface {
Expand Down Expand Up @@ -242,11 +246,11 @@ func (oa *operatorActions) DeployOperator(info *OperatorConfig) error {
}
}

cmd := fmt.Sprintf(`helm install /charts/%s/tidb-operator \
cmd := fmt.Sprintf(`helm install %s \
--name %s \
--namespace %s \
--set-string %s`,
info.Tag,
oa.operatorChartPath(info.Tag),
info.ReleaseName,
info.Namespace,
info.OperatorHelmSetString(nil))
Expand Down Expand Up @@ -284,9 +288,9 @@ func (oa *operatorActions) UpgradeOperator(info *OperatorConfig) error {
return err
}

cmd := fmt.Sprintf(`helm upgrade %s /charts/%s/tidb-operator
cmd := fmt.Sprintf(`helm upgrade %s %s
--set operatorImage=%s`,
info.ReleaseName, info.Tag,
info.ReleaseName, oa.operatorChartPath(info.Tag),
info.Image)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand All @@ -313,8 +317,8 @@ func (oa *operatorActions) DeployTidbCluster(info *TidbClusterConfig) error {
return fmt.Errorf("failed to create secret of cluster [%s]: %v", info.ClusterName, err)
}

cmd := fmt.Sprintf("helm install /charts/%s/tidb-cluster --name %s --namespace %s --set-string %s",
info.OperatorTag, info.ClusterName, info.Namespace, info.TidbClusterHelmSetString(nil))
cmd := fmt.Sprintf("helm install %s --name %s --namespace %s --set-string %s",
oa.tidbClusterChartPath(info.OperatorTag), info.ClusterName, info.Namespace, info.TidbClusterHelmSetString(nil))
if res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil {
return fmt.Errorf("failed to deploy tidbcluster: %s/%s, %v, %s",
info.Namespace, info.ClusterName, err, string(res))
Expand Down Expand Up @@ -495,13 +499,25 @@ func (oa *operatorActions) StopInsertDataTo(info *TidbClusterConfig) error {
return nil
}

func chartPath(name string, tag string) string {
return "/charts/" + tag + "/" + name
func (oa *operatorActions) chartPath(name string, tag string) string {
return filepath.Join(oa.cfg.ChartDir, tag, name)
}

func (oa *operatorActions) operatorChartPath(tag string) string {
return oa.chartPath(operartorChartName, tag)
}

func (oa *operatorActions) tidbClusterChartPath(tag string) string {
return oa.chartPath(tidbClusterChartName, tag)
}

func (oa *operatorActions) backupChartPath(tag string) string {
return oa.chartPath(backupChartName, tag)
}

func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterConfig) error {
cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.TidbClusterHelmSetString(nil))
info.ClusterName, oa.tidbClusterChartPath(info.OperatorTag), info.TidbClusterHelmSetString(nil))
glog.Info("[SCALE] " + cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -575,7 +591,7 @@ func (oa *operatorActions) CheckScaledCorrectly(info *TidbClusterConfig, podUIDs

func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterConfig) error {
cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.TidbClusterHelmSetString(nil))
info.ClusterName, oa.tidbClusterChartPath(info.OperatorTag), info.TidbClusterHelmSetString(nil))
glog.Info("[UPGRADE] " + cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -1333,11 +1349,11 @@ func (oa *operatorActions) checkoutTag(tagName string) error {
cmd := fmt.Sprintf(`cd %s &&
git stash -u &&
git checkout %s &&
mkdir -p /charts/%s &&
cp -rf charts/tidb-operator /charts/%s/tidb-operator &&
cp -rf charts/tidb-cluster /charts/%s/tidb-cluster &&
cp -rf charts/tidb-backup /charts/%s/tidb-backup`,
oa.cfg.OperatorRepoDir, tagName, tagName, tagName, tagName, tagName)
mkdir -p %s &&
cp -rf charts/tidb-operator %s &&
cp -rf charts/tidb-cluster %s &&
cp -rf charts/tidb-backup %s`,
oa.cfg.OperatorRepoDir, tagName, filepath.Join(oa.cfg.ChartDir, tagName), oa.operatorChartPath(tagName), oa.tidbClusterChartPath(tagName), oa.backupChartPath(tagName))
glog.Info(cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand All @@ -1361,8 +1377,8 @@ func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterConfig) error {
setString := info.BackupHelmSetString(sets)

fullbackupName := fmt.Sprintf("%s-backup", info.ClusterName)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s",
fullbackupName, info.Namespace, info.OperatorTag, setString)
cmd := fmt.Sprintf("helm install -n %s --namespace %s %s --set-string %s",
fullbackupName, info.Namespace, oa.backupChartPath(info.OperatorTag), setString)
glog.Infof("install adhoc deployment [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -1412,8 +1428,8 @@ func (oa *operatorActions) Restore(from *TidbClusterConfig, to *TidbClusterConfi
setString := to.BackupHelmSetString(sets)

restoreName := fmt.Sprintf("%s-restore", from.ClusterName)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s",
restoreName, to.Namespace, to.OperatorTag, setString)
cmd := fmt.Sprintf("helm install -n %s --namespace %s %s --set-string %s",
restoreName, to.Namespace, oa.backupChartPath(to.OperatorTag), setString)
glog.Infof("install restore [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -1558,8 +1574,8 @@ func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterConfig) error

setString := info.TidbClusterHelmSetString(sets)

cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s",
info.ClusterName, info.OperatorTag, setString)
cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
info.ClusterName, oa.tidbClusterChartPath(info.OperatorTag), setString)

glog.Infof("scheduled-backup delploy [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
Expand Down Expand Up @@ -1745,8 +1761,8 @@ func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterConfig, to *

setString := from.TidbClusterHelmSetString(sets)

cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s",
from.ClusterName, from.OperatorTag, setString)
cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
from.ClusterName, oa.tidbClusterChartPath(from.OperatorTag), setString)
glog.Infof(cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down
8 changes: 2 additions & 6 deletions tests/cmd/e2e/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,10 @@ func main() {
logs.InitLogs()
defer logs.FlushLogs()

conf := tests.NewConfig()
err := conf.Parse()
if err != nil {
glog.Fatalf("failed to parse config: %v", err)
}
conf := tests.ParseConfigOrDie()
conf.ChartDir = "/charts"
aylei marked this conversation as resolved.
Show resolved Hide resolved

cli, kubeCli := client.NewCliOrDie()

oa := tests.NewOperatorActions(cli, kubeCli, conf)

operatorInfo := &tests.OperatorConfig{
Expand Down
23 changes: 19 additions & 4 deletions tests/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ type Config struct {

// For local test
OperatorRepoDir string `yaml:"operator_repo_dir" json:"operator_repo_dir"`
// chart dir
ChartDir string `yaml:"chart_dir" json:"chart_dir"`
}

// Nodes defines a series of nodes that belong to the same physical node.
Expand All @@ -34,22 +36,35 @@ type Nodes struct {
}

// NewConfig creates a new config.
func NewConfig() *Config {
func NewConfig() (*Config, error) {
cfg := &Config{}
flag.StringVar(&cfg.configFile, "config", "", "Config file")
flag.StringVar(&cfg.LogDir, "log-dir", "/logDir", "log directory")
flag.IntVar(&cfg.FaultTriggerPort, "fault-trigger-port", 23332, "the http port of fault trigger service")
flag.StringVar(&cfg.TidbVersions, "tidb-versions", "v2.1.3,v2.1.4", "tidb versions")
flag.StringVar(&cfg.OperatorTag, "operator-tag", "master", "operator tag used to choose charts")
flag.StringVar(&cfg.OperatorImage, "operator-image", "pingcap/tidb-operator:latest", "operator image")
flag.StringVar(&cfg.OperatorRepoDir, "operator-repo-dir", "/tidb-operator", "local directory to which tidb-operator cloned")
flag.Parse()

return cfg
operatorRepo, err := ioutil.TempDir("", "tidb-operator")
if err != nil {
return nil, err
}
cfg.OperatorRepoDir = operatorRepo

chartDir, err := ioutil.TempDir("", "charts")
if err != nil {
return nil, err
}
cfg.ChartDir = chartDir
return cfg, nil
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where is the place to clean these temporary directories?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

}

func ParseConfigOrDie() *Config {
cfg := NewConfig()
cfg, err := NewConfig()
if err != nil {
panic(err)
}
if err := cfg.Parse(); err != nil {
panic(err)
}
Expand Down
3 changes: 0 additions & 3 deletions tests/fault.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,9 +361,6 @@ func getFaultNode(kubeCli kubernetes.Interface) (string, error) {
}

myNode := getMyNodeName()
if myNode == "" {
return "", fmt.Errorf("get own node name is empty")
}

index := rand.Intn(len(nodes.Items))
faultNode := nodes.Items[index].Name
Expand Down