From 5fa21ca807ba758572e190ee8a049bda578a6ddb Mon Sep 17 00:00:00 2001 From: Xuecheng Zhang Date: Wed, 23 Sep 2020 10:08:36 +0800 Subject: [PATCH] *: run chaos-mesh operations with GitHub action (#930) --- .github/workflows/chaos-mesh.yml | 265 +++++++++++++++++++++ Makefile | 5 +- chaos/cases/conf/source1.yaml | 7 + chaos/cases/conf/source2.yaml | 7 + chaos/cases/conf/task-single.yaml | 21 ++ chaos/cases/config.go | 90 ++++++++ chaos/cases/db.go | 139 +++++++++++ chaos/cases/diff.go | 93 ++++++++ chaos/cases/dml.go | 43 ++++ chaos/cases/main.go | 124 ++++++++++ chaos/cases/member.go | 87 +++++++ chaos/cases/schema.go | 61 +++++ chaos/cases/single.go | 343 ++++++++++++++++++++++++++++ chaos/cases/source.go | 73 ++++++ chaos/manifests/Dockerfile | 18 ++ chaos/manifests/cases.yaml | 29 +++ chaos/manifests/dm-master.yaml | 74 ++++++ chaos/manifests/dm-worker.yaml | 74 ++++++ chaos/manifests/mysql.yaml | 59 +++++ chaos/manifests/pod-failure-dm.yaml | 21 ++ chaos/manifests/pod-kill-dm.yaml | 20 ++ chaos/manifests/tidb.yaml | 52 +++++ go.mod | 8 +- go.sum | 18 +- 24 files changed, 1723 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/chaos-mesh.yml create mode 100644 chaos/cases/conf/source1.yaml create mode 100644 chaos/cases/conf/source2.yaml create mode 100644 chaos/cases/conf/task-single.yaml create mode 100644 chaos/cases/config.go create mode 100644 chaos/cases/db.go create mode 100644 chaos/cases/diff.go create mode 100644 chaos/cases/dml.go create mode 100644 chaos/cases/main.go create mode 100644 chaos/cases/member.go create mode 100644 chaos/cases/schema.go create mode 100644 chaos/cases/single.go create mode 100644 chaos/cases/source.go create mode 100644 chaos/manifests/Dockerfile create mode 100644 chaos/manifests/cases.yaml create mode 100644 chaos/manifests/dm-master.yaml create mode 100644 chaos/manifests/dm-worker.yaml create mode 100644 chaos/manifests/mysql.yaml create mode 100644 chaos/manifests/pod-failure-dm.yaml create mode 100644 chaos/manifests/pod-kill-dm.yaml create mode 100644 chaos/manifests/tidb.yaml diff --git a/.github/workflows/chaos-mesh.yml b/.github/workflows/chaos-mesh.yml new file mode 100644 index 0000000000..45ebc45f30 --- /dev/null +++ b/.github/workflows/chaos-mesh.yml @@ -0,0 +1,265 @@ +name: chaos + +# Controls when the action will run. Triggers the workflow on pull request +# events but only for the master and release-2.0 branch +on: + pull_request: + branches: + - master + - release-2.0 + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "base" + base: + # The type of runner that the job will run on + runs-on: ubuntu-18.04 + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Set up Go for building DM, now it's v1.13 + - name: Set up Go 1.13 + uses: actions/setup-go@v2 + with: + go-version: 1.13 + - name: Print Go version + run: go version + + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Check out code + uses: actions/checkout@v2 + + # Set up Kubernetes IN Docker + # - name: Set up kind cluster + # uses: helm/kind-action@v1.0.0 + # with: + # cluster_name: dm-chaos + # Set up Kubernetes with K3s + - name: Set up K3s cluster + run: | + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.18.9+k3s1 sh -s - \ + --write-kubeconfig-mode=644 \ + "${k3s_disable_command:---disable}" metrics-server \ + "${k3s_disable_command:---disable}" traefik \ + --flannel-backend=none \ + --docker + shell: bash + - name: Wait for coredns + run: | + kubectl rollout status --watch --timeout 300s deployment/coredns -n kube-system + shell: bash + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + - name: Export KUBECONFIG environment variable + run: echo "::set-env name=KUBECONFIG::/etc/rancher/k3s/k3s.yaml" + shell: bash + - name: Print cluster information + run: | + kubectl config view + kubectl cluster-info + kubectl get nodes + kubectl get pods -n kube-system + kubectl get sc + kubectl version + + # Disable AppArmor for MySQL, see https://github.com/moby/moby/issues/7512#issuecomment-61787845 + - name: Disable AppArmor for MySQL + run: | + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + - name: Build DM binary + run: make dm-master dm-worker dmctl chaos-case + + # NOTE: we also copy config files into `bin` directory, + # so we only need to send `bin` as the context into docker daemon when building image. + - name: Build DM docker image + run: | + cp $GITHUB_WORKSPACE/chaos/cases/conf/source1.yaml $GITHUB_WORKSPACE/bin/source1.yaml + cp $GITHUB_WORKSPACE/chaos/cases/conf/source2.yaml $GITHUB_WORKSPACE/bin/source2.yaml + cp $GITHUB_WORKSPACE/chaos/cases/conf/task-single.yaml $GITHUB_WORKSPACE/bin/task-single.yaml + docker build -f $GITHUB_WORKSPACE/chaos/manifests/Dockerfile -t dm:chaos $GITHUB_WORKSPACE/bin + docker image list + # Load DM docker image into KIND, see https://kind.sigs.k8s.io/docs/user/quick-start/#loading-an-image-into-your-cluster + # - name: Load DM docker image into KIND + # run: | + # kind load docker-image dm:chaos --name dm-chaos + + # Set up upstream MySQL instances + - name: Set up MySQL + run: | + kubectl apply -f $GITHUB_WORKSPACE/chaos/manifests/mysql.yaml + kubectl get -f $GITHUB_WORKSPACE/chaos/manifests/mysql.yaml + kubectl describe -f $GITHUB_WORKSPACE/chaos/manifests/mysql.yaml + - name: Wait for MySQL ready # kubectl wait --all not working + run: | + kubectl wait --for=condition=Ready pod/mysql-0 --timeout=120s || true + sleep 10 + kubectl wait --for=condition=Ready pod/mysql-1 --timeout=120s || true + echo show pvc + kubectl get pvc -l app=mysql -o wide + echo show pv + kubectl get pv -o wide + echo show svc + kubectl get svc -l app=mysql -o wide + echo show sts + kubectl get sts -l app=mysql -o wide + echo show po + kubectl get po -l app=mysql -o wide + echo describe po + kubectl describe po -l app=mysql + echo describe pvc + kubectl describe pvc -l app=mysql + kubectl wait --for=condition=Ready pod/mysql-0 --timeout=0s + kubectl wait --for=condition=Ready pod/mysql-1 --timeout=0s + + # Set up downstream TiDB instance (deploy a TiDB with mockTiKV, not a TidbCluster managed by TiDB-operator) + - name: Set up TiDB + run: | + kubectl apply -f $GITHUB_WORKSPACE/chaos/manifests/tidb.yaml + kubectl get -f $GITHUB_WORKSPACE/chaos/manifests/tidb.yaml + kubectl describe -f $GITHUB_WORKSPACE/chaos/manifests/tidb.yaml + - name: Wait for TiDB ready + run: | + kubectl wait --for=condition=Ready pod/tidb-0 --timeout=120s || true + echo show pvc + kubectl get pvc -l app=tidb -o wide + echo show pv + kubectl get pv -o wide + echo show svc + kubectl get svc -l app=tidb -o wide + echo show sts + kubectl get sts -l app=tidb -o wide + echo show po + kubectl get po -l app=tidb -o wide + echo describe po + kubectl describe po -l app=tidb + echo describe pvc + kubectl describe pvc -l app=tidb + kubectl wait --for=condition=Ready pod/tidb-0 --timeout=0s + + - name: Set up DM-master + run: | + kubectl apply -f $GITHUB_WORKSPACE/chaos/manifests/dm-master.yaml + kubectl get -f $GITHUB_WORKSPACE/chaos/manifests/dm-master.yaml + kubectl describe -f $GITHUB_WORKSPACE/chaos/manifests/dm-master.yaml + # NOTE: even some DM-master instances are not ready, we still continue and let chaos test cases to check again. + - name: Wait for DM-master ready + run: | + sleep 10 + kubectl wait --for=condition=Ready pod -l app=dm-master --all --timeout=120s || true + echo "<<<<< show pvc >>>>>" + kubectl get pvc -l app=dm-master -o wide + echo "<<<<< show pv >>>>>" + kubectl get pv -o wide + echo "<<<<< show svc >>>>>" + kubectl get svc -l app=dm-master -o wide + echo "<<<<< show sts >>>>>" + kubectl get sts -l app=dm-master -o wide + echo "<<<<< show po >>>>>" + kubectl get po -l app=dm-master -o wide + echo "<<<<< describe po >>>>>" + kubectl describe po -l app=dm-master + echo "<<<<< describe pvc >>>>>" + kubectl describe pvc -l app=dm-master + echo "<<<<< show current log for dm-master-0 >>>>>" + kubectl logs dm-master-0 || true + echo "<<<<< show previous log for dm-master-0 >>>>>" + kubectl logs dm-master-0 -p || true + echo "<<<<< show current log for dm-master-1 >>>>>" + kubectl logs dm-master-1 || true + echo "<<<<< show previous log for dm-master-1 >>>>>" + kubectl logs dm-master-1 -p || true + echo "<<<<< show current log for dm-master-2 >>>>>" + kubectl logs dm-master-2 || true + echo "<<<<< show previous log for dm-master-2 >>>>>" + kubectl logs dm-master-2 -p || true + + - name: Set up DM-worker + run: | + kubectl apply -f $GITHUB_WORKSPACE/chaos/manifests/dm-worker.yaml + kubectl get -f $GITHUB_WORKSPACE/chaos/manifests/dm-worker.yaml + kubectl describe -f $GITHUB_WORKSPACE/chaos/manifests/dm-worker.yaml + # NOTE: even some DM-worker instances are not ready, we still continue and let chaos test cases to check again. + - name: Wait for DM-worker ready + run: | + sleep 10 + kubectl wait --for=condition=Ready pod -l app=dm-worker --all --timeout=120s || true + echo "<<<<< show pvc >>>>>" + kubectl get pvc -l app=dm-worker -o wide + echo "<<<<< show pv >>>>>" + kubectl get pv -o wide + echo "<<<<< show svc >>>>>" + kubectl get svc -l app=dm-worker -o wide + echo "<<<<< show sts >>>>>" + kubectl get sts -l app=dm-worker -o wide + echo "<<<<< show po >>>>>" + kubectl get po -l app=dm-worker -o wide + echo "<<<<< describe po >>>>>" + kubectl describe po -l app=dm-worker + echo "<<<<< describe pvc >>>>>" + kubectl describe pvc -l app=dm-worker + echo "<<<<< show current log for dm-worker-0 >>>>>" + kubectl logs dm-worker-0 || true + echo "<<<<< show previous log for dm-worker-0 >>>>>" + kubectl logs dm-worker-0 -p || true + echo "<<<<< show current log for dm-worker-1 >>>>>" + kubectl logs dm-worker-1 || true + echo "<<<<< show previous log for worker-master-1 >>>>>" + kubectl logs dm-worker-1 -p || true + echo "<<<<< show current log for dm-worker-2 >>>>>" + kubectl logs dm-worker-2 || true + echo "<<<<< show previous log for dm-worker-2 >>>>>" + kubectl logs dm-worker-2 -p || true + + # NOTE: we sleep a while when check members ready in cases before applying any chaos operations. + - name: Set up chaos test cases + run: | + kubectl apply -f $GITHUB_WORKSPACE/chaos/manifests/cases.yaml + kubectl get -f $GITHUB_WORKSPACE/chaos/manifests/cases.yaml + kubectl describe -f $GITHUB_WORKSPACE/chaos/manifests/cases.yaml + sleep 60 + + - name: Encode chaos-mesh action + run: | + echo "::set-env name=CFG_BASE64::$(base64 -w 0 $GITHUB_WORKSPACE/chaos/manifests/pod-failure-dm.yaml)" + + - name: Run chaos mesh action + uses: chaos-mesh/chaos-mesh-action@master + env: + CFG_BASE64: ${{ env.CFG_BASE64 }} + + # TODO: fail ASAP for test cases. + - name: Wait for chaos test case complete + run: | + kubectl wait --for=condition=complete job/chaos-test-case --timeout=21m + + - name: Copy logs to hack permission + if: ${{ always() }} + run: | + mkdir ./logs + sudo cp -r -L /var/log/containers/. ./logs + sudo chown -R runner ./logs + - name: Upload logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: chaos-base-logs + path: | + ./logs + !./logs/coredns-* + !./logs/local-path-provisioner-* + + # send Slack notify if failed. + - name: Slack notification + if: ${{ failure() }} + uses: Ilshidur/action-slack@2.1.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFY }} + with: + args: "chaos job failed, see https://github.com/pingcap/dm/actions/runs/{{ GITHUB_RUN_ID }}" + + # Debug via SSH if previous steps failed + - name: Set up tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v2 diff --git a/Makefile b/Makefile index 875a11cdb2..066e37225c 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ else endif .PHONY: build retool_setup test unit_test dm_integration_test_build integration_test \ - coverage check dm-worker dm-master dm-tracer dmctl debug-tools + coverage check dm-worker dm-master dm-tracer chaos-case dmctl debug-tools build: check dm-worker dm-master dm-tracer dmctl dm-portal dm-syncer @@ -62,6 +62,9 @@ dm-master: dm-tracer: $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/dm-tracer ./cmd/dm-tracer +chaos-case: + $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/chaos-case ./chaos/cases + dmctl: $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/dmctl ./cmd/dm-ctl diff --git a/chaos/cases/conf/source1.yaml b/chaos/cases/conf/source1.yaml new file mode 100644 index 0000000000..686897f149 --- /dev/null +++ b/chaos/cases/conf/source1.yaml @@ -0,0 +1,7 @@ +source-id: "mysql-replica-01" + +from: + host: "mysql-0.mysql" # same namespace with MySQL + user: "root" + password: "" + port: 3306 diff --git a/chaos/cases/conf/source2.yaml b/chaos/cases/conf/source2.yaml new file mode 100644 index 0000000000..b9c01ef575 --- /dev/null +++ b/chaos/cases/conf/source2.yaml @@ -0,0 +1,7 @@ +source-id: "mysql-replica-02" + +from: + host: "mysql-1.mysql" # same namespace with MySQL + user: "root" + password: "" + port: 3306 diff --git a/chaos/cases/conf/task-single.yaml b/chaos/cases/conf/task-single.yaml new file mode 100644 index 0000000000..2ba5cdc1c2 --- /dev/null +++ b/chaos/cases/conf/task-single.yaml @@ -0,0 +1,21 @@ +--- +name: "task_single" +task-mode: all + +target-database: + host: "tidb-0.tidb" + port: 4000 + user: "root" + password: "" + +mysql-instances: + - + source-id: "mysql-replica-01" + black-white-list: "instance" + mydumper-thread: 4 + loader-thread: 16 + syncer-thread: 16 + +black-white-list: + instance: + do-dbs: ["db_single"] diff --git a/chaos/cases/config.go b/chaos/cases/config.go new file mode 100644 index 0000000000..c807b54979 --- /dev/null +++ b/chaos/cases/config.go @@ -0,0 +1,90 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "time" + + config2 "github.com/pingcap/dm/dm/config" +) + +// config is used to run chaos tests. +type config struct { + *flag.FlagSet `toml:"-" yaml:"-" json:"-"` + + ConfigDir string `toml:"config-dir" yaml:"config-dir" json:"config-dir"` + MasterAddr string `toml:"master-addr" yaml:"master-addr" json:"master-addr"` + Duration time.Duration `toml:"duration" yaml:"duration" json:"duration"` + + MasterCount int `toml:"master-count" yaml:"master-count" json:"master-count"` + WorkerCount int `toml:"worker-count" yaml:"worker-count" json:"worker-count"` + + Source1 config2.DBConfig `toml:"source-1" yaml:"source-1" json:"source-1"` + Source2 config2.DBConfig `toml:"source-2" yaml:"source-2" json:"source-2"` + Target config2.DBConfig `toml:"target" yaml:"target" json:"target"` +} + +// newConfig creates a config for this chaos testing suite. +func newConfig() *config { + cfg := &config{} + cfg.FlagSet = flag.NewFlagSet("chaos-case", flag.ContinueOnError) + fs := cfg.FlagSet + + fs.StringVar(&cfg.ConfigDir, "config-dir", "/", "path of the source and task config files") + fs.StringVar(&cfg.MasterAddr, "master-addr", "dm-master:8261", "address of dm-master") + fs.DurationVar(&cfg.Duration, "duration", 20*time.Minute, "duration of cases running") + + fs.IntVar(&cfg.MasterCount, "master-count", 3, "expect count of dm-master") + fs.IntVar(&cfg.WorkerCount, "worker-count", 3, "expect count of dm-worker") + + fs.StringVar(&cfg.Source1.Host, "source1.host", "mysql-0.mysql", "host of the first upstream source") + fs.IntVar(&cfg.Source1.Port, "source1.port", 3306, "port of the first upstream source") + fs.StringVar(&cfg.Source1.User, "source1.user", "root", "user of the first upstream source") + fs.StringVar(&cfg.Source1.Password, "source1.password", "", "password of the first upstream source") + + fs.StringVar(&cfg.Source2.Host, "source2.host", "mysql-1.mysql", "host of the second upstream source") + fs.IntVar(&cfg.Source2.Port, "source2.port", 3306, "port of the second upstream source") + fs.StringVar(&cfg.Source2.User, "source2.user", "root", "user of the second upstream source") + fs.StringVar(&cfg.Source2.Password, "source2.password", "", "password of the second upstream source") + + fs.StringVar(&cfg.Target.Host, "target.host", "tidb-0.tidb", "host of the downstream target") + fs.IntVar(&cfg.Target.Port, "target.port", 4000, "port of the downstream target") + fs.StringVar(&cfg.Target.User, "target.user", "root", "user of the downstream target") + fs.StringVar(&cfg.Target.Password, "target.password", "", "password of the downstream target") + + return cfg +} + +// parse parses flag definitions from the argument list. +func (c *config) parse(args []string) error { + err := c.FlagSet.Parse(args) + if err != nil { + return err + } + + c.adjust() + return nil +} + +func (c *config) adjust() { + // `go-sqlsmith` may generate ZERO time data, so we simply clear `sql_mode` now. + c.Source1.Session = map[string]string{"sql_mode": ""} + c.Source2.Session = map[string]string{"sql_mode": ""} + c.Target.Session = map[string]string{"sql_mode": ""} + + c.Source1.Adjust() + c.Source2.Adjust() + c.Target.Adjust() +} diff --git a/chaos/cases/db.go b/chaos/cases/db.go new file mode 100644 index 0000000000..db6656b1d2 --- /dev/null +++ b/chaos/cases/db.go @@ -0,0 +1,139 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/pingcap/errors" + "github.com/pingcap/tidb-tools/pkg/dbutil" + "github.com/pingcap/tidb/errno" + + "github.com/pingcap/dm/pkg/conn" + tcontext "github.com/pingcap/dm/pkg/context" + "github.com/pingcap/dm/pkg/log" + "github.com/pingcap/dm/pkg/retry" +) + +// dbConn holds a connection to a database and supports to reset the connection. +type dbConn struct { + baseConn *conn.BaseConn + currDB string // current database (will `USE` it when reseting the connection). + resetFunc func(ctx context.Context, baseConn *conn.BaseConn) (*conn.BaseConn, error) +} + +// createDBConn creates a dbConn instance. +func createDBConn(ctx context.Context, db *conn.BaseDB, currDB string) (*dbConn, error) { + c, err := db.GetBaseConn(ctx) + if err != nil { + return nil, err + } + + return &dbConn{ + baseConn: c, + currDB: currDB, + resetFunc: func(ctx context.Context, baseConn *conn.BaseConn) (*conn.BaseConn, error) { + db.CloseBaseConn(baseConn) + return db.GetBaseConn(ctx) + }, + }, nil +} + +// resetConn resets the underlying connection. +func (c *dbConn) resetConn(ctx context.Context) error { + baseConn, err := c.resetFunc(ctx, c.baseConn) + if err != nil { + return err + } + + _, err = baseConn.ExecuteSQL(tcontext.NewContext(ctx, log.L()), nil, "chaos-cases", []string{fmt.Sprintf("USE %s", c.currDB)}) + if err != nil { + return err + } + + c.baseConn = baseConn + return nil +} + +// execSQLs executes DDL or DML queries. +func (c *dbConn) execSQLs(ctx context.Context, queries ...string) error { + params := retry.Params{ + RetryCount: 3, + FirstRetryDuration: time.Second, + BackoffStrategy: retry.Stable, + IsRetryableFn: func(retryTime int, err error) bool { + if retry.IsConnectionError(err) || forceIgnoreExecSQLError(err) { + // HACK: for some errors like `invalid connection`, `sql: connection is already closed`, we can ignore them just for testing. + err = c.resetConn(ctx) + if err != nil { + return false + } + return true + } + return false + }, + } + + _, _, err := c.baseConn.ApplyRetryStrategy(tcontext.NewContext(ctx, log.L()), params, + func(tctx *tcontext.Context) (interface{}, error) { + ret, err2 := c.baseConn.ExecuteSQLWithIgnoreError(tctx, nil, "chaos-cases", ignoreExecSQLError, queries) + return ret, err2 + }) + return err +} + +// dropDatabase drops the database if exists. +func dropDatabase(ctx context.Context, conn2 *dbConn, name string) error { + query := fmt.Sprintf("DROP DATABASE IF EXISTS %s", dbutil.ColumnName(name)) + return conn2.execSQLs(ctx, query) +} + +// createDatabase creates a database if not exists. +func createDatabase(ctx context.Context, conn2 *dbConn, name string) error { + query := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", dbutil.ColumnName(name)) + return conn2.execSQLs(ctx, query) +} + +func ignoreExecSQLError(err error) bool { + err = errors.Cause(err) // check the original error + mysqlErr, ok := err.(*mysql.MySQLError) + if !ok { + return false + } + + switch mysqlErr.Number { + case errno.ErrParse: // HACK: the query generated by `go-sqlsmith` may be invalid, so we just ignore them. + return true + case errno.ErrDupEntry: // HACK: we tolerate `invalid connection`, then `Duplicate entry` may be reported. + return true + default: + return false + } +} + +// forceIgnoreExecSQLError returns true for some errors which can be ignored ONLY in these tests. +func forceIgnoreExecSQLError(err error) bool { + err = errors.Cause(err) + switch err { + case mysql.ErrInvalidConn: + return true + case sql.ErrConnDone: + return true + } + return false +} diff --git a/chaos/cases/diff.go b/chaos/cases/diff.go new file mode 100644 index 0000000000..ace2b16549 --- /dev/null +++ b/chaos/cases/diff.go @@ -0,0 +1,93 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/tidb-tools/pkg/dbutil" + "github.com/pingcap/tidb-tools/pkg/diff" + "go.uber.org/zap" + + "github.com/pingcap/dm/pkg/log" +) + +// diffDataLoop checks whether target has the same data with source via `sync-diff-inspector` multiple times. +func diffDataLoop(ctx context.Context, count int, interval time.Duration, schema string, tables []string, targetDB *sql.DB, sourceDBs ...*sql.DB) (err error) { + for i := 0; i < count; i++ { + select { + case <-ctx.Done(): + return nil + case <-time.After(interval): + err = diffData(ctx, schema, tables, targetDB, sourceDBs...) + if err == nil { + return nil + } + } + } + return err +} + +// diffData checks whether target has the same data with source via `sync-diff-inspector`. +func diffData(ctx context.Context, schema string, tables []string, targetDB *sql.DB, sourceDBs ...*sql.DB) error { + for _, table := range tables { + sourceTables := make([]*diff.TableInstance, 0, len(sourceDBs)) + for i, sourceDB := range sourceDBs { + sourceTables = append(sourceTables, &diff.TableInstance{ + Conn: sourceDB, + Schema: schema, + Table: table, + InstanceID: fmt.Sprintf("source-%d", i), + }) + } + + targetTable := &diff.TableInstance{ + Conn: targetDB, + Schema: schema, + Table: table, + InstanceID: "target", + } + + td := &diff.TableDiff{ + SourceTables: sourceTables, + TargetTable: targetTable, + ChunkSize: 1000, + Sample: 100, + CheckThreadCount: 1, + UseChecksum: true, + TiDBStatsSource: targetTable, + CpDB: targetDB, + } + + structEqual, dataEqual, err := td.Equal(ctx, func(dml string) error { + return nil + }) + + if errors.Cause(err) == context.Canceled || errors.Cause(err) == context.DeadlineExceeded { + return nil + } + if !structEqual { + return fmt.Errorf("different struct for table %s", dbutil.TableName(schema, table)) + } else if !dataEqual { + return fmt.Errorf("different data for table %s", dbutil.TableName(schema, table)) + } + log.L().Info("data equal for table", zap.String("schema", schema), zap.String("table", table)) + } + + return nil +} diff --git a/chaos/cases/dml.go b/chaos/cases/dml.go new file mode 100644 index 0000000000..8cab0592fb --- /dev/null +++ b/chaos/cases/dml.go @@ -0,0 +1,43 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "math/rand" + + "github.com/chaos-mesh/go-sqlsmith" +) + +type dmlType int + +const ( + insertDML dmlType = iota + 1 + updateDML + deleteDML +) + +// randDML generates DML (INSERT, UPDATE or DELETE). +// NOTE: 3 DML types have the same weight now. +func randDML(ss *sqlsmith.SQLSmith) (dml string, t dmlType, err error) { + t = dmlType(rand.Intn(3) + 1) + switch t { + case insertDML: + dml, _, err = ss.InsertStmt(false) + case updateDML: + dml, _, err = ss.UpdateStmt() + case deleteDML: + dml, _, err = ss.DeleteStmt() + } + return +} diff --git a/chaos/cases/main.go b/chaos/cases/main.go new file mode 100644 index 0000000000..f3173f81d9 --- /dev/null +++ b/chaos/cases/main.go @@ -0,0 +1,124 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "flag" + "fmt" + "math/rand" + "net/http" + _ "net/http/pprof" + "os" + "os/signal" + "syscall" + "time" + + "github.com/pingcap/errors" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + + "github.com/pingcap/dm/dm/pb" + "github.com/pingcap/dm/pkg/log" + "github.com/pingcap/dm/pkg/utils" +) + +// main starts to run the test case logic after MySQL, TiDB and DM have been set up. +// NOTE: run this in the same K8s namespace as DM-master. +func main() { + cfg := newConfig() + err := cfg.parse(os.Args[1:]) + switch errors.Cause(err) { + case nil: + case flag.ErrHelp: + os.Exit(0) + default: + fmt.Println("parse cmd flags err:", err.Error()) + os.Exit(2) + } + + err = log.InitLogger(&log.Config{Level: "info"}) + if err != nil { + fmt.Println("init logger error:", err.Error()) + os.Exit(2) + } + + go func() { + http.ListenAndServe("0.0.0.0:8899", nil) // for pprof + }() + + rand.Seed(time.Now().UnixNano()) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sc := make(chan os.Signal, 1) + signal.Notify(sc, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT) + go func() { + sig := <-sc + log.L().Info("got signal to exit", zap.Stringer("signal", sig)) + cancel() + }() + + ctx2, cancel2 := context.WithTimeout(ctx, 60*time.Second) + defer cancel2() + masterConn, err := grpc.DialContext(ctx2, cfg.MasterAddr, grpc.WithBlock(), grpc.WithInsecure()) // no TLS support in chaos cases now. + if err != nil { + log.L().Error("fail to dail DM-master", zap.String("address", cfg.MasterAddr), zap.Error(err)) + os.Exit(2) + } + masterCli := pb.NewMasterClient(masterConn) + + // check whether all members are ready. + err = checkMembersReadyLoop(ctx2, masterCli, cfg.MasterCount, cfg.WorkerCount) // ctx2, should be done in 60s. + if err != nil { + log.L().Error("fail to check members ready", zap.Error(err)) // only log a error, still try to do other things. + } + + // create two sources. + err = createSources(ctx, masterCli, cfg) + if err != nil { + log.L().Error("fail to create source", zap.Error(err)) + os.Exit(2) + } + + // context for the duration of running. + ctx3, cancel3 := context.WithTimeout(ctx, cfg.Duration) + defer cancel3() + + // run tests cases + var eg errgroup.Group + eg.Go(func() error { + st, err2 := newSingleTask(ctx3, masterCli, cfg.ConfigDir, cfg.Target, cfg.Source1) + if err2 != nil { + return err2 + } + err2 = st.run() + if utils.IsContextCanceledError(err2) { + err2 = nil // clear err + } + return err2 + }) + + err = eg.Wait() + if err != nil { + log.L().Error("run cases failed", zap.Error(err)) + os.Exit(2) + } +} diff --git a/chaos/cases/member.go b/chaos/cases/member.go new file mode 100644 index 0000000000..ae75a98718 --- /dev/null +++ b/chaos/cases/member.go @@ -0,0 +1,87 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "time" + + "github.com/pingcap/dm/dm/master/scheduler" + "github.com/pingcap/dm/dm/pb" +) + +const ( + checkMemberTimes = 5 + checkMemberInterval = 10 * time.Second +) + +// checkMembersReadyLoop checks whether all DM-master and DM-worker members have been ready. +// NOTE: in this chaos case, we ensure 3 DM-master and 3 DM-worker started. +func checkMembersReadyLoop(ctx context.Context, cli pb.MasterClient, masterCount, workerCount int) (err error) { + for i := 0; i < checkMemberTimes; i++ { + select { + case <-ctx.Done(): + return nil + case <-time.After(checkMemberInterval): + err = checkMembersReady(ctx, cli, masterCount, workerCount) + if err == nil { + return nil + } + } + } + return err +} + +func checkMembersReady(ctx context.Context, cli pb.MasterClient, masterCount, workerCount int) error { + resp, err := cli.ListMember(ctx, &pb.ListMemberRequest{}) + if err != nil { + return err + } else if !resp.Result { + return fmt.Errorf("fail to list member: %s", resp.Msg) + } + + var ( + hasLeader bool + allMasterAlive bool + allWorkerOnline bool + ) + + for _, m := range resp.Members { + if m.GetLeader() != nil { + hasLeader = true + } else if lm := m.GetMaster(); lm != nil { + var aliveCount int + for _, master := range lm.Masters { + if master.Alive { + aliveCount++ + } + } + allMasterAlive = aliveCount == masterCount + } else if lw := m.GetWorker(); lw != nil { + var onlineCount int + for _, worker := range lw.Workers { + if worker.Stage != string(scheduler.WorkerOffline) { + onlineCount++ + } + } + allWorkerOnline = onlineCount == workerCount + } + } + + if !hasLeader || !allMasterAlive || !allWorkerOnline { + return fmt.Errorf("not all members are ready: %s", resp.String()) + } + return nil +} diff --git a/chaos/cases/schema.go b/chaos/cases/schema.go new file mode 100644 index 0000000000..7028405880 --- /dev/null +++ b/chaos/cases/schema.go @@ -0,0 +1,61 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/pingcap/parser" + "github.com/pingcap/parser/ast" +) + +const ( + tableType = "BASE TABLE" +) + +// createTableToSmithSchema converts a `CREATE TABLE` statement to a schema representation needed by go-sqlsmith. +// for one column in `columns`: +// - columns[0]: DB name +// - columns[1]: table name +// - columns[2]: table type (only use `BASE TABLE` now) +// - columns[3]: column name +// - columns[4]: column type +// for `indexes`: +// - list of index name +func createTableToSmithSchema(dbName, sql string) (columns [][5]string, indexes []string, err error) { + parser2 := parser.New() // we should have clear `SQL_MODE. + stmt, err := parser2.ParseOneStmt(sql, "", "") + if err != nil { + return nil, nil, err + } + + createStmt := stmt.(*ast.CreateTableStmt) + for _, col := range createStmt.Cols { + columns = append(columns, [5]string{ + dbName, + createStmt.Table.Name.O, + tableType, + col.Name.Name.O, + col.Tp.String(), + }) + } + + for _, cons := range createStmt.Constraints { + switch cons.Tp { + case ast.ConstraintPrimaryKey, ast.ConstraintKey, ast.ConstraintIndex, ast.ConstraintUniq, ast.ConstraintUniqKey, ast.ConstraintUniqIndex: + indexes = append(indexes, cons.Name) + default: + continue + } + } + return +} diff --git a/chaos/cases/single.go b/chaos/cases/single.go new file mode 100644 index 0000000000..7f3c08ab53 --- /dev/null +++ b/chaos/cases/single.go @@ -0,0 +1,343 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/chaos-mesh/go-sqlsmith" + "github.com/pingcap/errors" + "go.uber.org/zap" + + config2 "github.com/pingcap/dm/dm/config" + "github.com/pingcap/dm/dm/pb" + "github.com/pingcap/dm/pkg/conn" + "github.com/pingcap/dm/pkg/log" +) + +const ( + singleDB = "db_single" // specified in `source1.yaml`. + singleTableCount = 10 // tables count for the table. + singleFullInsertCount = 100 // `INSERT INTO` count (not rows count) for tables in full stage. + singleDiffCount = 10 // diff data check count + singleDiffInterval = 10 * time.Second // diff data check interval + singleIncrRoundTime = 20 * time.Second // time to generate incremental data in one round +) + +// singleTask is a test case with only one upstream source. +type singleTask struct { + logger log.Logger + + ctx context.Context + cli pb.MasterClient + confDir string + ss *sqlsmith.SQLSmith + sourceDB *conn.BaseDB + targetDB *conn.BaseDB + sourceConn *dbConn + targetConn *dbConn + tables []string + taskCfg config2.TaskConfig + result *singleResult +} + +// singleResult holds the result of the single source task case. +type singleResult struct { + Insert int `json:"insert"` + Update int `json:"update"` + Delete int `json:"delete"` +} + +func (sr *singleResult) String() string { + data, err := json.Marshal(sr) + if err != nil { + return err.Error() + } + return string(data) +} + +// newSingleTask creates a new singleTask instance. +func newSingleTask(ctx context.Context, cli pb.MasterClient, confDir string, + targetCfg config2.DBConfig, sourcesCfg ...config2.DBConfig) (*singleTask, error) { + sourceDB, err := conn.DefaultDBProvider.Apply(sourcesCfg[0]) + if err != nil { + return nil, err + } + sourceConn, err := createDBConn(ctx, sourceDB, singleDB) + if err != nil { + return nil, err + } + + targetDB, err := conn.DefaultDBProvider.Apply(targetCfg) + if err != nil { + return nil, err + } + targetConn, err := createDBConn(ctx, targetDB, singleDB) + if err != nil { + return nil, err + } + + var taskCfg config2.TaskConfig + err = taskCfg.DecodeFile(filepath.Join(confDir, "task-single.yaml")) + if err != nil { + return nil, err + } + taskCfg.TargetDB = &targetCfg + + st := &singleTask{ + logger: log.L().WithFields(zap.String("case", "single-task")), + ctx: ctx, + cli: cli, + confDir: confDir, + ss: sqlsmith.New(), + sourceDB: sourceDB, + targetDB: targetDB, + sourceConn: sourceConn, + targetConn: targetConn, + taskCfg: taskCfg, + result: &singleResult{}, + } + st.ss.SetDB(singleDB) + return st, nil +} + +// run runs the case. +func (st *singleTask) run() error { + defer func() { + st.sourceDB.Close() + st.targetDB.Close() + + st.logger.Info("single task run result", zap.Stringer("result", st.result)) + }() + + if err := st.stopPreviousTask(); err != nil { + return err + } + if err := st.clearPreviousData(); err != nil { + return err + } + + if err := st.genFullData(); err != nil { + return err + } + + if err := st.createTask(); err != nil { + return err + } + + st.logger.Info("check data for full stage") + if err := diffDataLoop(st.ctx, singleDiffCount, singleDiffInterval, singleDB, st.tables, st.targetDB.DB, st.sourceDB.DB); err != nil { + return err + } + + if err := st.incrLoop(); err != nil { + return err + } + + return nil +} + +// stopPreviousTask stops the previous task with the same name if exists. +func (st *singleTask) stopPreviousTask() error { + st.logger.Info("stopping previous task") + resp, err := st.cli.OperateTask(st.ctx, &pb.OperateTaskRequest{ + Op: pb.TaskOp_Stop, + Name: st.taskCfg.Name, + }) + if err != nil { + return err + } else if !resp.Result && !strings.Contains(resp.Msg, "not exist") { + return fmt.Errorf("fail to stop task: %s", resp.Msg) + } + return nil +} + +// clearPreviousData clears previous data in upstream source and downstream target. +func (st *singleTask) clearPreviousData() error { + st.logger.Info("clearing previous source and target data") + if err := dropDatabase(st.ctx, st.sourceConn, singleDB); err != nil { + return err + } + if err := dropDatabase(st.ctx, st.targetConn, singleDB); err != nil { + return err + } + return nil +} + +// genFullData generates data for the full stage. +func (st *singleTask) genFullData() error { + st.logger.Info("generating data for full stage") + if err := createDatabase(st.ctx, st.sourceConn, singleDB); err != nil { + return err + } + + // NOTE: we set CURRENT database here. + if err := st.sourceConn.execSQLs(st.ctx, fmt.Sprintf("USE %s", singleDB)); err != nil { + return err + } + + var ( + columns = make([][5]string, 0) + indexes = make(map[string][]string) + ) + + // generate `CREATE TABLE` statements. + for i := 0; i < singleTableCount; i++ { + query, name, err := st.ss.CreateTableStmt() + if err != nil { + return err + } + err = st.sourceConn.execSQLs(st.ctx, query) + st.tables = append(st.tables, name) + + col2, idx2, err := createTableToSmithSchema(singleDB, query) + if err != nil { + return err + } + columns = append(columns, col2...) + indexes[name] = idx2 + } + + // go-sqlsmith needs to load schema before generating DML and `ALTER TABLE` statements. + st.ss.LoadSchema(columns, indexes) + + for i := 0; i < singleFullInsertCount; i++ { + query, _, err := st.ss.InsertStmt(false) + if err != nil { + return err + } + err = st.sourceConn.execSQLs(st.ctx, query) + if err != nil { + return err + } + } + + return nil +} + +// createSingleTask creates a single source task. +func (st *singleTask) createTask() error { + st.logger.Info("starting task") + resp, err := st.cli.StartTask(st.ctx, &pb.StartTaskRequest{ + Task: st.taskCfg.String(), + }) + if err != nil { + return err + } else if !resp.Result && !strings.Contains(resp.Msg, "already exist") { // imprecise match + return fmt.Errorf("fail to start task: %s", resp.Msg) + } + return nil +} + +// incrLoop enters the loop of generating incremental data and diff them. +func (st *singleTask) incrLoop() error { + for { + select { + case <-st.ctx.Done(): + return nil + default: + ctx2, cancel2 := context.WithTimeout(st.ctx, singleIncrRoundTime) + // generate data + err := st.genIncrData(ctx2) + if err != nil { + cancel2() + return err + } + + // diff data + err = st.diffIncrData(ctx2) + if err != nil { + cancel2() + return err + } + cancel2() + } + } +} + +// genIncrData generates data for the incremental stage in one round. +// NOTE: it return nil for context done. +func (st *singleTask) genIncrData(ctx context.Context) (err error) { + st.logger.Info("generating data for incremental stage") + + defer func() { + if errors.Cause(err) == context.Canceled || errors.Cause(err) == context.DeadlineExceeded { + err = nil // clear error for context done. + } else if err != nil { + select { + case <-ctx.Done(): + st.logger.Warn("ignore error when generating data for incremental stage", zap.Error(err)) + err = nil // some other errors like `connection is already closed` may also be reported for context done. + default: + if forceIgnoreExecSQLError(err) { + st.logger.Warn("ignore error when generating data for incremental stage", zap.Error(err)) + st.sourceConn.resetConn(ctx) // reset connection for the next round. + err = nil + } + } + } + }() + + for { + select { + case <-ctx.Done(): + return nil + default: + } + query, t, err := randDML(st.ss) + if err != nil { + return err + } + err = st.sourceConn.execSQLs(ctx, query) + if err != nil { + return err + } + + switch t { + case insertDML: + st.result.Insert++ + case updateDML: + st.result.Update++ + case deleteDML: + st.result.Delete++ + default: + } + } +} + +// diffIncrData checks data equal for the incremental stage in one round. +// NOTE: it return nil for context done. +func (st *singleTask) diffIncrData(ctx context.Context) (err error) { + st.logger.Info("check data for incremental stage") + + defer func() { + if errors.Cause(err) == context.Canceled || errors.Cause(err) == context.DeadlineExceeded { + err = nil // clear error for context done. + } else if err != nil { + select { + case <-ctx.Done(): + st.logger.Warn("ignore error when check data for incremental stage", zap.Error(err)) + err = nil // some other errors like `connection is already closed` may also be reported for context done. + default: + } + } + }() + + return diffDataLoop(ctx, singleDiffCount, singleDiffInterval, singleDB, st.tables, st.targetDB.DB, st.sourceDB.DB) +} diff --git a/chaos/cases/source.go b/chaos/cases/source.go new file mode 100644 index 0000000000..64150d691f --- /dev/null +++ b/chaos/cases/source.go @@ -0,0 +1,73 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "io/ioutil" + "path/filepath" + "strings" + + config2 "github.com/pingcap/dm/dm/config" + "github.com/pingcap/dm/dm/pb" +) + +// createSources does `operate-source create` operation for two sources. +// NOTE: we put two source config files (`source1.yaml` and `source2.yaml`) in `conf` directory. +func createSources(ctx context.Context, cli pb.MasterClient, cfg *config) error { + s1Path := filepath.Join(cfg.ConfigDir, "source1.yaml") + s2Path := filepath.Join(cfg.ConfigDir, "source2.yaml") + + s1Content, err := ioutil.ReadFile(s1Path) + if err != nil { + return err + } + s2Content, err := ioutil.ReadFile(s2Path) + if err != nil { + return err + } + + var cfg1 config2.SourceConfig + var cfg2 config2.SourceConfig + if err = cfg1.ParseYaml(string(s1Content)); err != nil { + return err + } + if err = cfg2.ParseYaml(string(s2Content)); err != nil { + return err + } + + // replace DB config. + cfg1.From = cfg.Source1 + cfg2.From = cfg.Source2 + s1Content2, err := cfg1.Yaml() + if err != nil { + return err + } + s2Content2, err := cfg2.Yaml() + if err != nil { + return err + } + + resp, err := cli.OperateSource(ctx, &pb.OperateSourceRequest{ + Op: pb.SourceOp_StartSource, + Config: []string{s1Content2, s2Content2}, + }) + if err != nil { + return err + } else if !resp.Result && !strings.Contains(resp.Msg, "already exists") { // imprecise match + return fmt.Errorf("fail to create source: %s", resp.Msg) + } + return nil +} diff --git a/chaos/manifests/Dockerfile b/chaos/manifests/Dockerfile new file mode 100644 index 0000000000..b6eacd46cd --- /dev/null +++ b/chaos/manifests/Dockerfile @@ -0,0 +1,18 @@ +FROM alpine:3.10 +RUN apk add --no-cache tzdata +ADD dm-master /dm-master +ADD dm-worker /dm-worker +ADD dmctl /dmctl +ADD chaos-case /chaos-case +ADD source1.yaml /source1.yaml +ADD source2.yaml /source2.yaml +ADD task-single.yaml /task-single.yaml + +RUN chmod +x /dm-master +RUN chmod +x /dm-worker +RUN chmod +x /dmctl +RUN chmod +x /chaos-case + +WORKDIR / + +EXPOSE 8291 8261 8262 diff --git a/chaos/manifests/cases.yaml b/chaos/manifests/cases.yaml new file mode 100644 index 0000000000..298c885070 --- /dev/null +++ b/chaos/manifests/cases.yaml @@ -0,0 +1,29 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: chaos-test-case +spec: + template: + spec: + containers: + - name: chaos-test-case + image: dm:chaos # build this image in GitHub action workflow + imagePullPolicy: IfNotPresent + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 80 + name: chaos-test-case + command: + - "/chaos-case" + - "--config-dir=/" # we put config files into the root directory when building the docker image + - "--duration=20m" + restartPolicy: Never + backoffLimit: 0 # fail immediately diff --git a/chaos/manifests/dm-master.yaml b/chaos/manifests/dm-master.yaml new file mode 100644 index 0000000000..540ed5f7d6 --- /dev/null +++ b/chaos/manifests/dm-master.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: Service +metadata: + name: dm-master + labels: + app: dm-master +spec: + ports: + - name: dm-master + port: 8261 + targetPort: 8261 + - name: dm-master-peer + port: 8291 + targetPort: 8291 + selector: + app: dm-master +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: dm-master + labels: + app: dm-master +spec: + selector: + matchLabels: + app: dm-master + serviceName: dm-master + replicas: 3 # TODO: 1 for debug; 3 DM-master instances + podManagementPolicy: Parallel + template: + metadata: + labels: + app: dm-master + spec: + containers: + - name: dm-master + image: dm:chaos # build this image in GitHub action workflow + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /data + name: dm-master + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 8261 + name: dm-master + - containerPort: 8291 + name: dm-master-peer + command: + - "/dm-master" + - "--data-dir=/data" + - "--name=$(MY_POD_NAME)" + - "--master-addr=0.0.0.0:8261" + - "--advertise-addr=http://$(MY_POD_NAME).dm-master.$(MY_POD_NAMESPACE):8261" + - "--peer-urls=:8291" + - "--advertise-peer-urls=http://$(MY_POD_NAME).dm-master.$(MY_POD_NAMESPACE):8291" + - "--initial-cluster=dm-master-0=http://dm-master-0.dm-master.$(MY_POD_NAMESPACE):8291,dm-master-1=http://dm-master-1.dm-master.$(MY_POD_NAMESPACE):8291,dm-master-2=http://dm-master-2.dm-master.$(MY_POD_NAMESPACE):8291" + volumeClaimTemplates: + - metadata: + name: dm-master + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/chaos/manifests/dm-worker.yaml b/chaos/manifests/dm-worker.yaml new file mode 100644 index 0000000000..e465e69832 --- /dev/null +++ b/chaos/manifests/dm-worker.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: Service +metadata: + name: dm-worker + labels: + app: dm-worker +spec: + ports: + - name: dm-worker + port: 8262 + targetPort: 8262 + selector: + app: dm-worker +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: dm-worker + labels: + app: dm-worker +spec: + selector: + matchLabels: + app: dm-worker + serviceName: dm-worker + replicas: 3 # 3 DM-worker instances + podManagementPolicy: Parallel + template: + metadata: + labels: + app: dm-worker + spec: + containers: + - name: dm-worker + image: dm:chaos # build this image in GitHub action workflow + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /data + name: dm-worker + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 8262 + name: dm-worker + command: + - "/dm-worker" + - "--name=$(MY_POD_NAME)" + - "--worker-addr=0.0.0.0:8262" + - "--advertise-addr=$(MY_POD_NAME).dm-worker.$(MY_POD_NAMESPACE):8262" + - "--join=dm-master-0.dm-master.$(MY_POD_NAMESPACE):8261,dm-master-1.dm-master.$(MY_POD_NAMESPACE):8261,dm-master-2.dm-master.$(MY_POD_NAMESPACE):8261" + readinessProbe: + httpGet: + port: 8262 + path: /status + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 5 + volumeClaimTemplates: + - metadata: + name: dm-worker + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/chaos/manifests/mysql.yaml b/chaos/manifests/mysql.yaml new file mode 100644 index 0000000000..abf2a61760 --- /dev/null +++ b/chaos/manifests/mysql.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Service +metadata: + name: mysql + labels: + app: mysql +spec: + ports: + - name: mysql + port: 3306 + targetPort: 3306 + selector: + app: mysql +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mysql + labels: + app: mysql +spec: + selector: + matchLabels: + app: mysql + serviceName: mysql + replicas: 2 # 2 MySQL instances + podManagementPolicy: Parallel + template: + metadata: + labels: + app: mysql + spec: + containers: + - name: mysql + image: mysql:5.7 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: "/var/lib/mysql" + name: mysql + env: + - name: MYSQL_ALLOW_EMPTY_PASSWORD + value: "true" + ports: + - containerPort: 3306 + name: mysql + args: + - "--server-id=1" + - "--log-bin=/var/lib/mysql/mysql-bin" + - "--enforce-gtid-consistency=ON" + - "--gtid-mode=ON" + volumeClaimTemplates: + - metadata: + name: mysql + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi diff --git a/chaos/manifests/pod-failure-dm.yaml b/chaos/manifests/pod-failure-dm.yaml new file mode 100644 index 0000000000..d388597b1a --- /dev/null +++ b/chaos/manifests/pod-failure-dm.yaml @@ -0,0 +1,21 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: pod-failure-dm + labels: + app: pod-failure-dm +spec: + action: pod-failure + mode: one + duration: "30s" + selector: + pods: + default: # default namespace + - dm-master-0 + - dm-master-1 + - dm-master-2 + - dm-worker-0 + - dm-worker-1 + - dm-worker-2 + scheduler: + cron: "@every 2m" diff --git a/chaos/manifests/pod-kill-dm.yaml b/chaos/manifests/pod-kill-dm.yaml new file mode 100644 index 0000000000..efc52411b6 --- /dev/null +++ b/chaos/manifests/pod-kill-dm.yaml @@ -0,0 +1,20 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: pod-kill-dm + labels: + app: pod-kill-dm +spec: + action: pod-kill + mode: one + selector: + pods: + default: # default namespace + - dm-master-0 + - dm-master-1 + - dm-master-2 + - dm-worker-0 + - dm-worker-1 + - dm-worker-2 + scheduler: + cron: "@every 1m" diff --git a/chaos/manifests/tidb.yaml b/chaos/manifests/tidb.yaml new file mode 100644 index 0000000000..3914aece28 --- /dev/null +++ b/chaos/manifests/tidb.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Service +metadata: + name: tidb + labels: + app: tidb +spec: + ports: + - name: tidb + port: 4000 + targetPort: 4000 + selector: + app: tidb +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tidb + labels: + app: tidb +spec: + selector: + matchLabels: + app: tidb + serviceName: tidb + replicas: 1 # only 1 TiDB instance + template: + metadata: + labels: + app: tidb + spec: + containers: + - name: tidb + image: pingcap/tidb:latest # latest release version + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /data + name: tidb + ports: + - containerPort: 4000 + name: tidb + args: + - "--path=/data" + volumeClaimTemplates: + - metadata: + name: tidb + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi diff --git a/go.mod b/go.mod index 8c30323694..fafe3b2d30 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/DATA-DOG/go-sqlmock v1.4.1 github.com/Masterminds/semver v1.5.0 // indirect + github.com/chaos-mesh/go-sqlsmith v0.0.0-00010101000000-000000000000 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/docker/go-units v0.4.0 github.com/dustin/go-humanize v1.0.0 @@ -17,8 +18,6 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.14.3 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d github.com/lance6716/retool v1.3.8-0.20200806070832-3469f70b2afe - github.com/mattn/go-colorable v0.1.7 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/dumpling v0.0.0-20200909092728-a45daad655d1 github.com/pingcap/errcode v0.3.0 // indirect @@ -26,7 +25,7 @@ require ( github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 github.com/pingcap/parser v0.0.0-20200905070518-a6534dd22500 - github.com/pingcap/tidb v1.1.0-beta.0.20200901032733-f82e5320ad75 + github.com/pingcap/tidb v1.1.0-beta.0.20200904065405-82a37247485a github.com/pingcap/tidb-tools v4.0.5-0.20200828085514-03575b185007+incompatible github.com/prometheus/client_golang v1.5.1 github.com/rakyll/statik v0.1.6 @@ -45,6 +44,7 @@ require ( github.com/unrolled/render v1.0.1 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 go.uber.org/zap v1.16.0 + golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb @@ -54,3 +54,5 @@ require ( ) go 1.13 + +replace github.com/chaos-mesh/go-sqlsmith => github.com/csuzhangxc/go-sqlsmith v0.0.0-20200904072619-4a3341a30495 diff --git a/go.sum b/go.sum index fb4c05140a..234fae27f7 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/aws/aws-sdk-go v1.26.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.30.24 h1:y3JPD51VuEmVqN3BEDVm4amGpDma2cKJcDPuAU1OR58= github.com/aws/aws-sdk-go v1.30.24/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.32.4 h1:J2OMvipVB5dPIn+VH7L5rOqM4WoTsBxOqv+I06sjYOM= +github.com/aws/aws-sdk-go v1.32.4/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= @@ -115,6 +117,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/csuzhangxc/go-sqlsmith v0.0.0-20200904072619-4a3341a30495 h1:3fDcmfJW4cTO4S8tGxASkpWK1JciZgXbw+6CGnEVmJY= +github.com/csuzhangxc/go-sqlsmith v0.0.0-20200904072619-4a3341a30495/go.mod h1:zWquKoCivkUOWTXUN5ZLxzITMpPTZGNo4qKPiGzZKYc= github.com/cznic/golex v0.0.0-20181122101858-9c343928389c/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= @@ -338,6 +342,8 @@ github.com/juju/clock v0.0.0-20180524022203-d293bb356ca4/go.mod h1:nD0vlnrUjcjJh github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 h1:hJix6idebFclqlfZCHE7EUX7uqLCyb70nHNHH1XKGBg= +github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9 h1:Y+lzErDTURqeXqlqYi4YBYbDd7ycU74gW1ADt57/bgY= github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= @@ -436,6 +442,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/ncw/directio v1.0.4 h1:CojwI07mCEmRkajgx42Pf8jyCwTs1ji9/Ij9/PJG12k= github.com/ncw/directio v1.0.4/go.mod h1:CKGdcN7StAaqjT7Qack3lAXeX4pjnyc46YeqZH1yWVY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac h1:wyheT2lPXRQqYPWY2IVW5BTLrbqCsnhL61zK2R5goLA= +github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac/go.mod h1:ueVCjKQllPmX7uEvCYnZD5b8qjidGf1TCH61arVe4SU= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= @@ -581,8 +589,7 @@ github.com/pingcap/parser v0.0.0-20200623082809-b74301ac298b/go.mod h1:vQdbJqobJ github.com/pingcap/parser v0.0.0-20200730092557-34a468e9b774/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= github.com/pingcap/parser v0.0.0-20200731033026-84f62115187c/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= github.com/pingcap/parser v0.0.0-20200813083329-a4bff035d3e2/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= -github.com/pingcap/parser v0.0.0-20200821073936-cf85e80665c4 h1:ATFD3gmwkSHcPt5DuQK3dZwqDU49WXOq/zRmwPJ6Nks= -github.com/pingcap/parser v0.0.0-20200821073936-cf85e80665c4/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= +github.com/pingcap/parser v0.0.0-20200902143951-126c14c456eb/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= github.com/pingcap/parser v0.0.0-20200905070518-a6534dd22500 h1:dnXiRdcubiMSkzHBBSkJo+9E+OX3+Z629ZEQEIVfBsI= github.com/pingcap/parser v0.0.0-20200905070518-a6534dd22500/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2 h1:JTzYYukREvxVSKW/ncrzNjFitd8snoQ/Xz32pw8i+s8= @@ -612,8 +619,8 @@ github.com/pingcap/tidb v1.1.0-beta.0.20200805053026-cd3e5ed82671/go.mod h1:+r9t github.com/pingcap/tidb v1.1.0-beta.0.20200806060043-574540aa06ba/go.mod h1:NHcZH46dkYwDd2IWUJaLOB0m54j7v2P5WdS4FvPR81w= github.com/pingcap/tidb v1.1.0-beta.0.20200810064414-d81150394f9d/go.mod h1:vLYo4E7Q6kzKYTskhP2MHBsodmZIRRUU63qdiFjlULA= github.com/pingcap/tidb v1.1.0-beta.0.20200820085534-0d997f2b8b3c/go.mod h1:z7Hn1KY8Crt9cHhWtbGPKMBcjvmSJXIoOjO4rMk165w= -github.com/pingcap/tidb v1.1.0-beta.0.20200901032733-f82e5320ad75 h1:kV9usN6vnG5cMNNNADlNPm5TPifLYBKYbGOZvG31ldI= -github.com/pingcap/tidb v1.1.0-beta.0.20200901032733-f82e5320ad75/go.mod h1:VXxiC2f+HY3/5phR1841YJrX4on56kTEXrtEzRezcj4= +github.com/pingcap/tidb v1.1.0-beta.0.20200904065405-82a37247485a h1:hNVNBwAls3XJBEmMiI1nkFjCpdh0KpJPGR4l7cUdyq8= +github.com/pingcap/tidb v1.1.0-beta.0.20200904065405-82a37247485a/go.mod h1:Pr9AFmhc250ZXP7VgK/n6xd19Dr03hCEF/v13fu+TjM= github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= @@ -948,6 +955,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 h1:eDrdRpKgkcCqKZQwyZRyeFZgfqt37SL7Kv3tok06cKE= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -998,6 +1006,7 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200819171115-d785dc25833f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -1055,6 +1064,7 @@ golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1/go.mod h1:Sl4aGygMT6LrqrWc golang.org/x/tools v0.0.0-20200521211927-2b542361a4fc/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d h1:SR+e35rACZFBohNb4Om1ibX6N3iO0FtdbwqGSuD9dBU= golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200823205832-c024452afbcd h1:KNSumuk5eGuQV7zbOrDDZ3MIkwsQr0n5oKiH4oE0/hU= golang.org/x/tools v0.0.0-20200823205832-c024452afbcd/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=