diff --git a/components/dm/command/prune.go b/components/dm/command/prune.go index 96db6e8762..659ef246cd 100644 --- a/components/dm/command/prune.go +++ b/components/dm/command/prune.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/tiup/components/dm/spec" "github.com/pingcap/tiup/pkg/cluster/api" operator "github.com/pingcap/tiup/pkg/cluster/operation" + tidbspec "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/spf13/cobra" "go.uber.org/zap" ) @@ -63,7 +64,11 @@ func clearOutDatedEtcdInfo(clusterName string, metadata *spec.Metadata, opt oper existedWorkers[workerSpec.Name] = struct{}{} } - dmMasterClient := api.NewDMMasterClient(topo.GetMasterList(), 10*time.Second, nil) + tlsCfg, err := topo.TLSConfig(dmspec.Path(clusterName, tidbspec.TLSCertKeyDir)) + if err != nil { + return err + } + dmMasterClient := api.NewDMMasterClient(topo.GetMasterList(), 10*time.Second, tlsCfg) registeredMasters, registeredWorkers, err := dmMasterClient.GetRegisteredMembers() if err != nil { return err diff --git a/components/dm/command/scale_in.go b/components/dm/command/scale_in.go index 985f777aca..40f165fca2 100644 --- a/components/dm/command/scale_in.go +++ b/components/dm/command/scale_in.go @@ -46,7 +46,7 @@ func newScaleInCmd() *cobra.Command { b.Func( fmt.Sprintf("ScaleInCluster: options=%+v", gOpt), func(ctx context.Context) error { - return ScaleInDMCluster(ctx, metadata.Topology, gOpt) + return ScaleInDMCluster(ctx, metadata.Topology, gOpt, tlsCfg) }, ).Serial(dmtask.NewUpdateDMMeta(clusterName, metadata, gOpt.Nodes)) } @@ -68,6 +68,7 @@ func ScaleInDMCluster( ctx context.Context, topo *dm.Specification, options operator.Options, + tlsCfg *tls.Config, ) error { // instances by uuid instances := map[string]dm.Instance{} @@ -125,7 +126,7 @@ func ScaleInDMCluster( return errors.New("cannot find available dm-master instance") } - dmMasterClient = api.NewDMMasterClient(dmMasterEndpoint, 10*time.Second, nil) + dmMasterClient = api.NewDMMasterClient(dmMasterEndpoint, 10*time.Second, tlsCfg) noAgentHosts := set.NewStringSet() topo.IterInstance(func(inst dm.Instance) { diff --git a/components/dm/spec/logic.go b/components/dm/spec/logic.go index 11775a02ba..0566d3003f 100644 --- a/components/dm/spec/logic.go +++ b/components/dm/spec/logic.go @@ -119,6 +119,7 @@ func (i *MasterInstance) InitConfig( return err } + enableTLS := i.topo.GlobalOptions.TLSEnabled spec := i.InstanceSpec.(*MasterSpec) cfg := scripts.NewDMMasterScript( spec.Name, @@ -126,6 +127,7 @@ func (i *MasterInstance) InitConfig( paths.Deploy, paths.Data[0], paths.Log, + enableTLS, ).WithPort(spec.Port).WithNumaNode(spec.NumaNode).WithPeerPort(spec.PeerPort).AppendEndpoints(i.topo.Endpoints(deployUser)...).WithV1SourcePath(spec.V1SourcePath) fp := filepath.Join(paths.Cache, fmt.Sprintf("run_dm-master_%s_%d.sh", i.GetHost(), i.GetPort())) @@ -136,12 +138,12 @@ func (i *MasterInstance) InitConfig( if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil { return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } - // doesn't work - if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + if spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths); err != nil { return err } @@ -152,7 +154,40 @@ func (i *MasterInstance) InitConfig( // setTLSConfig set TLS Config to support enable/disable TLS // MasterInstance no need to configure TLS func (i *MasterInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { - return nil, nil + // set TLS configs + if enableTLS { + if configs == nil { + configs = make(map[string]interface{}) + } + configs["ssl-ca"] = fmt.Sprintf( + "%s/tls/%s", + paths.Deploy, + "ca.crt", + ) + configs["ssl-cert"] = fmt.Sprintf( + "%s/tls/%s.crt", + paths.Deploy, + i.Role()) + configs["ssl-key"] = fmt.Sprintf( + "%s/tls/%s.pem", + paths.Deploy, + i.Role()) + } else { + // dm-master tls config list + tlsConfigs := []string{ + "ssl-ca", + "ssl-cert", + "ssl-key", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } + } + + return configs, nil } // ScaleConfig deploy temporary config on scaling @@ -169,6 +204,7 @@ func (i *MasterInstance) ScaleConfig( return err } + enableTLS := i.topo.GlobalOptions.TLSEnabled c := topo.(*Specification) spec := i.InstanceSpec.(*MasterSpec) cfg := scripts.NewDMMasterScaleScript( @@ -177,6 +213,7 @@ func (i *MasterInstance) ScaleConfig( paths.Deploy, paths.Data[0], paths.Log, + enableTLS, ).WithPort(spec.Port).WithNumaNode(spec.NumaNode).WithPeerPort(spec.PeerPort).AppendEndpoints(c.Endpoints(deployUser)...) fp := filepath.Join(paths.Cache, fmt.Sprintf("run_dm-master_%s_%d.sh", i.GetHost(), i.GetPort())) @@ -261,6 +298,7 @@ func (i *WorkerInstance) InitConfig( return err } + enableTLS := i.topo.GlobalOptions.TLSEnabled spec := i.InstanceSpec.(*WorkerSpec) cfg := scripts.NewDMWorkerScript( i.Name, @@ -278,12 +316,12 @@ func (i *WorkerInstance) InitConfig( return err } - if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + _, _, err := e.Execute(ctx, "chmod +x "+dst, false) + if err != nil { return err } - // doesn't work - if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil { + if spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths); err != nil { return err } @@ -294,7 +332,40 @@ func (i *WorkerInstance) InitConfig( // setTLSConfig set TLS Config to support enable/disable TLS // workrsInstance no need to configure TLS func (i *WorkerInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) { - return nil, nil + // set TLS configs + if enableTLS { + if configs == nil { + configs = make(map[string]interface{}) + } + configs["ssl-ca"] = fmt.Sprintf( + "%s/tls/%s", + paths.Deploy, + "ca.crt", + ) + configs["ssl-cert"] = fmt.Sprintf( + "%s/tls/%s.crt", + paths.Deploy, + i.Role()) + configs["ssl-key"] = fmt.Sprintf( + "%s/tls/%s.pem", + paths.Deploy, + i.Role()) + } else { + // dm-worker tls config list + tlsConfigs := []string{ + "ssl-ca", + "ssl-cert", + "ssl-key", + } + // delete TLS configs + if configs != nil { + for _, config := range tlsConfigs { + delete(configs, config) + } + } + } + + return configs, nil } // ScaleConfig deploy temporary config on scaling @@ -411,7 +482,8 @@ func (topo *Specification) Endpoints(user string) []*scripts.DMMasterScript { s.Host, deployDir, dataDir, - logDir). + logDir, + topo.GlobalOptions.TLSEnabled). WithPort(s.Port). WithPeerPort(s.PeerPort) ends = append(ends, script) diff --git a/embed/templates/config/prometheus.yml.tpl b/embed/templates/config/prometheus.yml.tpl index 6e780f4cc5..68e31bfdfc 100644 --- a/embed/templates/config/prometheus.yml.tpl +++ b/embed/templates/config/prometheus.yml.tpl @@ -373,6 +373,14 @@ scrape_configs: {{- if .DMMasterAddrs}} - job_name: "dm_master" honor_labels: true # don't overwrite job & instance labels +{{- if .TLSEnabled}} + scheme: https + tls_config: + insecure_skip_verify: false + ca_file: ../tls/ca.crt + cert_file: ../tls/prometheus.crt + key_file: ../tls/prometheus.pem +{{- end}} static_configs: - targets: {{- range .DMMasterAddrs}} @@ -383,6 +391,14 @@ scrape_configs: {{- if .DMWorkerAddrs}} - job_name: "dm_worker" honor_labels: true # don't overwrite job & instance labels +{{- if .TLSEnabled}} + scheme: https + tls_config: + insecure_skip_verify: false + ca_file: ../tls/ca.crt + cert_file: ../tls/prometheus.crt + key_file: ../tls/prometheus.pem +{{- end}} static_configs: - targets: {{- range .DMWorkerAddrs}} diff --git a/embed/templates/scripts/run_dm-master.sh.tpl b/embed/templates/scripts/run_dm-master.sh.tpl index 271a15286e..246e2097ad 100644 --- a/embed/templates/scripts/run_dm-master.sh.tpl +++ b/embed/templates/scripts/run_dm-master.sh.tpl @@ -25,10 +25,10 @@ exec bin/dm-master/dm-master \ --v1-sources-path="{{.V1SourcePath}}" \ {{- end}} --name="{{.Name}}" \ - --master-addr="0.0.0.0:{{.Port}}" \ + --master-addr="{{.IP}}:{{.Port}}" \ --advertise-addr="{{.IP}}:{{.Port}}" \ - --peer-urls="{{.IP}}:{{.PeerPort}}" \ - --advertise-peer-urls="{{.IP}}:{{.PeerPort}}" \ + --peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ + --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ --log-file="{{.LogDir}}/dm-master.log" \ --data-dir="{{.DataDir}}" \ --initial-cluster="{{template "MasterList" .Endpoints}}" \ diff --git a/embed/templates/scripts/run_dm-master_scale.sh.tpl b/embed/templates/scripts/run_dm-master_scale.sh.tpl index 13a31977aa..90572b0241 100644 --- a/embed/templates/scripts/run_dm-master_scale.sh.tpl +++ b/embed/templates/scripts/run_dm-master_scale.sh.tpl @@ -22,7 +22,7 @@ exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/dm-master/d exec bin/dm-master/dm-master \ {{- end}} --name="{{.Name}}" \ - --master-addr="0.0.0.0:{{.Port}}" \ + --master-addr="{{.IP}}:{{.Port}}" \ --advertise-addr="{{.IP}}:{{.Port}}" \ --peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ diff --git a/embed/templates/scripts/run_dm-worker.sh.tpl b/embed/templates/scripts/run_dm-worker.sh.tpl index f1074b4967..df11e96e20 100644 --- a/embed/templates/scripts/run_dm-worker.sh.tpl +++ b/embed/templates/scripts/run_dm-worker.sh.tpl @@ -23,7 +23,7 @@ exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/dm-worker/d exec bin/dm-worker/dm-worker \ {{- end}} --name="{{.Name}}" \ - --worker-addr="0.0.0.0:{{.Port}}" \ + --worker-addr="{{.IP}}:{{.Port}}" \ --advertise-addr="{{.IP}}:{{.Port}}" \ --log-file="{{.LogDir}}/dm-worker.log" \ --join="{{template "MasterList" .Endpoints}}" \ diff --git a/pkg/cluster/template/scripts/dm_master.go b/pkg/cluster/template/scripts/dm_master.go index d0e52e8e58..3dce4bca17 100644 --- a/pkg/cluster/template/scripts/dm_master.go +++ b/pkg/cluster/template/scripts/dm_master.go @@ -21,6 +21,7 @@ import ( "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // DMMasterScript represent the data to generate TiDB config @@ -39,10 +40,10 @@ type DMMasterScript struct { } // NewDMMasterScript returns a DMMasterScript with given arguments -func NewDMMasterScript(name, ip, deployDir, dataDir, logDir string) *DMMasterScript { +func NewDMMasterScript(name, ip, deployDir, dataDir, logDir string, enableTLS bool) *DMMasterScript { return &DMMasterScript{ Name: name, - Scheme: "http", + Scheme: utils.Ternary(enableTLS, "https", "http").(string), IP: ip, Port: 8261, PeerPort: 8291, @@ -137,8 +138,8 @@ type DMMasterScaleScript struct { } // NewDMMasterScaleScript return a new DMMasterScaleScript -func NewDMMasterScaleScript(name, ip, deployDir, dataDir, logDir string) *DMMasterScaleScript { - return &DMMasterScaleScript{*NewDMMasterScript(name, ip, deployDir, dataDir, logDir)} +func NewDMMasterScaleScript(name, ip, deployDir, dataDir, logDir string, enableTLS bool) *DMMasterScaleScript { + return &DMMasterScaleScript{*NewDMMasterScript(name, ip, deployDir, dataDir, logDir, enableTLS)} } // WithScheme set Scheme field of DMMasterScaleScript diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 62a22092e7..65b29d1ace 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -62,3 +62,11 @@ func Base62Tag() string { } return string(b) } + +// Ternary operator +func Ternary(condition bool, a, b interface{}) interface{} { + if condition { + return a + } + return b +}