dm: add TLS support (#1745)

components/dm/command/prune.go

@@ -20,6 +20,7 @@ import (
 	"github.com/pingcap/tiup/components/dm/spec"
 	"github.com/pingcap/tiup/pkg/cluster/api"
 	operator "github.com/pingcap/tiup/pkg/cluster/operation"
+	tidbspec "github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 )
@@ -63,7 +64,11 @@ func clearOutDatedEtcdInfo(clusterName string, metadata *spec.Metadata, opt oper
 		existedWorkers[workerSpec.Name] = struct{}{}
 	}
 
-	dmMasterClient := api.NewDMMasterClient(topo.GetMasterList(), 10*time.Second, nil)
+	tlsCfg, err := topo.TLSConfig(dmspec.Path(clusterName, tidbspec.TLSCertKeyDir))
+	if err != nil {
+		return err
+	}
+	dmMasterClient := api.NewDMMasterClient(topo.GetMasterList(), 10*time.Second, tlsCfg)
 	registeredMasters, registeredWorkers, err := dmMasterClient.GetRegisteredMembers()
 	if err != nil {
 		return err

components/dm/command/scale_in.go

@@ -46,7 +46,7 @@ func newScaleInCmd() *cobra.Command {
 				b.Func(
 					fmt.Sprintf("ScaleInCluster: options=%+v", gOpt),
 					func(ctx context.Context) error {
-						return ScaleInDMCluster(ctx, metadata.Topology, gOpt)
+						return ScaleInDMCluster(ctx, metadata.Topology, gOpt, tlsCfg)
 					},
 				).Serial(dmtask.NewUpdateDMMeta(clusterName, metadata, gOpt.Nodes))
 			}
@@ -68,6 +68,7 @@ func ScaleInDMCluster(
 	ctx context.Context,
 	topo *dm.Specification,
 	options operator.Options,
+	tlsCfg *tls.Config,
 ) error {
 	// instances by uuid
 	instances := map[string]dm.Instance{}
@@ -125,7 +126,7 @@ func ScaleInDMCluster(
 		return errors.New("cannot find available dm-master instance")
 	}
 
-	dmMasterClient = api.NewDMMasterClient(dmMasterEndpoint, 10*time.Second, nil)
+	dmMasterClient = api.NewDMMasterClient(dmMasterEndpoint, 10*time.Second, tlsCfg)
 
 	noAgentHosts := set.NewStringSet()
 	topo.IterInstance(func(inst dm.Instance) {

components/dm/spec/logic.go

@@ -119,13 +119,15 @@ func (i *MasterInstance) InitConfig(
 		return err
 	}
 
+	enableTLS := i.topo.GlobalOptions.TLSEnabled
 	spec := i.InstanceSpec.(*MasterSpec)
 	cfg := scripts.NewDMMasterScript(
 		spec.Name,
 		i.GetHost(),
 		paths.Deploy,
 		paths.Data[0],
 		paths.Log,
+		enableTLS,
 	).WithPort(spec.Port).WithNumaNode(spec.NumaNode).WithPeerPort(spec.PeerPort).AppendEndpoints(i.topo.Endpoints(deployUser)...).WithV1SourcePath(spec.V1SourcePath)
 
 	fp := filepath.Join(paths.Cache, fmt.Sprintf("run_dm-master_%s_%d.sh", i.GetHost(), i.GetPort()))
@@ -136,12 +138,12 @@ func (i *MasterInstance) InitConfig(
 	if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil {
 		return err
 	}
-	if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil {
+	_, _, err := e.Execute(ctx, "chmod +x "+dst, false)
+	if err != nil {
 		return err
 	}
 
-	// doesn't work
-	if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil {
+	if spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths); err != nil {
 		return err
 	}
 
@@ -152,7 +154,40 @@ func (i *MasterInstance) InitConfig(
 // setTLSConfig set TLS Config to support enable/disable TLS
 // MasterInstance no need to configure TLS
 func (i *MasterInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) {
-	return nil, nil
+	// set TLS configs
+	if enableTLS {
+		if configs == nil {
+			configs = make(map[string]interface{})
+		}
+		configs["ssl-ca"] = fmt.Sprintf(
+			"%s/tls/%s",
+			paths.Deploy,
+			"ca.crt",
+		)
+		configs["ssl-cert"] = fmt.Sprintf(
+			"%s/tls/%s.crt",
+			paths.Deploy,
+			i.Role())
+		configs["ssl-key"] = fmt.Sprintf(
+			"%s/tls/%s.pem",
+			paths.Deploy,
+			i.Role())
+	} else {
+		// dm-master tls config list
+		tlsConfigs := []string{
+			"ssl-ca",
+			"ssl-cert",
+			"ssl-key",
+		}
+		// delete TLS configs
+		if configs != nil {
+			for _, config := range tlsConfigs {
+				delete(configs, config)
+			}
+		}
+	}
+
+	return configs, nil
 }
 
 // ScaleConfig deploy temporary config on scaling
@@ -169,6 +204,7 @@ func (i *MasterInstance) ScaleConfig(
 		return err
 	}
 
+	enableTLS := i.topo.GlobalOptions.TLSEnabled
 	c := topo.(*Specification)
 	spec := i.InstanceSpec.(*MasterSpec)
 	cfg := scripts.NewDMMasterScaleScript(
@@ -177,6 +213,7 @@ func (i *MasterInstance) ScaleConfig(
 		paths.Deploy,
 		paths.Data[0],
 		paths.Log,
+		enableTLS,
 	).WithPort(spec.Port).WithNumaNode(spec.NumaNode).WithPeerPort(spec.PeerPort).AppendEndpoints(c.Endpoints(deployUser)...)
 
 	fp := filepath.Join(paths.Cache, fmt.Sprintf("run_dm-master_%s_%d.sh", i.GetHost(), i.GetPort()))
@@ -261,6 +298,7 @@ func (i *WorkerInstance) InitConfig(
 		return err
 	}
 
+	enableTLS := i.topo.GlobalOptions.TLSEnabled
 	spec := i.InstanceSpec.(*WorkerSpec)
 	cfg := scripts.NewDMWorkerScript(
 		i.Name,
@@ -278,12 +316,12 @@ func (i *WorkerInstance) InitConfig(
 		return err
 	}
 
-	if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil {
+	_, _, err := e.Execute(ctx, "chmod +x "+dst, false)
+	if err != nil {
 		return err
 	}
 
-	// doesn't work
-	if _, err := i.setTLSConfig(ctx, false, nil, paths); err != nil {
+	if spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths); err != nil {
 		return err
 	}
 
@@ -294,7 +332,40 @@ func (i *WorkerInstance) InitConfig(
 // setTLSConfig set TLS Config to support enable/disable TLS
 // workrsInstance no need to configure TLS
 func (i *WorkerInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]interface{}, paths meta.DirPaths) (map[string]interface{}, error) {
-	return nil, nil
+	// set TLS configs
+	if enableTLS {
+		if configs == nil {
+			configs = make(map[string]interface{})
+		}
+		configs["ssl-ca"] = fmt.Sprintf(
+			"%s/tls/%s",
+			paths.Deploy,
+			"ca.crt",
+		)
+		configs["ssl-cert"] = fmt.Sprintf(
+			"%s/tls/%s.crt",
+			paths.Deploy,
+			i.Role())
+		configs["ssl-key"] = fmt.Sprintf(
+			"%s/tls/%s.pem",
+			paths.Deploy,
+			i.Role())
+	} else {
+		// dm-worker tls config list
+		tlsConfigs := []string{
+			"ssl-ca",
+			"ssl-cert",
+			"ssl-key",
+		}
+		// delete TLS configs
+		if configs != nil {
+			for _, config := range tlsConfigs {
+				delete(configs, config)
+			}
+		}
+	}
+
+	return configs, nil
 }
 
 // ScaleConfig deploy temporary config on scaling
@@ -411,7 +482,8 @@ func (topo *Specification) Endpoints(user string) []*scripts.DMMasterScript {
 			s.Host,
 			deployDir,
 			dataDir,
-			logDir).
+			logDir,
+			topo.GlobalOptions.TLSEnabled).
 			WithPort(s.Port).
 			WithPeerPort(s.PeerPort)
 		ends = append(ends, script)

embed/templates/config/prometheus.yml.tpl

@@ -373,6 +373,14 @@ scrape_configs:
 {{- if .DMMasterAddrs}}
   - job_name: "dm_master"
     honor_labels: true # don't overwrite job & instance labels
+{{- if .TLSEnabled}}
+    scheme: https
+    tls_config:
+      insecure_skip_verify: false
+      ca_file: ../tls/ca.crt
+      cert_file: ../tls/prometheus.crt
+      key_file: ../tls/prometheus.pem
+{{- end}}
     static_configs:
     - targets:
     {{- range .DMMasterAddrs}}
@@ -383,6 +391,14 @@ scrape_configs:
 {{- if .DMWorkerAddrs}}
   - job_name: "dm_worker"
     honor_labels: true # don't overwrite job & instance labels
+{{- if .TLSEnabled}}
+    scheme: https
+    tls_config:
+      insecure_skip_verify: false
+      ca_file: ../tls/ca.crt
+      cert_file: ../tls/prometheus.crt
+      key_file: ../tls/prometheus.pem
+{{- end}}
     static_configs:
     - targets:
     {{- range .DMWorkerAddrs}}

embed/templates/scripts/run_dm-master.sh.tpl

@@ -25,10 +25,10 @@ exec bin/dm-master/dm-master \
     --v1-sources-path="{{.V1SourcePath}}" \
 {{- end}}
     --name="{{.Name}}" \
-    --master-addr="0.0.0.0:{{.Port}}" \
+    --master-addr="{{.IP}}:{{.Port}}" \
     --advertise-addr="{{.IP}}:{{.Port}}" \
-    --peer-urls="{{.IP}}:{{.PeerPort}}" \
-    --advertise-peer-urls="{{.IP}}:{{.PeerPort}}" \
+    --peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \
+    --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \
     --log-file="{{.LogDir}}/dm-master.log" \
     --data-dir="{{.DataDir}}" \
     --initial-cluster="{{template "MasterList" .Endpoints}}" \

embed/templates/scripts/run_dm-master_scale.sh.tpl

@@ -22,7 +22,7 @@ exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/dm-master/d
 exec bin/dm-master/dm-master \
 {{- end}}
     --name="{{.Name}}" \
-    --master-addr="0.0.0.0:{{.Port}}" \
+    --master-addr="{{.IP}}:{{.Port}}" \
     --advertise-addr="{{.IP}}:{{.Port}}" \
     --peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \
     --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \

embed/templates/scripts/run_dm-worker.sh.tpl

@@ -23,7 +23,7 @@ exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/dm-worker/d
 exec bin/dm-worker/dm-worker \
 {{- end}}
     --name="{{.Name}}" \
-    --worker-addr="0.0.0.0:{{.Port}}" \
+    --worker-addr="{{.IP}}:{{.Port}}" \
     --advertise-addr="{{.IP}}:{{.Port}}" \
     --log-file="{{.LogDir}}/dm-worker.log" \
     --join="{{template "MasterList" .Endpoints}}" \

pkg/cluster/template/scripts/dm_master.go

@@ -21,6 +21,7 @@ import (
 	"text/template"
 
 	"github.com/pingcap/tiup/embed"
+	"github.com/pingcap/tiup/pkg/utils"
 )
 
 // DMMasterScript represent the data to generate TiDB config
@@ -39,10 +40,10 @@ type DMMasterScript struct {
 }
 
 // NewDMMasterScript returns a DMMasterScript with given arguments
-func NewDMMasterScript(name, ip, deployDir, dataDir, logDir string) *DMMasterScript {
+func NewDMMasterScript(name, ip, deployDir, dataDir, logDir string, enableTLS bool) *DMMasterScript {
 	return &DMMasterScript{
 		Name:      name,
-		Scheme:    "http",
+		Scheme:    utils.Ternary(enableTLS, "https", "http").(string),
 		IP:        ip,
 		Port:      8261,
 		PeerPort:  8291,
@@ -137,8 +138,8 @@ type DMMasterScaleScript struct {
 }
 
 // NewDMMasterScaleScript return a new DMMasterScaleScript
-func NewDMMasterScaleScript(name, ip, deployDir, dataDir, logDir string) *DMMasterScaleScript {
-	return &DMMasterScaleScript{*NewDMMasterScript(name, ip, deployDir, dataDir, logDir)}
+func NewDMMasterScaleScript(name, ip, deployDir, dataDir, logDir string, enableTLS bool) *DMMasterScaleScript {
+	return &DMMasterScaleScript{*NewDMMasterScript(name, ip, deployDir, dataDir, logDir, enableTLS)}
 }
 
 // WithScheme set Scheme field of DMMasterScaleScript

pkg/utils/utils.go

@@ -62,3 +62,11 @@ func Base62Tag() string {
 	}
 	return string(b)
 }
+
+// Ternary operator
+func Ternary(condition bool, a, b interface{}) interface{} {
+	if condition {
+		return a
+	}
+	return b
+}