Merge branch 'master' into update-embed

CHANGELOG.md

@@ -1,5 +1,17 @@
 TiUP Changelog
 
+## [1.5.4] 2021-08-05
+
+### Fixes
+
+- Allow editing of `lerner_config` field in TiFlash spec ([#1494](https://github.com/pingcap/tiup/pull/1494), [@AstroProfundis](https://github.com/AstroProfundis))
+- Fix incorrect timeout for telemetry requests ([#1500](https://github.com/pingcap/tiup/pull/1500), [@AstroProfundis](https://github.com/AstroProfundis))
+- Ingore `data_dir` of monitor agents when checking for directory overlaps ([#1510](https://github.com/pingcap/tiup/pull/1510), [@AstroProfundis](https://github.com/AstroProfundis))
+
+### Improvements
+
+- Distinguish cookie names of multiple grafana instances on the same host ([#1491](https://github.com/pingcap/tiup/pull/1491), [@AstroProfundis](https://github.com/AstroProfundis))
+
 ## [1.5.3] 2021-07-15
 
 ### Fixes

components/dm/command/scale_in.go

@@ -128,14 +128,21 @@ func ScaleInDMCluster(
 
 	dmMasterClient = api.NewDMMasterClient(dmMasterEndpoint, 10*time.Second, nil)
 
+	noAgentHosts := set.NewStringSet()
+	topo.IterInstance(func(inst dm.Instance) {
+		if inst.IgnoreMonitorAgent() {
+			noAgentHosts.Insert(inst.GetHost())
+		}
+	})
+
 	// Delete member from cluster
 	for _, component := range topo.ComponentsByStartOrder() {
 		for _, instance := range component.Instances() {
 			if !deletedNodes.Exist(instance.ID()) {
 				continue
 			}
 
-			if err := operator.StopComponent(ctx, []dm.Instance{instance}, options.OptTimeout); err != nil {
+			if err := operator.StopComponent(ctx, []dm.Instance{instance}, noAgentHosts, options.OptTimeout); err != nil {
 				return errors.Annotatef(err, "failed to stop %s", component.Name())
 			}
 

components/dm/spec/logic.go

@@ -45,6 +45,7 @@ type (
 		SSH() (string, int)
 		GetMainPort() int
 		IsImported() bool
+		IgnoreMonitorAgent() bool
 	}
 )
 

components/dm/spec/topology_dm.go

@@ -118,10 +118,11 @@ func AllDMComponentNames() (roles []string) {
 
 // MasterSpec represents the Master topology specification in topology.yaml
 type MasterSpec struct {
-	Host     string `yaml:"host"`
-	SSHPort  int    `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"`
-	Imported bool   `yaml:"imported,omitempty"`
-	Patched  bool   `yaml:"patched,omitempty"`
+	Host           string `yaml:"host"`
+	SSHPort        int    `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"`
+	Imported       bool   `yaml:"imported,omitempty"`
+	Patched        bool   `yaml:"patched,omitempty"`
+	IgnoreExporter bool   `yaml:"ignore_exporter,omitempty"`
 	// Use Name to get the name with a default value if it's empty.
 	Name            string                 `yaml:"name,omitempty"`
 	Port            int                    `yaml:"port,omitempty" default:"8261"`
@@ -178,12 +179,18 @@ func (s *MasterSpec) IsImported() bool {
 	return s.Imported
 }
 
+// IgnoreMonitorAgent returns if the node does not have monitor agents available
+func (s *MasterSpec) IgnoreMonitorAgent() bool {
+	return s.IgnoreExporter
+}
+
 // WorkerSpec represents the Master topology specification in topology.yaml
 type WorkerSpec struct {
-	Host     string `yaml:"host"`
-	SSHPort  int    `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"`
-	Imported bool   `yaml:"imported,omitempty"`
-	Patched  bool   `yaml:"patched,omitempty"`
+	Host           string `yaml:"host"`
+	SSHPort        int    `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"`
+	Imported       bool   `yaml:"imported,omitempty"`
+	Patched        bool   `yaml:"patched,omitempty"`
+	IgnoreExporter bool   `yaml:"ignore_exporter,omitempty"`
 	// Use Name to get the name with a default value if it's empty.
 	Name            string                 `yaml:"name,omitempty"`
 	Port            int                    `yaml:"port,omitempty" default:"8262"`
@@ -233,6 +240,11 @@ func (s *WorkerSpec) IsImported() bool {
 	return s.Imported
 }
 
+// IgnoreMonitorAgent returns if the node does not have monitor agents available
+func (s *WorkerSpec) IgnoreMonitorAgent() bool {
+	return s.IgnoreExporter
+}
+
 // UnmarshalYAML sets default values when unmarshaling the topology file
 func (s *Specification) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type topology Specification

pkg/cluster/manager/builder.go

@@ -258,6 +258,7 @@ func buildScaleOutTask(
 	}
 
 	hasImported := false
+	noAgentHosts := set.NewStringSet()
 
 	mergedTopo.IterInstance(func(inst spec.Instance) {
 		deployDir := spec.Abs(base.User, inst.DeployDir())
@@ -278,6 +279,11 @@ func buildScaleOutTask(
 			hasImported = true
 		}
 
+		// add the instance to ignore list if it marks itself as ignore_exporter
+		if inst.IgnoreMonitorAgent() {
+			noAgentHosts.Insert(inst.GetHost())
+		}
+
 		// Refresh all configuration
 		t := tb.InitConfig(name,
 			base.Version,
@@ -307,6 +313,7 @@ func buildScaleOutTask(
 		m,
 		name,
 		uninitializedHosts,
+		noAgentHosts,
 		topo.BaseTopo().GlobalOptions,
 		topo.BaseTopo().MonitoredOptions,
 		base.Version,
@@ -370,6 +377,7 @@ func buildMonitoredDeployTask(
 	m *Manager,
 	name string,
 	uniqueHosts map[string]hostInfo, // host -> ssh-port, os, arch
+	noAgentHosts set.StringSet, // hosts that do not deploy monitor agents
 	globalOptions *spec.GlobalOptions,
 	monitoredOptions *spec.MonitoredOptions,
 	version string,
@@ -386,6 +394,11 @@ func buildMonitoredDeployTask(
 		version := m.bindVersion(comp, version)
 
 		for host, info := range uniqueHosts {
+			// skip deploying monitoring agents if the instance is marked so
+			if noAgentHosts.Exist(host) {
+				continue
+			}
+
 			// populate unique comp-os-arch set
 			key := fmt.Sprintf("%s-%s-%s", comp, info.os, info.arch)
 			if found := uniqueCompOSArch.Exist(key); !found {
@@ -493,6 +506,7 @@ func buildRefreshMonitoredConfigTasks(
 	specManager *spec.SpecManager,
 	name string,
 	uniqueHosts map[string]hostInfo, // host -> ssh-port, os, arch
+	noAgentHosts set.StringSet,
 	globalOptions spec.GlobalOptions,
 	monitoredOptions *spec.MonitoredOptions,
 	sshTimeout, exeTimeout uint64,
@@ -507,6 +521,10 @@ func buildRefreshMonitoredConfigTasks(
 	// monitoring agents
 	for _, comp := range []string{spec.ComponentNodeExporter, spec.ComponentBlackboxExporter} {
 		for host, info := range uniqueHosts {
+			if noAgentHosts.Exist(host) {
+				continue
+			}
+
 			deployDir := spec.Abs(globalOptions.User, monitoredOptions.DeployDir)
 			// data dir would be empty for components which don't need it
 			dataDir := monitoredOptions.DataDir

pkg/cluster/manager/cleanup.go

@@ -58,6 +58,15 @@ func (m *Manager) CleanCluster(name string, gOpt operator.Options, cleanOpt oper
 		retainDataNodes := set.NewStringSet(cleanOpt.RetainDataNodes...)
 
 		for _, ins := range instances {
+			// not cleaning files of monitor agents if the instance does not have one
+			switch ins.ComponentName() {
+			case spec.ComponentNodeExporter,
+				spec.ComponentBlackboxExporter:
+				if ins.IgnoreMonitorAgent() {
+					continue
+				}
+			}
+
 			// Some data of instances will be retained
 			dataRetained := retainDataRoles.Exist(ins.ComponentName()) ||
 				retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost())

pkg/cluster/manager/deploy.go

@@ -177,6 +177,7 @@ func (m *Manager) Deploy(
 
 	// Initialize environment
 	uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch
+	noAgentHosts := set.NewStringSet()
 	globalOptions := base.GlobalOptions
 
 	// generate CA and client cert for TLS enabled cluster
@@ -212,6 +213,11 @@ func (m *Manager) Deploy(
 				return // skip the host to avoid issues
 			}
 
+			// add the instance to ignore list if it marks itself as ignore_exporter
+			if inst.IgnoreMonitorAgent() {
+				noAgentHosts.Insert(inst.GetHost())
+			}
+
 			uniqueHosts[inst.GetHost()] = hostInfo{
 				ssh:  inst.GetSSHPort(),
 				os:   inst.OS(),
@@ -373,6 +379,7 @@ func (m *Manager) Deploy(
 		m,
 		name,
 		uniqueHosts,
+		noAgentHosts,
 		globalOptions,
 		topo.GetMonitoredOptions(),
 		clusterVersion,

pkg/cluster/manager/reload.go

@@ -27,6 +27,7 @@ import (
 	operator "github.com/pingcap/tiup/pkg/cluster/operation"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/logger/log"
+	"github.com/pingcap/tiup/pkg/set"
 	"github.com/pingcap/tiup/pkg/tui"
 )
 
@@ -69,7 +70,13 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo
 	base := metadata.GetBaseMeta()
 
 	uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch
+	noAgentHosts := set.NewStringSet()
 	topo.IterInstance(func(inst spec.Instance) {
+		// add the instance to ignore list if it marks itself as ignore_exporter
+		if inst.IgnoreMonitorAgent() {
+			noAgentHosts.Insert(inst.GetHost())
+		}
+
 		if _, found := uniqueHosts[inst.GetHost()]; !found {
 			uniqueHosts[inst.GetHost()] = hostInfo{
 				ssh:  inst.GetSSHPort(),
@@ -84,6 +91,7 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo
 		m.specManager,
 		name,
 		uniqueHosts,
+		noAgentHosts,
 		*topo.BaseTopo().GlobalOptions,
 		topo.GetMonitoredOptions(),
 		sshTimeout,
@@ -103,7 +111,7 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo
 	if err != nil {
 		return err
 	}
-	if topo.Type() == spec.TopoTypeTiDB {
+	if topo.Type() == spec.TopoTypeTiDB && !skipRestart {
 		b.UpdateTopology(
 			name,
 			m.specManager.Path(name),
@@ -117,11 +125,11 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo
 		b.ParallelStep("+ Refresh monitor configs", gOpt.Force, monitorConfigTasks...)
 	}
 
-	tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir))
-	if err != nil {
-		return err
-	}
 	if !skipRestart {
+		tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir))
+		if err != nil {
+			return err
+		}
 		b.Func("UpgradeCluster", func(ctx context.Context) error {
 			return operator.Upgrade(ctx, topo, gOpt, tlsCfg)
 		})

pkg/cluster/manager/upgrade.go

@@ -76,6 +76,16 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio
 	for _, comp := range topo.ComponentsByUpdateOrder() {
 		for _, inst := range comp.Instances() {
 			compName := inst.ComponentName()
+
+			// ignore monitor agents for instances marked as ignore_exporter
+			switch compName {
+			case spec.ComponentNodeExporter,
+				spec.ComponentBlackboxExporter:
+				if inst.IgnoreMonitorAgent() {
+					continue
+				}
+			}
+
 			version := m.bindVersion(inst.ComponentName(), clusterVersion)
 
 			// Download component from repository