cluster: check active before reload promtheus when scale-in/scale-out (…

…#1775)
pingcap · Mar 7, 2022 · caab563 · caab563
1 parent 82c3791
commit caab563
Show file tree

Hide file tree

Showing 7 changed files with 22 additions and 13 deletions.
diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go
@@ -56,16 +56,17 @@ func buildReloadPromAndGrafanaTasks(
 		if deletedNodes.Exist(inst.ID()) {
 			continue
 		}
-		// reload Prometheus
-		action := "reload"
-		if inst.ComponentName() == spec.ComponentGrafana {
+
+		t := task.NewBuilder(logger)
+		if inst.ComponentName() == spec.ComponentPrometheus {
+			// reload Prometheus
+			t = t.SystemCtl(inst.GetHost(), inst.ServiceName(), "reload", true, true)
+		} else {
 			// restart grafana
-			action = "restart"
+			t = t.SystemCtl(inst.GetHost(), inst.ServiceName(), "restart", true, false)
 		}
-		t := task.NewBuilder(logger).
-			SystemCtl(inst.GetHost(), inst.ServiceName(), action, true).
-			BuildAsStep(fmt.Sprintf("  - Reload %s -> %s", inst.ComponentName(), inst.ID()))
-		tasks = append(tasks, t)
+
+		tasks = append(tasks, t.BuildAsStep(fmt.Sprintf("  - Reload %s -> %s", inst.ComponentName(), inst.ID())))
 	}
 	return tasks
 }

diff --git a/pkg/cluster/manager/check.go b/pkg/cluster/manager/check.go
@@ -600,7 +600,7 @@ func fixFailedChecks(host string, res *operator.CheckResult, t *task.Builder) (s
 		if len(fields) < 2 {
 			return "", fmt.Errorf("can not perform action of service, %s", res.Msg)
 		}
-		t.SystemCtl(host, fields[1], fields[0], false)
+		t.SystemCtl(host, fields[1], fields[0], false, false)
 		msg = fmt.Sprintf("will try to '%s'", color.HiBlueString(res.Msg))
 	case operator.CheckNameSysctl:
 		fields := strings.Fields(res.Msg)

diff --git a/pkg/cluster/manager/destroy.go b/pkg/cluster/manager/destroy.go
@@ -166,7 +166,7 @@ func (m *Manager) DestroyTombstone(
 		UpdateMeta(name, clusterMeta, nodes).
 		UpdateTopology(name, m.specManager.Path(name), clusterMeta, nodes).
 		ParallelStep("+ Refresh instance configs", gOpt.Force, regenConfigTasks...).
-		ParallelStep("+ Reloda prometheus and grafana", gOpt.Force,
+		ParallelStep("+ Reload prometheus and grafana", gOpt.Force,
 			buildReloadPromAndGrafanaTasks(metadata.GetTopology(), m.logger, gOpt)...).
 		Build()
 

diff --git a/pkg/cluster/manager/scale_in.go b/pkg/cluster/manager/scale_in.go
@@ -116,7 +116,7 @@ func (m *Manager) ScaleIn(
 
 	t := b.
 		ParallelStep("+ Refresh instance configs", force, regenConfigTasks...).
-		ParallelStep("+ Reloda prometheus and grafana", gOpt.Force,
+		ParallelStep("+ Reload prometheus and grafana", gOpt.Force,
 			buildReloadPromAndGrafanaTasks(metadata.GetTopology(), m.logger, gOpt, nodes...)...).
 		Build()
 

diff --git a/pkg/cluster/module/systemd.go b/pkg/cluster/module/systemd.go
@@ -34,6 +34,7 @@ type SystemdModuleConfig struct {
 	Unit         string        // the name of systemd unit(s)
 	Action       string        // the action to perform with the unit
 	ReloadDaemon bool          // run daemon-reload before other actions
+	CheckActive  bool          // run is-active before action
 	Scope        string        // user, system or global
 	Force        bool          // add the `--force` arg to systemctl command
 	Signal       string        // specify the signal to send to process
@@ -72,11 +73,15 @@ func NewSystemdModule(config SystemdModuleConfig) *SystemdModule {
 	cmd := fmt.Sprintf("%s %s %s",
 		systemctl, strings.ToLower(config.Action), config.Unit)
 
+	if config.CheckActive {
+		cmd = fmt.Sprintf("if [[ $(%s is-active %s) == \"active\" ]]; then %s; fi",
+			systemctl, config.Unit, cmd)
+	}
+
 	if config.ReloadDaemon {
 		cmd = fmt.Sprintf("%s daemon-reload && %s",
 			systemctl, cmd)
 	}
-
 	mod := &SystemdModule{
 		cmd:     cmd,
 		sudo:    sudo,

diff --git a/pkg/cluster/task/builder.go b/pkg/cluster/task/builder.go
@@ -383,12 +383,13 @@ func (b *Builder) Shell(host, command, cmdID string, sudo bool) *Builder {
 }
 
 // SystemCtl run systemctl on host
-func (b *Builder) SystemCtl(host, unit, action string, daemonReload bool) *Builder {
+func (b *Builder) SystemCtl(host, unit, action string, daemonReload, checkActive bool) *Builder {
 	b.tasks = append(b.tasks, &SystemCtl{
 		host:         host,
 		unit:         unit,
 		action:       action,
 		daemonReload: daemonReload,
+		checkactive:  checkActive,
 	})
 	return b
 }

diff --git a/pkg/cluster/task/systemd.go b/pkg/cluster/task/systemd.go
@@ -28,6 +28,7 @@ type SystemCtl struct {
 	unit         string
 	action       string
 	daemonReload bool
+	checkactive  bool
 }
 
 // Execute implements the Task interface
@@ -41,6 +42,7 @@ func (c *SystemCtl) Execute(ctx context.Context) error {
 		Unit:         c.unit,
 		Action:       c.action,
 		ReloadDaemon: c.daemonReload,
+		CheckActive:  c.checkactive,
 	}
 	systemd := module.NewSystemdModule(cfg)
 	stdout, stderr, err := systemd.Execute(ctx, e)