cluster: fix Grafana error datasource not found

components/playground/grafana.go

@@ -55,8 +55,6 @@ func writeDatasourceConfig(fname string, clusterName string, p8sURL string) erro
 	}
 
 	tpl := `apiVersion: 1
-deleteDatasources:
-  - name: %s
 datasources:
   - name: %s
     type: prometheus
@@ -70,7 +68,7 @@ datasources:
     editable: true
 `
 
-	s := fmt.Sprintf(tpl, clusterName, clusterName, p8sURL)
+	s := fmt.Sprintf(tpl, clusterName, p8sURL)
 	err = os.WriteFile(fname, []byte(s), 0644)
 	if err != nil {
 		return errors.AddStack(err)

embed/templates/config/datasource.yml.tpl

@@ -1,6 +1,4 @@
 apiVersion: 1
-deleteDatasources:
-  - name: {{.ClusterName}}
 datasources:
   - name: {{.ClusterName}}
     type: prometheus

pkg/cluster/manager/builder.go

@@ -32,29 +32,38 @@ import (
 	"github.com/pingcap/tiup/pkg/utils"
 )
 
-// buildReloadPromTasks reloads Prometheus configuration
-func buildReloadPromTasks(
+// buildReloadPromTasks reloads Prometheus and Grafana configuration
+func buildReloadPromAndGrafanaTasks(
 	topo spec.Topology,
 	logger *logprinter.Logger,
 	gOpt operator.Options,
 	nodes ...string,
 ) []*task.StepDisplay {
+	var instances []spec.Instance
+	// get promtheus and grafana instance list
 	monitor := spec.FindComponent(topo, spec.ComponentPrometheus)
-	if monitor == nil {
-		return nil
-	}
-	instances := monitor.Instances()
+	grafanas := spec.FindComponent(topo, spec.ComponentGrafana)
+
+	instances = append(instances, monitor.Instances()...)
+	instances = append(instances, grafanas.Instances()...)
+
 	if len(instances) == 0 {
 		return nil
 	}
 	var tasks []*task.StepDisplay
 	deletedNodes := set.NewStringSet(nodes...)
-	for _, inst := range monitor.Instances() {
+	for _, inst := range instances {
 		if deletedNodes.Exist(inst.ID()) {
 			continue
 		}
+		// reload Prometheus
+		action := "reload"
+		if inst.ComponentName() == spec.ComponentGrafana {
+			// restart grafana
+			action = "restart"
+		}
 		t := task.NewBuilder(logger).
-			SystemCtl(inst.GetHost(), inst.ServiceName(), "reload", true).
+			SystemCtl(inst.GetHost(), inst.ServiceName(), action, true).
 			BuildAsStep(fmt.Sprintf("  - Reload %s -> %s", inst.ComponentName(), inst.ID()))
 		tasks = append(tasks, t)
 	}
@@ -340,7 +349,8 @@ func buildScaleOutTask(
 			return operator.Start(ctx, newPart, operator.Options{OptTimeout: gOpt.OptTimeout, Operation: operator.ScaleOutOperation}, tlsCfg)
 		}).
 			ParallelStep("+ Refresh components conifgs", gOpt.Force, refreshConfigTasks...).
-			ParallelStep("+ Reload prometheus", gOpt.Force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt)...)
+			ParallelStep("+ Reload prometheus and grafanas", gOpt.Force,
+				buildReloadPromAndGrafanaTasks(metadata.GetTopology(), m.logger, gOpt)...)
 	}
 
 	// remove scale-out file lock

pkg/cluster/manager/destroy.go

@@ -166,7 +166,8 @@ func (m *Manager) DestroyTombstone(
 		UpdateMeta(name, clusterMeta, nodes).
 		UpdateTopology(name, m.specManager.Path(name), clusterMeta, nodes).
 		ParallelStep("+ Refresh instance configs", gOpt.Force, regenConfigTasks...).
-		ParallelStep("+ Reloda prometheus", gOpt.Force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt)...).
+		ParallelStep("+ Reloda prometheus and grafanas", gOpt.Force,
+			buildReloadPromAndGrafanaTasks(metadata.GetTopology(), m.logger, gOpt)...).
 		Build()
 
 	if err := t.Execute(ctx); err != nil {

pkg/cluster/manager/scale_in.go

@@ -116,7 +116,8 @@ func (m *Manager) ScaleIn(
 
 	t := b.
 		ParallelStep("+ Refresh instance configs", force, regenConfigTasks...).
-		ParallelStep("+ Reloda prometheus", gOpt.Force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt, nodes...)...).
+		ParallelStep("+ Reloda prometheus and grafanas", gOpt.Force,
+			buildReloadPromAndGrafanaTasks(metadata.GetTopology(), m.logger, gOpt, nodes...)...).
 		Build()
 
 	ctx := ctxt.New(