cluster: support configure grafana in server_configs (#1703)

components/dm/spec/topology_dm.go

@@ -89,8 +89,9 @@ type (
 type (
 	// DMServerConfigs represents the server runtime configuration
 	DMServerConfigs struct {
-		Master map[string]interface{} `yaml:"master"`
-		Worker map[string]interface{} `yaml:"worker"`
+		Master  map[string]interface{} `yaml:"master"`
+		Worker  map[string]interface{} `yaml:"worker"`
+		Grafana map[string]string      `yaml:"grafana"`
 	}
 
 	// Specification represents the specification of topology.yaml
@@ -838,3 +839,8 @@ func getPort(v reflect.Value) string {
 	}
 	return ""
 }
+
+// GetGrafanaConfig returns global grafana configurations
+func (s *Specification) GetGrafanaConfig() map[string]string {
+	return s.ServerConfigs.Grafana
+}

pkg/cluster/ansible/import_test.go

@@ -129,6 +129,7 @@ server_configs:
     pump: {}
     drainer: {}
     cdc: {}
+    grafana: {}
 tidb_servers: []
 tikv_servers: []
 tiflash_servers: []

pkg/cluster/ansible/test-data/meta.yaml

@@ -24,6 +24,7 @@ topology:
     pump: {}
     drainer: {}
     cdc: {}
+    grafana: {}
   tidb_servers:
     - host: 172.16.1.218
       ssh_port: 9999

pkg/cluster/spec/grafana.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -27,6 +28,7 @@ import (
 	"github.com/pingcap/tiup/pkg/cluster/template/config"
 	"github.com/pingcap/tiup/pkg/cluster/template/scripts"
 	"github.com/pingcap/tiup/pkg/meta"
+	"gopkg.in/ini.v1"
 )
 
 // GrafanaSpec represents the Grafana topology specification in topology.yaml
@@ -38,6 +40,7 @@ type GrafanaSpec struct {
 	IgnoreExporter  bool                 `yaml:"ignore_exporter,omitempty"`
 	Port            int                  `yaml:"port" default:"3000"`
 	DeployDir       string               `yaml:"deploy_dir,omitempty"`
+	Config          map[string]string    `yaml:"config,omitempty" validate:"config:ignore"`
 	ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"`
 	Arch            string               `yaml:"arch,omitempty"`
 	OS              string               `yaml:"os,omitempty"`
@@ -181,11 +184,22 @@ func (i *GrafanaInstance) InitConfig(
 		return err
 	}
 
+	userConfig := i.topo.GetGrafanaConfig()
+	if userConfig == nil {
+		userConfig = make(map[string]string)
+	}
+	for k, v := range spec.Config {
+		userConfig[k] = v
+	}
+	err := mergeAdditionalGrafanaConf(fp, userConfig)
+	if err != nil {
+		return err
+	}
+
 	dst = filepath.Join(paths.Deploy, "conf", "grafana.ini")
 	if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil {
 		return err
 	}
-
 	if err := i.installDashboards(ctx, e, paths.Deploy, clusterName, clusterVersion); err != nil {
 		return errors.Annotate(err, "install dashboards")
 	}
@@ -330,3 +344,24 @@ func (i *GrafanaInstance) ScaleConfig(
 	i.topo = topo.Merge(i.topo)
 	return i.InitConfig(ctx, e, clusterName, clusterVersion, deployUser, paths)
 }
+
+func mergeAdditionalGrafanaConf(source string, addition map[string]string) error {
+	bytes, err := os.ReadFile(source)
+	if err != nil {
+		return err
+	}
+	result, err := ini.Load(bytes)
+	if err != nil {
+		return err
+	}
+	for k, v := range addition {
+		// convert "log.file.level to [log.file] level"
+		for i := len(k) - 1; i >= 0; i-- {
+			if k[i] == '.' {
+				result.Section(k[:i]).Key(k[i+1:]).SetValue(v)
+				break
+			}
+		}
+	}
+	return result.SaveTo(source)
+}

pkg/cluster/spec/grafana_test.go

@@ -15,6 +15,7 @@ package spec
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/user"
 	"path"
@@ -67,3 +68,93 @@ func TestLocalDashboards(t *testing.T) {
 		assert.FileExists(t, path.Join(deployDir, "dashboards", f.Name()))
 	}
 }
+
+func TestMergeAdditionalGrafanaConf(t *testing.T) {
+	file, err := os.CreateTemp("", "tiup-cluster-spec-test")
+	if err != nil {
+		panic(fmt.Sprintf("create temp file: %s", err))
+	}
+	defer os.Remove(file.Name())
+
+	_, err = file.WriteString(`#################################### SMTP / Emailing ##########################
+[smtp]
+;enabled = false
+;host = localhost:25
+;user =
+;password =
+;cert_file =
+;key_file =
+;skip_verify = false
+;from_address = admin@grafana.localhost
+
+[emails]
+;welcome_email_on_sign_up = false
+
+#################################### Logging ##########################
+[log]
+# Either "console", "file", "syslog". Default is console and  file
+# Use space to separate multiple modes, e.g. "console file"
+mode = file
+
+# Either "trace", "debug", "info", "warn", "error", "critical", default is "info"
+;level = info
+# For "console" mode only
+[log.console]
+;level =
+
+# log line format, valid options are text, console and json
+;format = console
+
+# For "file" mode only
+[log.file]
+level = info
+`)
+	assert.Nil(t, err)
+
+	expected := `# ################################### SMTP / Emailing ##########################
+[smtp]
+enabled = true
+
+; enabled = false
+; host = localhost:25
+; user =
+; password =
+; cert_file =
+; key_file =
+; skip_verify = false
+; from_address = admin@grafana.localhost
+[emails]
+
+; welcome_email_on_sign_up = false
+# ################################### Logging ##########################
+[log]
+# Either "console", "file", "syslog". Default is console and  file
+# Use space to separate multiple modes, e.g. "console file"
+mode = file
+
+# Either "trace", "debug", "info", "warn", "error", "critical", default is "info"
+; level = info
+# For "console" mode only
+[log.console]
+
+; level =
+# log line format, valid options are text, console and json
+; format = console
+# For "file" mode only
+[log.file]
+level = warning
+
+`
+
+	addition := map[string]string{
+		"log.file.level": "warning",
+		"smtp.enabled":   "true",
+	}
+
+	err = mergeAdditionalGrafanaConf(file.Name(), addition)
+	assert.Nil(t, err)
+	result, err := os.ReadFile(file.Name())
+	assert.Nil(t, err)
+
+	assert.Equal(t, expected, string(result))
+}

pkg/cluster/spec/spec.go

@@ -101,6 +101,7 @@ type (
 		Pump           map[string]interface{} `yaml:"pump"`
 		Drainer        map[string]interface{} `yaml:"drainer"`
 		CDC            map[string]interface{} `yaml:"cdc"`
+		Grafana        map[string]string      `yaml:"grafana"`
 	}
 
 	// Specification represents the specification of topology.yaml
@@ -153,6 +154,7 @@ type Topology interface {
 	TLSConfig(dir string) (*tls.Config, error)
 	Merge(that Topology) Topology
 	FillHostArch(hostArchmap map[string]string) error
+	GetGrafanaConfig() map[string]string
 
 	ScaleOutTopology
 }
@@ -856,3 +858,8 @@ func setHostArch(field reflect.Value, hostArch map[string]string) error {
 
 	return nil
 }
+
+// GetGrafanaConfig returns global grafana configurations
+func (s *Specification) GetGrafanaConfig() map[string]string {
+	return s.ServerConfigs.Grafana
+}

pkg/cluster/spec/spec_manager_test.go

@@ -116,6 +116,9 @@ func (t *TestTopology) CountDir(host string, dir string) int {
 	return 0
 }
 
+func (t *TestTopology) GetGrafanaConfig() map[string]string {
+	return nil
+}
 func TestSpec(t *testing.T) {
 	dir, err := os.MkdirTemp("", "test-*")
 	assert.Nil(t, err)