From e2922452d1feed9baf1bacbb7240480d4409adb7 Mon Sep 17 00:00:00 2001 From: nexustar Date: Thu, 6 Jan 2022 16:40:37 +0800 Subject: [PATCH] cluster: support configure grafana in server_configs (#1703) --- components/dm/spec/topology_dm.go | 10 ++- pkg/cluster/ansible/import_test.go | 1 + pkg/cluster/ansible/test-data/meta.yaml | 1 + pkg/cluster/spec/grafana.go | 37 +++++++++- pkg/cluster/spec/grafana_test.go | 91 +++++++++++++++++++++++++ pkg/cluster/spec/spec.go | 7 ++ pkg/cluster/spec/spec_manager_test.go | 3 + 7 files changed, 147 insertions(+), 3 deletions(-) diff --git a/components/dm/spec/topology_dm.go b/components/dm/spec/topology_dm.go index 6870fe12fd..1b06f4c11f 100644 --- a/components/dm/spec/topology_dm.go +++ b/components/dm/spec/topology_dm.go @@ -89,8 +89,9 @@ type ( type ( // DMServerConfigs represents the server runtime configuration DMServerConfigs struct { - Master map[string]interface{} `yaml:"master"` - Worker map[string]interface{} `yaml:"worker"` + Master map[string]interface{} `yaml:"master"` + Worker map[string]interface{} `yaml:"worker"` + Grafana map[string]string `yaml:"grafana"` } // Specification represents the specification of topology.yaml @@ -838,3 +839,8 @@ func getPort(v reflect.Value) string { } return "" } + +// GetGrafanaConfig returns global grafana configurations +func (s *Specification) GetGrafanaConfig() map[string]string { + return s.ServerConfigs.Grafana +} diff --git a/pkg/cluster/ansible/import_test.go b/pkg/cluster/ansible/import_test.go index f02a2b4bb6..e06c24da32 100644 --- a/pkg/cluster/ansible/import_test.go +++ b/pkg/cluster/ansible/import_test.go @@ -129,6 +129,7 @@ server_configs: pump: {} drainer: {} cdc: {} + grafana: {} tidb_servers: [] tikv_servers: [] tiflash_servers: [] diff --git a/pkg/cluster/ansible/test-data/meta.yaml b/pkg/cluster/ansible/test-data/meta.yaml index 2239b5b4d3..d179ff869e 100644 --- a/pkg/cluster/ansible/test-data/meta.yaml +++ b/pkg/cluster/ansible/test-data/meta.yaml @@ -24,6 +24,7 @@ topology: pump: {} drainer: {} cdc: {} + grafana: {} tidb_servers: - host: 172.16.1.218 ssh_port: 9999 diff --git a/pkg/cluster/spec/grafana.go b/pkg/cluster/spec/grafana.go index 285ddc44a7..6158dd0eb6 100644 --- a/pkg/cluster/spec/grafana.go +++ b/pkg/cluster/spec/grafana.go @@ -17,6 +17,7 @@ import ( "context" "crypto/tls" "fmt" + "os" "path/filepath" "reflect" "strings" @@ -27,6 +28,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/template/config" "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" + "gopkg.in/ini.v1" ) // GrafanaSpec represents the Grafana topology specification in topology.yaml @@ -38,6 +40,7 @@ type GrafanaSpec struct { IgnoreExporter bool `yaml:"ignore_exporter,omitempty"` Port int `yaml:"port" default:"3000"` DeployDir string `yaml:"deploy_dir,omitempty"` + Config map[string]string `yaml:"config,omitempty" validate:"config:ignore"` ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` Arch string `yaml:"arch,omitempty"` OS string `yaml:"os,omitempty"` @@ -181,11 +184,22 @@ func (i *GrafanaInstance) InitConfig( return err } + userConfig := i.topo.GetGrafanaConfig() + if userConfig == nil { + userConfig = make(map[string]string) + } + for k, v := range spec.Config { + userConfig[k] = v + } + err := mergeAdditionalGrafanaConf(fp, userConfig) + if err != nil { + return err + } + dst = filepath.Join(paths.Deploy, "conf", "grafana.ini") if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil { return err } - if err := i.installDashboards(ctx, e, paths.Deploy, clusterName, clusterVersion); err != nil { return errors.Annotate(err, "install dashboards") } @@ -330,3 +344,24 @@ func (i *GrafanaInstance) ScaleConfig( i.topo = topo.Merge(i.topo) return i.InitConfig(ctx, e, clusterName, clusterVersion, deployUser, paths) } + +func mergeAdditionalGrafanaConf(source string, addition map[string]string) error { + bytes, err := os.ReadFile(source) + if err != nil { + return err + } + result, err := ini.Load(bytes) + if err != nil { + return err + } + for k, v := range addition { + // convert "log.file.level to [log.file] level" + for i := len(k) - 1; i >= 0; i-- { + if k[i] == '.' { + result.Section(k[:i]).Key(k[i+1:]).SetValue(v) + break + } + } + } + return result.SaveTo(source) +} diff --git a/pkg/cluster/spec/grafana_test.go b/pkg/cluster/spec/grafana_test.go index 437c54a58a..66c677035c 100644 --- a/pkg/cluster/spec/grafana_test.go +++ b/pkg/cluster/spec/grafana_test.go @@ -15,6 +15,7 @@ package spec import ( "context" + "fmt" "os" "os/user" "path" @@ -67,3 +68,93 @@ func TestLocalDashboards(t *testing.T) { assert.FileExists(t, path.Join(deployDir, "dashboards", f.Name())) } } + +func TestMergeAdditionalGrafanaConf(t *testing.T) { + file, err := os.CreateTemp("", "tiup-cluster-spec-test") + if err != nil { + panic(fmt.Sprintf("create temp file: %s", err)) + } + defer os.Remove(file.Name()) + + _, err = file.WriteString(`#################################### SMTP / Emailing ########################## +[smtp] +;enabled = false +;host = localhost:25 +;user = +;password = +;cert_file = +;key_file = +;skip_verify = false +;from_address = admin@grafana.localhost + +[emails] +;welcome_email_on_sign_up = false + +#################################### Logging ########################## +[log] +# Either "console", "file", "syslog". Default is console and file +# Use space to separate multiple modes, e.g. "console file" +mode = file + +# Either "trace", "debug", "info", "warn", "error", "critical", default is "info" +;level = info +# For "console" mode only +[log.console] +;level = + +# log line format, valid options are text, console and json +;format = console + +# For "file" mode only +[log.file] +level = info +`) + assert.Nil(t, err) + + expected := `# ################################### SMTP / Emailing ########################## +[smtp] +enabled = true + +; enabled = false +; host = localhost:25 +; user = +; password = +; cert_file = +; key_file = +; skip_verify = false +; from_address = admin@grafana.localhost +[emails] + +; welcome_email_on_sign_up = false +# ################################### Logging ########################## +[log] +# Either "console", "file", "syslog". Default is console and file +# Use space to separate multiple modes, e.g. "console file" +mode = file + +# Either "trace", "debug", "info", "warn", "error", "critical", default is "info" +; level = info +# For "console" mode only +[log.console] + +; level = +# log line format, valid options are text, console and json +; format = console +# For "file" mode only +[log.file] +level = warning + +` + + addition := map[string]string{ + "log.file.level": "warning", + "smtp.enabled": "true", + } + + err = mergeAdditionalGrafanaConf(file.Name(), addition) + assert.Nil(t, err) + result, err := os.ReadFile(file.Name()) + assert.Nil(t, err) + + assert.Equal(t, expected, string(result)) +} diff --git a/pkg/cluster/spec/spec.go b/pkg/cluster/spec/spec.go index eb7e653321..985e8c3887 100644 --- a/pkg/cluster/spec/spec.go +++ b/pkg/cluster/spec/spec.go @@ -101,6 +101,7 @@ type ( Pump map[string]interface{} `yaml:"pump"` Drainer map[string]interface{} `yaml:"drainer"` CDC map[string]interface{} `yaml:"cdc"` + Grafana map[string]string `yaml:"grafana"` } // Specification represents the specification of topology.yaml @@ -153,6 +154,7 @@ type Topology interface { TLSConfig(dir string) (*tls.Config, error) Merge(that Topology) Topology FillHostArch(hostArchmap map[string]string) error + GetGrafanaConfig() map[string]string ScaleOutTopology } @@ -856,3 +858,8 @@ func setHostArch(field reflect.Value, hostArch map[string]string) error { return nil } + +// GetGrafanaConfig returns global grafana configurations +func (s *Specification) GetGrafanaConfig() map[string]string { + return s.ServerConfigs.Grafana +} diff --git a/pkg/cluster/spec/spec_manager_test.go b/pkg/cluster/spec/spec_manager_test.go index fcb52fe0ad..f9f48cb392 100644 --- a/pkg/cluster/spec/spec_manager_test.go +++ b/pkg/cluster/spec/spec_manager_test.go @@ -116,6 +116,9 @@ func (t *TestTopology) CountDir(host string, dir string) int { return 0 } +func (t *TestTopology) GetGrafanaConfig() map[string]string { + return nil +} func TestSpec(t *testing.T) { dir, err := os.MkdirTemp("", "test-*") assert.Nil(t, err)