diff --git a/components/dm/spec/topology_dm.go b/components/dm/spec/topology_dm.go
index 6870fe12fd..1b06f4c11f 100644
--- a/components/dm/spec/topology_dm.go
+++ b/components/dm/spec/topology_dm.go
@@ -89,8 +89,9 @@ type (
 type (
 	// DMServerConfigs represents the server runtime configuration
 	DMServerConfigs struct {
-		Master map[string]interface{} `yaml:"master"`
-		Worker map[string]interface{} `yaml:"worker"`
+		Master  map[string]interface{} `yaml:"master"`
+		Worker  map[string]interface{} `yaml:"worker"`
+		Grafana map[string]string      `yaml:"grafana"`
 	}
 
 	// Specification represents the specification of topology.yaml
@@ -838,3 +839,8 @@ func getPort(v reflect.Value) string {
 	}
 	return ""
 }
+
+// GetGrafanaConfig returns global grafana configurations
+func (s *Specification) GetGrafanaConfig() map[string]string {
+	return s.ServerConfigs.Grafana
+}
diff --git a/embed/examples/cluster/minimal.yaml b/embed/examples/cluster/minimal.yaml
index b8c11ff335..447f7943d9 100644
--- a/embed/examples/cluster/minimal.yaml
+++ b/embed/examples/cluster/minimal.yaml
@@ -224,6 +224,8 @@ monitoring_servers:
     # ssh_port: 22
     # # Prometheus Service communication port.
     # port: 9090
+    # # ng-monitoring servive communication port
+    # ng_port: 12020
     # # Prometheus deployment file, startup script, configuration file storage directory.
     # deploy_dir: "/tidb-deploy/prometheus-8249"
     # # Prometheus data storage directory.
diff --git a/embed/examples/cluster/multi-dc.yaml b/embed/examples/cluster/multi-dc.yaml
index 05013d42e6..37e8ded064 100644
--- a/embed/examples/cluster/multi-dc.yaml
+++ b/embed/examples/cluster/multi-dc.yaml
@@ -276,6 +276,8 @@ monitoring_servers:
     # ssh_port: 22
     # # Prometheus Service communication port.
     # port: 9090
+    # # ng-monitoring servive communication port
+    # ng_port: 12020
     # # Prometheus deployment file, startup script, configuration file storage directory.
     # deploy_dir: "/tidb-deploy/prometheus-8249"
     # # Prometheus data storage directory.
diff --git a/embed/examples/cluster/topology.example.yaml b/embed/examples/cluster/topology.example.yaml
index b2cf3d301c..bebc7f7487 100644
--- a/embed/examples/cluster/topology.example.yaml
+++ b/embed/examples/cluster/topology.example.yaml
@@ -286,6 +286,8 @@ monitoring_servers:
     # ssh_port: 22
     # # Prometheus Service communication port.
     # port: 9090
+    # # ng-monitoring servive communication port
+    # ng_port: 12020
     # # Prometheus deployment file, startup script, configuration file storage directory.
     # deploy_dir: "/tidb-deploy/prometheus-8249"
     # # Prometheus data storage directory.
diff --git a/pkg/cluster/ansible/import_test.go b/pkg/cluster/ansible/import_test.go
index f02a2b4bb6..e06c24da32 100644
--- a/pkg/cluster/ansible/import_test.go
+++ b/pkg/cluster/ansible/import_test.go
@@ -129,6 +129,7 @@ server_configs:
     pump: {}
     drainer: {}
     cdc: {}
+    grafana: {}
 tidb_servers: []
 tikv_servers: []
 tiflash_servers: []
diff --git a/pkg/cluster/ansible/test-data/meta.yaml b/pkg/cluster/ansible/test-data/meta.yaml
index 2239b5b4d3..d179ff869e 100644
--- a/pkg/cluster/ansible/test-data/meta.yaml
+++ b/pkg/cluster/ansible/test-data/meta.yaml
@@ -24,6 +24,7 @@ topology:
     pump: {}
     drainer: {}
     cdc: {}
+    grafana: {}
   tidb_servers:
     - host: 172.16.1.218
       ssh_port: 9999
diff --git a/pkg/cluster/manager/upgrade.go b/pkg/cluster/manager/upgrade.go
index a6e259c29d..f3f4de863d 100644
--- a/pkg/cluster/manager/upgrade.go
+++ b/pkg/cluster/manager/upgrade.go
@@ -53,6 +53,11 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio
 	topo := metadata.GetTopology()
 	base := metadata.GetBaseMeta()
 
+	// Adjust topo by new version
+	if clusterTopo, ok := topo.(*spec.Specification); ok {
+		clusterTopo.AdjustByVersion(clusterVersion)
+	}
+
 	var (
 		downloadCompTasks []task.Task // tasks which are used to download components
 		copyCompTasks     []task.Task // tasks which are used to copy components to remote host
diff --git a/pkg/cluster/spec/grafana.go b/pkg/cluster/spec/grafana.go
index 285ddc44a7..6158dd0eb6 100644
--- a/pkg/cluster/spec/grafana.go
+++ b/pkg/cluster/spec/grafana.go
@@ -17,6 +17,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -27,6 +28,7 @@ import (
 	"github.com/pingcap/tiup/pkg/cluster/template/config"
 	"github.com/pingcap/tiup/pkg/cluster/template/scripts"
 	"github.com/pingcap/tiup/pkg/meta"
+	"gopkg.in/ini.v1"
 )
 
 // GrafanaSpec represents the Grafana topology specification in topology.yaml
@@ -38,6 +40,7 @@ type GrafanaSpec struct {
 	IgnoreExporter  bool                 `yaml:"ignore_exporter,omitempty"`
 	Port            int                  `yaml:"port" default:"3000"`
 	DeployDir       string               `yaml:"deploy_dir,omitempty"`
+	Config          map[string]string    `yaml:"config,omitempty" validate:"config:ignore"`
 	ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"`
 	Arch            string               `yaml:"arch,omitempty"`
 	OS              string               `yaml:"os,omitempty"`
@@ -181,11 +184,22 @@ func (i *GrafanaInstance) InitConfig(
 		return err
 	}
 
+	userConfig := i.topo.GetGrafanaConfig()
+	if userConfig == nil {
+		userConfig = make(map[string]string)
+	}
+	for k, v := range spec.Config {
+		userConfig[k] = v
+	}
+	err := mergeAdditionalGrafanaConf(fp, userConfig)
+	if err != nil {
+		return err
+	}
+
 	dst = filepath.Join(paths.Deploy, "conf", "grafana.ini")
 	if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil {
 		return err
 	}
-
 	if err := i.installDashboards(ctx, e, paths.Deploy, clusterName, clusterVersion); err != nil {
 		return errors.Annotate(err, "install dashboards")
 	}
@@ -330,3 +344,24 @@ func (i *GrafanaInstance) ScaleConfig(
 	i.topo = topo.Merge(i.topo)
 	return i.InitConfig(ctx, e, clusterName, clusterVersion, deployUser, paths)
 }
+
+func mergeAdditionalGrafanaConf(source string, addition map[string]string) error {
+	bytes, err := os.ReadFile(source)
+	if err != nil {
+		return err
+	}
+	result, err := ini.Load(bytes)
+	if err != nil {
+		return err
+	}
+	for k, v := range addition {
+		// convert "log.file.level to [log.file] level"
+		for i := len(k) - 1; i >= 0; i-- {
+			if k[i] == '.' {
+				result.Section(k[:i]).Key(k[i+1:]).SetValue(v)
+				break
+			}
+		}
+	}
+	return result.SaveTo(source)
+}
diff --git a/pkg/cluster/spec/grafana_test.go b/pkg/cluster/spec/grafana_test.go
index 437c54a58a..66c677035c 100644
--- a/pkg/cluster/spec/grafana_test.go
+++ b/pkg/cluster/spec/grafana_test.go
@@ -15,6 +15,7 @@ package spec
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/user"
 	"path"
@@ -67,3 +68,93 @@ func TestLocalDashboards(t *testing.T) {
 		assert.FileExists(t, path.Join(deployDir, "dashboards", f.Name()))
 	}
 }
+
+func TestMergeAdditionalGrafanaConf(t *testing.T) {
+	file, err := os.CreateTemp("", "tiup-cluster-spec-test")
+	if err != nil {
+		panic(fmt.Sprintf("create temp file: %s", err))
+	}
+	defer os.Remove(file.Name())
+
+	_, err = file.WriteString(`#################################### SMTP / Emailing ##########################
+[smtp]
+;enabled = false
+;host = localhost:25
+;user =
+;password =
+;cert_file =
+;key_file =
+;skip_verify = false
+;from_address = admin@grafana.localhost
+
+[emails]
+;welcome_email_on_sign_up = false
+
+#################################### Logging ##########################
+[log]
+# Either "console", "file", "syslog". Default is console and  file
+# Use space to separate multiple modes, e.g. "console file"
+mode = file
+
+# Either "trace", "debug", "info", "warn", "error", "critical", default is "info"
+;level = info
+# For "console" mode only
+[log.console]
+;level =
+
+# log line format, valid options are text, console and json
+;format = console
+
+# For "file" mode only
+[log.file]
+level = info
+`)
+	assert.Nil(t, err)
+
+	expected := `# ################################### SMTP / Emailing ##########################
+[smtp]
+enabled = true
+
+; enabled = false
+; host = localhost:25
+; user =
+; password =
+; cert_file =
+; key_file =
+; skip_verify = false
+; from_address = admin@grafana.localhost
+[emails]
+
+; welcome_email_on_sign_up = false
+# ################################### Logging ##########################
+[log]
+# Either "console", "file", "syslog". Default is console and  file
+# Use space to separate multiple modes, e.g. "console file"
+mode = file
+
+# Either "trace", "debug", "info", "warn", "error", "critical", default is "info"
+; level = info
+# For "console" mode only
+[log.console]
+
+; level =
+# log line format, valid options are text, console and json
+; format = console
+# For "file" mode only
+[log.file]
+level = warning
+
+`
+
+	addition := map[string]string{
+		"log.file.level": "warning",
+		"smtp.enabled":   "true",
+	}
+
+	err = mergeAdditionalGrafanaConf(file.Name(), addition)
+	assert.Nil(t, err)
+	result, err := os.ReadFile(file.Name())
+	assert.Nil(t, err)
+
+	assert.Equal(t, expected, string(result))
+}
diff --git a/pkg/cluster/spec/monitoring.go b/pkg/cluster/spec/monitoring.go
index dcde45027b..da01b25c44 100644
--- a/pkg/cluster/spec/monitoring.go
+++ b/pkg/cluster/spec/monitoring.go
@@ -41,7 +41,7 @@ type PrometheusSpec struct {
 	Patched               bool                   `yaml:"patched,omitempty"`
 	IgnoreExporter        bool                   `yaml:"ignore_exporter,omitempty"`
 	Port                  int                    `yaml:"port" default:"9090"`
-	NgPort                int                    `yaml:"ng_port,omitempty" validate:"ng_port:editable"`
+	NgPort                int                    `yaml:"ng_port,omitempty" validate:"ng_port:editable"` // ng_port is usable since v5.3.0 and default as 12020 since v5.4.0, so the default value is set in spec.go/AdjustByVersion
 	DeployDir             string                 `yaml:"deploy_dir,omitempty"`
 	DataDir               string                 `yaml:"data_dir,omitempty"`
 	LogDir                string                 `yaml:"log_dir,omitempty"`
diff --git a/pkg/cluster/spec/spec.go b/pkg/cluster/spec/spec.go
index b93a9b7688..f8b6370e87 100644
--- a/pkg/cluster/spec/spec.go
+++ b/pkg/cluster/spec/spec.go
@@ -101,6 +101,7 @@ type (
 		Pump           map[string]interface{} `yaml:"pump"`
 		Drainer        map[string]interface{} `yaml:"drainer"`
 		CDC            map[string]interface{} `yaml:"cdc"`
+		Grafana        map[string]string      `yaml:"grafana"`
 	}
 
 	// Specification represents the specification of topology.yaml
@@ -153,6 +154,7 @@ type Topology interface {
 	TLSConfig(dir string) (*tls.Config, error)
 	Merge(that Topology) Topology
 	FillHostArch(hostArchmap map[string]string) error
+	GetGrafanaConfig() map[string]string
 
 	ScaleOutTopology
 }
@@ -404,6 +406,13 @@ func (s *Specification) AdjustByVersion(clusterVersion string) {
 			server.DataDir = ""
 		}
 	}
+	if semver.Compare(clusterVersion, "v5.4.0") >= 0 {
+		for _, m := range s.Monitors {
+			if m.NgPort == 0 {
+				m.NgPort = 12020
+			}
+		}
+	}
 }
 
 // GetDashboardAddress returns the cluster's dashboard addr
@@ -874,3 +883,8 @@ func (s *Specification) removeCommitTS() {
 		spec.CommitTS = nil
 	}
 }
+
+// GetGrafanaConfig returns global grafana configurations
+func (s *Specification) GetGrafanaConfig() map[string]string {
+	return s.ServerConfigs.Grafana
+}
diff --git a/pkg/cluster/spec/spec_manager_test.go b/pkg/cluster/spec/spec_manager_test.go
index fcb52fe0ad..f9f48cb392 100644
--- a/pkg/cluster/spec/spec_manager_test.go
+++ b/pkg/cluster/spec/spec_manager_test.go
@@ -116,6 +116,9 @@ func (t *TestTopology) CountDir(host string, dir string) int {
 	return 0
 }
 
+func (t *TestTopology) GetGrafanaConfig() map[string]string {
+	return nil
+}
 func TestSpec(t *testing.T) {
 	dir, err := os.MkdirTemp("", "test-*")
 	assert.Nil(t, err)
diff --git a/pkg/cluster/spec/validate.go b/pkg/cluster/spec/validate.go
index e751903366..4c289d4674 100644
--- a/pkg/cluster/spec/validate.go
+++ b/pkg/cluster/spec/validate.go
@@ -568,9 +568,8 @@ func (s *Specification) portInvalidDetect() error {
 		for i := 0; i < compSpec.NumField(); i++ {
 			if strings.HasSuffix(compSpec.Type().Field(i).Name, "Port") {
 				port := int(compSpec.Field(i).Int())
-				portTags := strings.Split(compSpec.Type().Field(i).Tag.Get("yaml"), ",")
-				// when use not specify ng_port, its default value is 0
-				if port == 0 && len(portTags) > 1 && portTags[1] == "omitempty" {
+				// for NgPort, 0 means default and -1 means disable
+				if compSpec.Type().Field(i).Name == "NgPort" && (port == -1 || port == 0) {
 					continue
 				}
 				if port < 1 || port > 65535 {