Monitoring: Add Node & Pod info in TiDB Grafana

charts/tidb-cluster/templates/monitor-configmap.yaml

@@ -13,8 +13,6 @@ data:
   prometheus-config: |-
 {{ tuple "config/_prometheus-config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- if .Values.monitor.grafana.create }}
-  datasource-config: |-
-{{ tuple "config/_grafana-datasource.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   dashboard-config: |-
 {{ tuple "config/_grafana-dashboard.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

charts/tidb-cluster/templates/monitor-deployment.yaml

@@ -56,12 +56,20 @@ spec:
         env:
         - name: GF_PROVISIONING_PATH
           value: /grafana-dashboard-definitions/tidb
+        - name: GF_DATASOURCE_PATH
+          value: /etc/grafana/provisioning/datasources
         - name: TIDB_CLUSTER_NAME
           value: {{ template "cluster.name" . }}
         - name: TIDB_ENABLE_BINLOG
           value: {{ .Values.binlog.pump.create | default false | quote }}
         - name:  PROM_CONFIG_PATH
           value: /prometheus-rules
+        - name: GF_K8S_PROMETHEUS_URL
+          value: {{ .Values.monitor.initializer.config.K8S_PROMETHEUS_URL }}
+        - name: GF_TIDB_PROMETHEUS_URL
+          value: http://127.0.0.1:9090
+        - name: TIDB_CLUSTER_NAMESPACE
+          value: {{ .Release.Namespace }}
         command:
         - /bin/sh
         - -c
@@ -80,6 +88,9 @@ spec:
           readOnly: false
         - name: monitor-data
           mountPath: /data
+        - name: datasource
+          mountPath: /etc/grafana/provisioning/datasources
+          readOnly: false
         resources:
 {{ toYaml .Values.monitor.initializer.resources | indent 10 }}
       containers:
@@ -195,11 +206,7 @@ spec:
           - key: prometheus-config
             path: prometheus.yml
       {{- if .Values.monitor.grafana.create }}
-      - configMap:
-          name: {{ template "cluster.name" . }}-monitor
-          items:
-          - key: datasource-config
-            path: datasource.yaml
+      - emptyDir: {}
         name: datasource
       - configMap:
           name: {{ template "cluster.name" . }}-monitor

charts/tidb-cluster/values.yaml

@@ -394,7 +394,9 @@ monitor:
   storage: 10Gi
   initializer:
     image: pingcap/tidb-monitor-initializer:v3.0.1
-    imagePullPolicy: IfNotPresent
+    imagePullPolicy: Always
+    config:
+      K8S_PROMETHEUS_URL: http://prometheus-k8s.monitoring.svc:9090
     resources: {}
     # limits:
     #  cpu: 50m

tests/actions.go

@@ -1750,7 +1750,13 @@ func (oa *operatorActions) checkGrafanaData(clusterInfo *TidbClusterConfig) erro
 	values.Set("start", fmt.Sprintf("%d", start.Unix()))
 	values.Set("end", fmt.Sprintf("%d", end.Unix()))
 	values.Set("step", "30")
-	u := fmt.Sprintf("http://%s.%s.svc.cluster.local:3000/api/datasources/proxy/1/api/v1/query_range?%s", svcName, ns, values.Encode())
+
+	datasourceID, err := getDatasourceID(svcName, ns)
+	if err != nil {
+		return err
+	}
+
+	u := fmt.Sprintf("http://%s.%s.svc.cluster.local:3000/api/datasources/proxy/%d/api/v1/query_range?%s", svcName, ns, datasourceID, values.Encode())
 	req, err := http.NewRequest(http.MethodGet, u, nil)
 	if err != nil {
 		return err
@@ -1797,6 +1803,47 @@ func (oa *operatorActions) checkGrafanaData(clusterInfo *TidbClusterConfig) erro
 	return nil
 }
 
+func getDatasourceID(svcName, namespace string) (int, error) {
+	u := fmt.Sprintf("http://%s.%s.svc.cluster.local:3000/api/datasources", svcName, namespace)
+	req, err := http.NewRequest(http.MethodGet, u, nil)
+	if err != nil {
+		return 0, err
+	}
+
+	req.SetBasicAuth(grafanaUsername, grafanaPassword)
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, err
+	}
+	defer func() {
+		err := resp.Body.Close()
+		glog.Warning("close response failed", err)
+	}()
+
+	buf, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return 0, err
+	}
+
+	datasources := []struct {
+		Id   int    `json:"id"`
+		Name string `json:"name"`
+	}{}
+
+	if err := json.Unmarshal(buf, &datasources); err != nil {
+		return 0, err
+	}
+
+	for _, ds := range datasources {
+		if ds.Name == "tidb-cluster" {
+			return ds.Id, nil
+		}
+	}
+
+	return 0, pingcapErrors.New("not found tidb-cluster datasource")
+}
+
 func GetD(ns, tcName, databaseName, password string) string {
 	return fmt.Sprintf("root:%s@(%s-tidb.%s:4000)/%s?charset=utf8", password, tcName, ns, databaseName)
 }