From d6122b8346e128e6d3679f46cc0d8eeeff3cd98a Mon Sep 17 00:00:00 2001 From: qiffang <947321353@qq.com> Date: Fri, 6 Sep 2019 13:58:44 +0800 Subject: [PATCH 1/8] Monitoring: add node and pod information in tidb cluster grafana Signed-off-by: qiffang <947321353@qq.com> --- .../templates/monitor-configmap.yaml | 2 -- .../templates/monitor-deployment.yaml | 23 +++++++++++++------ charts/tidb-cluster/values.yaml | 6 +++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/charts/tidb-cluster/templates/monitor-configmap.yaml b/charts/tidb-cluster/templates/monitor-configmap.yaml index 771a90808b..8d6ea1c4b8 100644 --- a/charts/tidb-cluster/templates/monitor-configmap.yaml +++ b/charts/tidb-cluster/templates/monitor-configmap.yaml @@ -13,8 +13,6 @@ data: prometheus-config: |- {{ tuple "config/_prometheus-config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- if .Values.monitor.grafana.create }} - datasource-config: |- -{{ tuple "config/_grafana-datasource.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} dashboard-config: |- {{ tuple "config/_grafana-dashboard.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/charts/tidb-cluster/templates/monitor-deployment.yaml b/charts/tidb-cluster/templates/monitor-deployment.yaml index e2f520f193..6161f48d34 100644 --- a/charts/tidb-cluster/templates/monitor-deployment.yaml +++ b/charts/tidb-cluster/templates/monitor-deployment.yaml @@ -56,12 +56,22 @@ spec: env: - name: GF_PROVISIONING_PATH value: /grafana-dashboard-definitions/tidb + - name: GF_DATASOURCE_PATH + value: /etc/grafana/provisioning/datasources - name: TIDB_CLUSTER_NAME value: {{ template "cluster.name" . }} - name: TIDB_ENABLE_BINLOG value: {{ .Values.binlog.pump.create | default false | quote }} + - name: TIDB_ENABLE_IMPORTER + value: {{ .Values.importer.create | default false | quote }} - name: PROM_CONFIG_PATH value: /prometheus-rules + - name: GF_NODE_EXPORTER_DATASOURCE_URL + value: {{ .Values.monitor.initializer.config.NODE_EXPORTER_PROM_URL }} + - name: GF_TIDB_CLUSTER_URL + value: http://127.0.0.1:9090 + - name: TIDB_CLUSTER_NAMESPACE + value: {{ .Release.Namespace }} command: - /bin/sh - -c @@ -80,6 +90,9 @@ spec: readOnly: false - name: monitor-data mountPath: /data + - name: grafana-datasources + mountPath: /etc/grafana/provisioning/datasources + readOnly: false resources: {{ toYaml .Values.monitor.initializer.resources | indent 10 }} containers: @@ -171,7 +184,7 @@ spec: - name: monitor-data mountPath: /data - mountPath: /etc/grafana/provisioning/datasources - name: datasource + name: grafana-datasources readOnly: false - mountPath: /etc/grafana/provisioning/dashboards name: dashboards-provisioning @@ -195,12 +208,8 @@ spec: - key: prometheus-config path: prometheus.yml {{- if .Values.monitor.grafana.create }} - - configMap: - name: {{ template "cluster.name" . }}-monitor - items: - - key: datasource-config - path: datasource.yaml - name: datasource + - emptyDir: {} + name: grafana-datasources - configMap: name: {{ template "cluster.name" . }}-monitor items: diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 120d2730db..ef195c835e 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -358,8 +358,10 @@ monitor: storageClassName: local-storage storage: 10Gi initializer: - image: pingcap/tidb-monitor-initializer:v3.0.1 - imagePullPolicy: IfNotPresent + image: pingcap/tidb-monitor-initializer:v3.0.3 + imagePullPolicy: Always + config: + NODE_EXPORTER_PROM_URL: http://prometheus-k8s.monitoring.svc:9090 resources: {} # limits: # cpu: 50m From 75110120ef486dc4307f7c0056f4e2fed696cc49 Mon Sep 17 00:00:00 2001 From: qiffang <947321353@qq.com> Date: Fri, 6 Sep 2019 13:59:06 +0800 Subject: [PATCH 2/8] Monitoring: add node and pod information in tidb cluster grafana Signed-off-by: qiffang <947321353@qq.com> --- charts/tidb-cluster/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index ef195c835e..c01908145d 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -358,7 +358,7 @@ monitor: storageClassName: local-storage storage: 10Gi initializer: - image: pingcap/tidb-monitor-initializer:v3.0.3 + image: pingcap/tidb-monitor-initializer:v3.0.1 imagePullPolicy: Always config: NODE_EXPORTER_PROM_URL: http://prometheus-k8s.monitoring.svc:9090 From cb1dee5de0f40dea45546a6880f12cf8f3043a36 Mon Sep 17 00:00:00 2001 From: qiffang <947321353@qq.com> Date: Fri, 6 Sep 2019 14:47:52 +0800 Subject: [PATCH 3/8] Monitoring: add node and pod information in tidb cluster grafana Signed-off-by: qiffang <947321353@qq.com> --- .../templates/config/_grafana-datasource.tpl | 14 -------------- .../tidb-cluster/templates/monitor-deployment.yaml | 4 ++-- 2 files changed, 2 insertions(+), 16 deletions(-) delete mode 100644 charts/tidb-cluster/templates/config/_grafana-datasource.tpl diff --git a/charts/tidb-cluster/templates/config/_grafana-datasource.tpl b/charts/tidb-cluster/templates/config/_grafana-datasource.tpl deleted file mode 100644 index 5713bfb6c7..0000000000 --- a/charts/tidb-cluster/templates/config/_grafana-datasource.tpl +++ /dev/null @@ -1,14 +0,0 @@ -{ - "apiVersion": 1, - "datasources": [ - { - "access": "proxy", - "editable": false, - "name": "tidb-cluster", - "orgId": 1, - "type": "prometheus", - "url": "http://127.0.0.1:9090", - "version": 1 - } - ] -} diff --git a/charts/tidb-cluster/templates/monitor-deployment.yaml b/charts/tidb-cluster/templates/monitor-deployment.yaml index 6161f48d34..1c0c2dd9be 100644 --- a/charts/tidb-cluster/templates/monitor-deployment.yaml +++ b/charts/tidb-cluster/templates/monitor-deployment.yaml @@ -66,9 +66,9 @@ spec: value: {{ .Values.importer.create | default false | quote }} - name: PROM_CONFIG_PATH value: /prometheus-rules - - name: GF_NODE_EXPORTER_DATASOURCE_URL + - name: GF_K8S_PROMETHEUS_URL value: {{ .Values.monitor.initializer.config.NODE_EXPORTER_PROM_URL }} - - name: GF_TIDB_CLUSTER_URL + - name: GF_TIDB_PROMETHEUS_URL value: http://127.0.0.1:9090 - name: TIDB_CLUSTER_NAMESPACE value: {{ .Release.Namespace }} From a104f9ef564fe60676d8dc14ad853084daf3de2b Mon Sep 17 00:00:00 2001 From: qiffang <947321353@qq.com> Date: Mon, 16 Sep 2019 11:42:29 +0800 Subject: [PATCH 4/8] Monitoring-Add-Node-Port-Info-In-TIDB-Grafana Signed-off-by: qiffang <947321353@qq.com> --- charts/tidb-cluster/templates/monitor-deployment.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/charts/tidb-cluster/templates/monitor-deployment.yaml b/charts/tidb-cluster/templates/monitor-deployment.yaml index 1c0c2dd9be..fee88d1311 100644 --- a/charts/tidb-cluster/templates/monitor-deployment.yaml +++ b/charts/tidb-cluster/templates/monitor-deployment.yaml @@ -62,8 +62,6 @@ spec: value: {{ template "cluster.name" . }} - name: TIDB_ENABLE_BINLOG value: {{ .Values.binlog.pump.create | default false | quote }} - - name: TIDB_ENABLE_IMPORTER - value: {{ .Values.importer.create | default false | quote }} - name: PROM_CONFIG_PATH value: /prometheus-rules - name: GF_K8S_PROMETHEUS_URL @@ -90,7 +88,7 @@ spec: readOnly: false - name: monitor-data mountPath: /data - - name: grafana-datasources + - name: datasource mountPath: /etc/grafana/provisioning/datasources readOnly: false resources: @@ -184,7 +182,7 @@ spec: - name: monitor-data mountPath: /data - mountPath: /etc/grafana/provisioning/datasources - name: grafana-datasources + name: datasource readOnly: false - mountPath: /etc/grafana/provisioning/dashboards name: dashboards-provisioning @@ -209,7 +207,7 @@ spec: path: prometheus.yml {{- if .Values.monitor.grafana.create }} - emptyDir: {} - name: grafana-datasources + name: datasource - configMap: name: {{ template "cluster.name" . }}-monitor items: From c4b4e91b96ed600e35eab549dc79951fe81a71c9 Mon Sep 17 00:00:00 2001 From: qiffang <947321353@qq.com> Date: Mon, 16 Sep 2019 11:52:21 +0800 Subject: [PATCH 5/8] Monitoring-Add-Node-Port-Info-In-TIDB-Grafana Signed-off-by: qiffang <947321353@qq.com> --- charts/tidb-cluster/templates/monitor-deployment.yaml | 2 +- charts/tidb-cluster/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/tidb-cluster/templates/monitor-deployment.yaml b/charts/tidb-cluster/templates/monitor-deployment.yaml index fee88d1311..d8952b46a3 100644 --- a/charts/tidb-cluster/templates/monitor-deployment.yaml +++ b/charts/tidb-cluster/templates/monitor-deployment.yaml @@ -65,7 +65,7 @@ spec: - name: PROM_CONFIG_PATH value: /prometheus-rules - name: GF_K8S_PROMETHEUS_URL - value: {{ .Values.monitor.initializer.config.NODE_EXPORTER_PROM_URL }} + value: {{ .Values.monitor.initializer.config.K8S_PROMETHEUS_URL }} - name: GF_TIDB_PROMETHEUS_URL value: http://127.0.0.1:9090 - name: TIDB_CLUSTER_NAMESPACE diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index c01908145d..feed7108db 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -361,7 +361,7 @@ monitor: image: pingcap/tidb-monitor-initializer:v3.0.1 imagePullPolicy: Always config: - NODE_EXPORTER_PROM_URL: http://prometheus-k8s.monitoring.svc:9090 + K8S_PROMETHEUS_URL: http://prometheus-k8s.monitoring.svc:9090 resources: {} # limits: # cpu: 50m From 5691a21bfc1d50f9e8bc461f828b90381210e4b5 Mon Sep 17 00:00:00 2001 From: qiffang <947321353@qq.com> Date: Thu, 26 Sep 2019 16:55:43 +0800 Subject: [PATCH 6/8] Monitoring: Add Node & Pod info in TiDB Grafana --- tests/actions.go | 49 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tests/actions.go b/tests/actions.go index e429906e5a..48f7f46c43 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -1750,7 +1750,13 @@ func (oa *operatorActions) checkGrafanaData(clusterInfo *TidbClusterConfig) erro values.Set("start", fmt.Sprintf("%d", start.Unix())) values.Set("end", fmt.Sprintf("%d", end.Unix())) values.Set("step", "30") - u := fmt.Sprintf("http://%s.%s.svc.cluster.local:3000/api/datasources/proxy/1/api/v1/query_range?%s", svcName, ns, values.Encode()) + + datasourceID, err := getDatasourceID(svcName, ns) + if err != nil { + return err + } + + u := fmt.Sprintf("http://%s.%s.svc.cluster.local:3000/api/datasources/proxy/%d/api/v1/query_range?%s", svcName, ns, datasourceID, values.Encode()) req, err := http.NewRequest(http.MethodGet, u, nil) if err != nil { return err @@ -1797,6 +1803,47 @@ func (oa *operatorActions) checkGrafanaData(clusterInfo *TidbClusterConfig) erro return nil } +func getDatasourceID(svcName, namespace string) (int, error) { + u := fmt.Sprintf("http://%s.%s.svc.cluster.local:3000/api/datasources", svcName, namespace) + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + return 0, err + } + + req.SetBasicAuth(grafanaUsername, grafanaPassword) + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return 0, err + } + defer func() { + err := resp.Body.Close() + glog.Warning("close reponse failed", err) + }() + + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return 0, err + } + + datasources := []struct { + Id int `json:"id"` + Name string `json:"name"` + }{} + + if err := json.Unmarshal(buf, &datasources); err != nil { + return 0, err + } + + for _, ds := range datasources { + if ds.Name == "tidb-cluster" { + return ds.Id + } + } + + return 0, pingcapErrors.New("not found tidb-cluster datasource") +} + func GetD(ns, tcName, databaseName, password string) string { return fmt.Sprintf("root:%s@(%s-tidb.%s:4000)/%s?charset=utf8", password, tcName, ns, databaseName) } From a534ba8377b81fbc057d306985f11cebebe50510 Mon Sep 17 00:00:00 2001 From: qiffang <947321353@qq.com> Date: Fri, 27 Sep 2019 10:56:39 +0800 Subject: [PATCH 7/8] Monitoring: Add Node & Pod info in TiDB Grafana --- tests/actions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/actions.go b/tests/actions.go index 48f7f46c43..630e7ee80e 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -1837,7 +1837,7 @@ func getDatasourceID(svcName, namespace string) (int, error) { for _, ds := range datasources { if ds.Name == "tidb-cluster" { - return ds.Id + return ds.Id, nil } } From 97259e9ca1ee5a44c73f3beca54e405c9c5288e2 Mon Sep 17 00:00:00 2001 From: qiffang <947321353@qq.com> Date: Fri, 27 Sep 2019 11:25:25 +0800 Subject: [PATCH 8/8] Monitoring: Add Node & Pod info in TiDB Grafana --- tests/actions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/actions.go b/tests/actions.go index 630e7ee80e..598d2d50d6 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -1818,7 +1818,7 @@ func getDatasourceID(svcName, namespace string) (int, error) { } defer func() { err := resp.Body.Close() - glog.Warning("close reponse failed", err) + glog.Warning("close response failed", err) }() buf, err := ioutil.ReadAll(resp.Body)