-
Notifications
You must be signed in to change notification settings - Fork 0
/
call_prom.py
181 lines (153 loc) · 8.19 KB
/
call_prom.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
from subprocess import call, PIPE, run, os
import json, datetime, argparse
import matplotlib.pyplot as plt
import configparser
# home path
home_path = '/home/tommygood/telegram_bot'
# config
config = configparser.ConfigParser()
config.read(home_path + '/config.ini')
host = config["env"]["prometheus_host"]
ap = argparse.ArgumentParser()
ap.add_argument("-t", "--type", required = True, help = " : metric type") # get metric type
ap.add_argument("-n", "--namespace", required = False, help = " : specify namespace") # specify namespace
args = vars(ap.parse_args())
# prometheus server
#host = 'http://localhost:31111'
# metric type
metric_type = args['type']
# namespace
def getNamespace(namespace) :
if namespace == None or namespace == '' :
return ''
else :
return f"namespace='{namespace}'"
namespace = getNamespace(args['namespace'])
# time range
interval = "5h"
# total metric type
total_metric_type = ['podMemUseInNode', 'eachConatinerMemUsage', 'weirdPodNumInNamespace', 'runningPodNumInNamespace', 'nodeMemSecTotal', 'nodeCpuSecTotal', 'containerCpuPerSecTotal', 'conatinerPerCpuUsage', 'namespacePerPodCpuUsage']
def main() :
# check metric available
if not checkRightMetric() :
print('this metric is not available !')
return
# search with prometheus
result = eval(metric_type + "()")
# draw the graph, and output to a png
#print(result.stdout)
output_len = 0 # count the length of output png
# output each result with a png
for i in range(len(eval(result.stdout))) :
# data value
data = eval(result.stdout)[i]['values']
# data metric infomation
data_metric = list(eval(result.stdout)[i]['metric'].items())
# draw the graph with value and infomation
plot_graph(data, f'{home_path}/image/output{i}.png', metric_type, data_metric)
output_len += 1
# final quantity of data
print(f'Successfully output, total num of output is :{output_len}')
# check metric available
def checkRightMetric() :
for i in total_metric_type :
if i == metric_type :
return True
return False
# node exporter - node cpu
def nodeCpuSecTotal() :
# execute command
command = '100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'
result = run([home_path + "/microk8s/promql_cli/promql", "--host", host, command, "--start", interval, "--output", "json"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
return result
# kube-state exporter
# total container cpu usage percentage
def containerCpuPerSecTotal() :
# execute command
#command = 'sum (rate (container_cpu_usage_seconds_total[1m]))'
#command = 'sum (rate (container_cpu_usage_seconds_total{image!=""}[1m]))'
# percertange
command = 'sum (rate (container_cpu_usage_seconds_total{id="/"}[20m])) / sum (machine_cpu_cores) * 100'
result = run([home_path + "/microk8s/promql_cli/promql", "--host", host, command, "--start", interval, "--output", "json"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
return result
# each container cpu usage percentage
def conatinerPerCpuUsage() :
# execute command
#command = 'sum(irate(container_cpu_usage_seconds_total[5m])*100)by(pod)'
#command = 'sum (rate (container_cpu_usage_seconds_total{image!=""}[5m])) by (pod)'
command = f'sum(rate(container_cpu_usage_seconds_total{{image!="",{namespace}}}[5m])) by (pod, container, namespace) / sum(container_spec_cpu_quota{{image!="", {namespace}}}/container_spec_cpu_period{{image!="", {namespace}}}) by (pod, container, namespace) * 100'
result = run([home_path + "/microk8s/promql_cli/promql", "--host", host, command, "--start", interval, "--output", "json"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
return result
# the cpu usage percentage of per container in each namespace
def namespacePerPodCpuUsage() :
# execute command
#command = 'sum(rate(container_cpu_usage_seconds_total{image!="", namespace ="%s"}[5m])) by (pod, container) / sum(container_spec_cpu_quota{image!="", namespace = "%s"}/container_spec_cpu_period{image!="", namespace = "%s"}) by (pod, container)'
command = 'sum(rate(container_cpu_usage_seconds_total{image!="", namespace ="%s"}[5m]))/ sum(container_spec_cpu_quota{image!="", namespace = "%s"}/container_spec_cpu_period{image!="", namespace = "%s"}) / count(kube_pod_status_phase{phase="Running", namespace= "%s"})' % (namespace, namespace, namespace, namespace)
result = run([home_path + "/microk8s/promql_cli/promql", "--host", host, command, "--start", interval, "--output", "json"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
return result
# node available memory percent
def nodeMemSecTotal() :
# execute command
command = '(node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes ) / node_memory_MemTotal_bytes * 100'
result = run([home_path + "/microk8s/promql_cli/promql", "--host", host, command, "--start", interval, "--output", "json"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
return result
# number of running pod in each namespace
def runningPodNumInNamespace() :
# execute command
command = f'sum(kube_pod_container_status_running{{{namespace}}}) by (namespace)'
result = run([home_path + "/microk8s/promql_cli/promql", "--host", host, command, "--start", interval, "--output", "json"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
return result
# number of weird status pod in each namespace
def weirdPodNumInNamespace() :
# execute command
command = f"sum by (namespace) (kube_pod_status_ready{{condition='false', {namespace}}})"
result = run([home_path + "/microk8s/promql_cli/promql", "--host", host, command, "--start", interval, "--output", "json"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
return result
# the memory usage percent of each container
def eachConatinerMemUsage() :
# execute command
command = f'sum (container_memory_working_set_bytes{{{namespace}}}) by (container_name , pod) / (sum (container_spec_memory_limit_bytes{{{namespace}}}>0 ) by (container_name, pod)) * 100'
result = run([home_path + "/microk8s/promql_cli/promql", "--host", host, command, "--start", interval, "--output", "json"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
return result
# the percent of pod memory use on the node deploying it
def podMemUseInNode() :
# execute command
#command = 'sum(kube_pod_container_resource_limits{resource="memory"}) / sum(kube_node_status_capacity{resource="memory"}) * 100'
#command = 'sum(kube_pod_container_resource_limits{resource="memory"}) by (node) / sum(kube_node_status_capacity{resource="memory"}) by (node) * 100'
command = 'sum(kube_pod_container_resource_requests{resource="memory"}) by (node) / sum(kube_node_status_capacity{resource="memory"}) by (node) * 100'
result = run([home_path + "/microk8s/promql_cli/promql", "--host", host, command, "--start", interval, "--output", "json"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
return result
# draw the graph
def plot_graph(data, filename, graph_type, data_metric):
timestamps = [str(datetime.datetime.fromtimestamp(d[0]))[11:-3] for d in data]
values = [float(d[1]) for d in data]
# start monitor time, end monitor time
start_time = str(datetime.datetime.fromtimestamp(data[0][0]))[:-7]
end_time = str(datetime.datetime.fromtimestamp(data[len(data)-1][0]))[:-7]
# put metric info in mark
mark = ""
for i in range(len(data_metric)) :
for j in range(len(data_metric[i])) :
mark += data_metric[i][j] + " - "
mark = removeMarkLast(mark)
mark += "\n"
#plt.text(4.5, 1.5, mark, fontsize=30, color='red', wrap=True)
fig, ax = plt.subplots()
plt.text(0, -0.15, mark, ha='left', wrap=True, transform=ax.transAxes)
# put metric into graph
plt.plot(timestamps, values)
plt.xlabel("Timestamp")
plt.ylabel("Percentage")
# put metric type on title
plt.title(graph_type + "\n" + f"{start_time} - {end_time}")
# disable x-axis : time
plt.xticks([])
# Save the graph as a PNG file
plt.savefig(filename)
plt.clf()
def removeMarkLast(mark) :
new_mark = ''
for i in mark[:-2] :
new_mark += i
return new_mark
main()