-
Notifications
You must be signed in to change notification settings - Fork 3
/
prometheus-rules.yaml
167 lines (151 loc) · 6.69 KB
/
prometheus-rules.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: prow-rules
namespace: monitoring
spec:
groups:
- name: prow
rules:
# These metrics are based on generic pods, not just Prow jobs.
# group interesting information into a single metric
# squash metrics from multiple kube-state-metrics pods
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="..."} 1
- record: prow:pod
expr: |
max by (namespace, pod, node, pod_ip) (kube_pod_info) *
on (namespace, pod)
group_left (phase)
(kube_pod_status_phase{namespace="default"} == 1)
# all pending pods
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="Pending"} 1
- record: prow:pod:pending
expr: prow:pod{phase="Pending"}
# same for the pod creation time
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="..."} [unix timestamp]
- record: prow:pod:created_time
expr: |
prow:pod *
on (namespace, pod)
group_right (node, pod_ip, phase)
max by (namespace, pod) (kube_pod_created)
# time since a pod has been in pending phase
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="Pending"} [unix timestamp]
- record: prow:pod:pending_since_time
expr: prow:pod:pending * prow:pod:created_time
# These metrics are specific to Prow jobs.
# This monster of a rule prepares a tidy list of jobs with their
# namespace and pod name attached.
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="...",org="...",repo="...",type="...",name="...",id="<UUID>"} 1
- record: prow:job
expr: |
max by (org, repo, type, name, id, namespace, pod, phase, node, pod_ip) (
label_replace(
label_replace(
label_replace(
label_replace(
label_replace(
prow:pod *
on (namespace, pod)
group_left (label_prow_k8s_io_id, label_prow_k8s_io_refs_repo, label_prow_k8s_io_type, label_prow_k8s_io_job, label_prow_k8s_io_refs_org)
kube_pod_labels{label_prow_k8s_io_id!=""},
"id",
"$0",
"label_prow_k8s_io_id",
".*"
),
"repo",
"$0",
"label_prow_k8s_io_refs_repo",
".*"
),
"type",
"$0",
"label_prow_k8s_io_type",
".*"
),
"name",
"$0",
"label_prow_k8s_io_job",
".*"
),
"org",
"$0",
"label_prow_k8s_io_refs_org",
".*"
)
)
# list of all pending pods
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="Pending",org="...",repo="...",type="...",name="...",id="<UUID>"} 1
- record: prow:job:pending
expr: |
prow:pod:pending *
on (namespace, pod)
group_left (org, repo, type, name, id)
prow:job
# time since a job has been in pending phase
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="Pending",org="...",repo="...",type="...",name="...",id="<UUID>"} [unix timestamp]
- record: prow:job:pending_since_time
expr: |
prow:pod:created_time *
on (namespace, pod)
group_left (org, repo, type, name, id)
prow:job:pending
# total memory used by a Prow job
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="...",org="...",repo="...",type="...",name="...",id="<UUID>"} [bytes]
- record: prow:job:memory_working_set_bytes
expr: |
sum by (namespace, pod, node, pod_ip) (container_memory_working_set_bytes{container="test"}) *
on(namespace, pod)
group_left(org, repo, type, name, id, node, pod_ip, phase)
prow:job
# total memory requests for a Prow job
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="...",org="...",repo="...",type="...",name="...",id="<UUID>"} [bytes]
- record: prow:job:resource_requests_memory_bytes
expr: |
sum by (namespace, pod, node, pod_ip) (kube_pod_container_resource_requests_memory_bytes{container="test"}) *
on(namespace, pod)
group_left(org, repo, type, name, id, node, pod_ip, phase)
prow:job
# total memory limit for a Prow job
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="...",org="...",repo="...",type="...",name="...",id="<UUID>"} [bytes]
- record: prow:job:resource_limits_memory_bytes
expr: |
sum by (namespace, pod, node, pod_ip) (kube_pod_container_resource_limits_memory_bytes{container="test"}) *
on(namespace, pod)
group_left(org, repo, type, name, id, node, pod_ip, phase)
prow:job
# total CPU used by a Prow job
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="...",org="...",repo="...",type="...",name="...",id="<UUID>"} [seconds]
- record: prow:job:cpu_usage_seconds_rate:1m
expr: |
sum by (namespace, pod, node, pod_ip) (rate(container_cpu_usage_seconds_total{container="test"}[1m])) *
on(namespace, pod)
group_left(org, repo, type, name, id, node, pod_ip, phase)
prow:job
# total CPU requests for a Prow job
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="...",org="...",repo="...",type="...",name="...",id="<UUID>"} [cores]
- record: prow:job:resource_requests_cpu_cores
expr: |
sum by (namespace, pod, node, pod_ip) (kube_pod_container_resource_requests_cpu_cores{container="test"}) *
on(namespace, pod)
group_left(org, repo, type, name, id, node, pod_ip, phase)
prow:job
# total CPU limit for a Prow job
# {namespace="...",pod="...",node="...",pod_ip="1.2.3.4",phase="...",org="...",repo="...",type="...",name="...",id="<UUID>"} [cores]
- record: prow:job:resource_limits_cpu_cores
expr: |
sum by (namespace, pod, node, pod_ip) (kube_pod_container_resource_limits_cpu_cores{container="test"}) *
on(namespace, pod)
group_left(org, repo, type, name, id, node, pod_ip, phase)
prow:job
# total CPU available on *worker* nodes
# {node="..."} [cores]
- record: prow:node:cpu_capacity_cores
expr: |
sum by (node) (kube_node_status_capacity_cpu_cores{node=~"worker-.*"})
# total memory available on *worker* nodes
# {node="..."} [bytes]
- record: prow:node:memory_capacity_bytes
expr: |
sum by (node) (kube_node_status_capacity_memory_bytes{node=~"worker-.*"})