Add reason tag to kubernetes_state.job.failed

pkg/collector/corechecks/cluster/ksm/kubernetes_state_transformers.go

@@ -273,9 +273,11 @@ func containerWaitingReasonTransformer(s sender.Sender, name string, metric ksms
 }
 
 var allowedTerminatedReasons = map[string]struct{}{
-	"oomkilled":          {},
-	"containercannotrun": {},
-	"error":              {},
+	"oomkilled":            {},
+	"containercannotrun":   {},
+	"error":                {},
+	"deadlineexceeded":     {},
+	"backofflimitexceeded": {},
 }
 
 // containerTerminatedReasonTransformer validates the container waiting reasons for metric kube_pod_container_status_terminated_reason
@@ -426,7 +428,7 @@ func validateJob(val float64, tags []string) ([]string, bool) {
 	kubeCronjob := ""
 	for _, tag := range tags {
 		split := strings.Split(tag, ":")
-		if len(split) == 2 && split[0] == "kube_job" || split[0] == "job" || split[0] == "job_name" {
+		if len(split) == 2 && split[0] == "kube_job" || split[0] == "job" || split[0] == "job_name" || split[0] == "reason" {
 			// Trim the timestamp suffix to avoid high cardinality
 			if name, trimmed := trimJobTag(split[1]); trimmed {
 				// The trimmed job name corresponds to the parent cronjob name

pkg/collector/corechecks/cluster/ksm/kubernetes_state_transformers_test.go

@@ -343,17 +343,17 @@ func Test_jobFailedTransformer(t *testing.T) {
 						"condition": "true",
 					},
 				},
-				tags: []string{"job_name:foo-1509998340", "namespace:default", "condition:true"},
+				tags: []string{"job_name:foo-1509998340", "namespace:default", "condition:true", "reason:backofflimitexceeded"},
 			},
 			expectedServiceCheck: &serviceCheck{
 				name:   "kubernetes_state.job.complete",
 				status: servicecheck.ServiceCheckCritical,
-				tags:   []string{"kube_cronjob:foo", "namespace:default"},
+				tags:   []string{"kube_cronjob:foo", "namespace:default", "reason:backofflimitexceeded"},
 			},
 			expectedMetric: &metricsExpected{
 				name: "kubernetes_state.job.completion.failed",
 				val:  1,
-				tags: []string{"kube_cronjob:foo", "namespace:default"},
+				tags: []string{"kube_cronjob:foo", "namespace:default", "reason:backofflimitexceeded"},
 			},
 		},
 		{
@@ -368,17 +368,17 @@ func Test_jobFailedTransformer(t *testing.T) {
 						"condition": "true",
 					},
 				},
-				tags: []string{"job:foo-1509998340", "namespace:default", "condition:true"},
+				tags: []string{"job:foo-1509998340", "namespace:default", "condition:true", "reason:deadlineexceeded"},
 			},
 			expectedServiceCheck: &serviceCheck{
 				name:   "kubernetes_state.job.complete",
 				status: servicecheck.ServiceCheckCritical,
-				tags:   []string{"kube_cronjob:foo", "namespace:default"},
+				tags:   []string{"kube_cronjob:foo", "namespace:default", "reason:deadlineexceeded"},
 			},
 			expectedMetric: &metricsExpected{
 				name: "kubernetes_state.job.completion.failed",
 				val:  1,
-				tags: []string{"kube_cronjob:foo", "namespace:default"},
+				tags: []string{"kube_cronjob:foo", "namespace:default", "reason:deadlineexceeded"},
 			},
 		},
 		{
@@ -393,7 +393,7 @@ func Test_jobFailedTransformer(t *testing.T) {
 						"condition": "true",
 					},
 				},
-				tags: []string{"job_name:foo-1509998340", "namespace:default", "condition:true"},
+				tags: []string{"job_name:foo-1509998340", "namespace:default", "condition:true", "reason:backofflimitexceeded"},
 			},
 			expectedServiceCheck: nil,
 			expectedMetric:       nil,
@@ -1009,6 +1009,48 @@ func Test_containerTerminatedReasonTransformer(t *testing.T) {
 				tags: []string{"container:foo", "pod:bar", "namespace:default", "reason:Error"},
 			},
 		},
+		{
+			name: "BackoffLimitExceeded",
+			args: args{
+				name: "kube_pod_container_status_terminated_reason",
+				metric: ksmstore.DDMetric{
+					Val: 1,
+					Labels: map[string]string{
+						"container": "foo",
+						"pod":       "bar",
+						"namespace": "default",
+						"reason":    "BackoffLimitExceeded",
+					},
+				},
+				tags: []string{"container:foo", "pod:bar", "namespace:default", "reason:BackoffLimitExceeded"},
+			},
+			expected: &metricsExpected{
+				name: "kubernetes_state.container.status_report.count.terminated",
+				val:  1,
+				tags: []string{"container:foo", "pod:bar", "namespace:default", "reason:BackoffLimitExceeded"},
+			},
+		},
+		{
+			name: "DeadlineExceeded",
+			args: args{
+				name: "kube_pod_container_status_terminated_reason",
+				metric: ksmstore.DDMetric{
+					Val: 1,
+					Labels: map[string]string{
+						"container": "foo",
+						"pod":       "bar",
+						"namespace": "default",
+						"reason":    "DeadlineExceeded",
+					},
+				},
+				tags: []string{"container:foo", "pod:bar", "namespace:default", "reason:DeadlineExceeded"},
+			},
+			expected: &metricsExpected{
+				name: "kubernetes_state.container.status_report.count.terminated",
+				val:  1,
+				tags: []string{"container:foo", "pod:bar", "namespace:default", "reason:DeadlineExceeded"},
+			},
+		},
 	}
 	for _, tt := range tests {
 		s := mocksender.NewMockSender("ksm")

pkg/collector/corechecks/containers/kubelet/provider/pod/provider.go

@@ -37,7 +37,13 @@ var includeContainerStateReason = map[string][]string{
 		"invalidimagename",
 		"createcontainerconfigerror",
 	},
-	"terminated": {"oomkilled", "containercannotrun", "error"},
+	"terminated": {
+		"oomkilled",
+		"containercannotrun",
+		"error",
+		"deadlineexceeded",
+		"backofflimitexceeded",
+	},
 }
 
 const kubeNamespaceTag = "kube_namespace"

releasenotes-dca/notes/add-reason-tags-to-ksm-26b5404f117c96e5.yaml

@@ -0,0 +1,13 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+enhancements:
+  - |
+    Add ``reason:deadlineexceeded`` and ``reason:backofflimitexceeded``
+    to ``kubernetes_state.container.status_report.count.terminated``.
+    Add ``reason`` tags to ``kubernetes_state.job.*``.

releasenotes/notes/add-reason-tags-to-kubelet-metrics-c3c8800ae018992b.yaml

@@ -0,0 +1,12 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+enhancements:
+  - |
+    Add ``reason:deadlineexceeded`` and ``reason:backofflimitexceeded`` to
+    ``kubernetes.containers.{state,last_status}.terminated``.