[metricbeat] Add state_job metricset (#26479)

elastic · Jul 1, 2021 · 24c07b2 · 24c07b2
1 parent 2822e04
commit 24c07b2
Show file tree

Hide file tree

Showing 35 changed files with 43,714 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -134,6 +134,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove xpack enabled flag on ES, Logstash, Beats and Kibana {pull}24427[24427]
 - Adjust host fields to adopt new names from 1.9.0 ECS. {pull}24312[24312]
 - Add replicas.ready field to state_statefulset in Kubernetes module{pull}26088[26088]
+- Add state_job metricset to Kubernetes module{pull}26479[26479]
 
 *Packetbeat*
 

diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
@@ -81,6 +81,15 @@ data:
             hosts:
               - 'kube-state-metrics:8080'
             period: 10s
+          - data_stream:
+              dataset: kubernetes.state_job
+              type: metrics
+            metricsets:
+              - state_job
+            add_metadata: true
+            hosts:
+              - 'kube-state-metrics:8080'
+            period: 10s
           - data_stream:
               dataset: kubernetes.state_node
               type: metrics
@@ -593,6 +602,10 @@ rules:
       - deployments
       - replicasets
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["batch"]
+    resources:
+      - jobs
+    verbs: ["get", "list", "watch"]
   - apiGroups:
       - ""
     resources:

diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml
@@ -81,6 +81,15 @@ data:
             hosts:
               - 'kube-state-metrics:8080'
             period: 10s
+          - data_stream:
+              dataset: kubernetes.state_job
+              type: metrics
+            metricsets:
+              - state_job
+            add_metadata: true
+            hosts:
+              - 'kube-state-metrics:8080'
+            period: 10s
           - data_stream:
               dataset: kubernetes.state_node
               type: metrics

diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml
@@ -29,6 +29,10 @@ rules:
       - deployments
       - replicasets
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["batch"]
+    resources:
+      - jobs
+    verbs: ["get", "list", "watch"]
   - apiGroups:
       - ""
     resources:

diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml
@@ -33,6 +33,7 @@ data:
                     - state_replicaset
                     - state_pod
                     - state_container
+                    - state_job
                     - state_cronjob
                     - state_resourcequota
                     - state_statefulset
@@ -288,6 +289,10 @@ rules:
   - deployments
   - replicasets
   verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+  - jobs
+  verbs: ["get", "list", "watch"]
 - apiGroups:
   - ""
   resources:

diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml
@@ -33,6 +33,7 @@ data:
                     - state_replicaset
                     - state_pod
                     - state_container
+                    - state_job
                     - state_cronjob
                     - state_resourcequota
                     - state_statefulset

diff --git a/deploy/kubernetes/metricbeat/metricbeat-role.yaml b/deploy/kubernetes/metricbeat/metricbeat-role.yaml
@@ -28,6 +28,10 @@ rules:
   - deployments
   - replicasets
   verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+  - jobs
+  verbs: ["get", "list", "watch"]
 - apiGroups:
   - ""
   resources:

diff --git a/libbeat/common/kubernetes/informer.go b/libbeat/common/kubernetes/informer.go
@@ -149,6 +149,18 @@ func NewInformer(client kubernetes.Interface, resource Resource, opts WatchOptio
 		}
 
 		objType = "service"
+	case *Job:
+		job := client.BatchV1().Jobs(opts.Namespace)
+		listwatch = &cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+				return job.List(ctx, options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				return job.Watch(ctx, options)
+			},
+		}
+
+		objType = "job"
 	default:
 		return nil, "", fmt.Errorf("unsupported resource type for watching %T", resource)
 	}

diff --git a/libbeat/common/kubernetes/types.go b/libbeat/common/kubernetes/types.go
@@ -22,6 +22,7 @@ import (
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -72,6 +73,9 @@ type StatefulSet = appsv1.StatefulSet
 // Service data
 type Service = v1.Service
 
+// Job data
+type Job = batchv1.Job
+
 const (
 	// PodPending phase
 	PodPending = v1.PodPending

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -37007,6 +37007,185 @@ type: integer
 
 --
 
+[float]
+=== job
+
+Kubernetes job metrics
+
+
+
+*`kubernetes.job.name`*::
++
+--
+The name of the job resource
+
+
+type: keyword
+
+--
+
+[float]
+=== pods
+
+Pod metrics for the job
+
+
+
+*`kubernetes.job.pods.active`*::
++
+--
+Number of active pods
+
+
+type: long
+
+--
+
+*`kubernetes.job.pods.failed`*::
++
+--
+Number of failed pods
+
+
+type: long
+
+--
+
+*`kubernetes.job.pods.succeeded`*::
++
+--
+Number of successful pods
+
+
+type: long
+
+--
+
+[float]
+=== time
+
+Kubernetes job timestamps
+
+
+
+*`kubernetes.job.time.created`*::
++
+--
+The time at which the job was created
+
+
+type: date
+
+--
+
+*`kubernetes.job.time.completed`*::
++
+--
+The time at which the job completed
+
+
+type: date
+
+--
+
+[float]
+=== completions
+
+Kubernetes job completion settings
+
+
+
+*`kubernetes.job.completions.desired`*::
++
+--
+The configured completion count for the job (Spec)
+
+
+type: long
+
+--
+
+[float]
+=== parallelism
+
+Kubernetes job parallelism settings
+
+
+
+*`kubernetes.job.parallelism.desired`*::
++
+--
+The configured parallelism of the job (Spec)
+
+
+type: long
+
+--
+
+[float]
+=== owner
+
+Kubernetes job owner information
+
+
+
+*`kubernetes.job.owner.name`*::
++
+--
+The name of the resource that owns this job
+
+
+type: keyword
+
+--
+
+*`kubernetes.job.owner.kind`*::
++
+--
+The kind of resource that owns this job (eg. "CronJob")
+
+
+type: keyword
+
+--
+
+*`kubernetes.job.owner.is_controller`*::
++
+--
+Owner is controller ("true", "false", or "<none>")
+
+
+type: keyword
+
+--
+
+[float]
+=== status
+
+Kubernetes job status information
+
+
+
+*`kubernetes.job.status.complete`*::
++
+--
+Whether the job completed ("true", "false", or "unknown")
+
+
+type: keyword
+
+--
+
+*`kubernetes.job.status.failed`*::
++
+--
+Whether the job failed ("true", "false", or "unknown")
+
+
+type: keyword
+
+--
+
 [float]
 === node
 

diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc
@@ -102,6 +102,10 @@ rules:
   - events
   - pods
   verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+  - jobs
+  verbs: ["get", "list", "watch"]
 - apiGroups: ["extensions"]
   resources:
   - replicasets
@@ -216,6 +220,7 @@ metricbeat.modules:
     - state_statefulset
     - state_pod
     - state_container
+    - state_job
     - state_cronjob
     - state_resourcequota
     - state_service
@@ -308,6 +313,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-kubernetes-state_deployment,state_deployment>>
 
+* <<metricbeat-metricset-kubernetes-state_job,state_job>>
+
 * <<metricbeat-metricset-kubernetes-state_node,state_node>>
 
 * <<metricbeat-metricset-kubernetes-state_persistentvolumeclaim,state_persistentvolumeclaim>>
@@ -352,6 +359,8 @@ include::kubernetes/state_daemonset.asciidoc[]
 
 include::kubernetes/state_deployment.asciidoc[]
 
+include::kubernetes/state_job.asciidoc[]
+
 include::kubernetes/state_node.asciidoc[]
 
 include::kubernetes/state_persistentvolumeclaim.asciidoc[]

diff --git a/metricbeat/docs/modules/kubernetes/state_job.asciidoc b/metricbeat/docs/modules/kubernetes/state_job.asciidoc
@@ -0,0 +1,21 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-kubernetes-state_job]]
+=== Kubernetes state_job metricset
+
+include::../../../module/kubernetes/state_job/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-kubernetes,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/kubernetes/state_job/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -157,7 +157,7 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-kibana-stats,stats>>   
 |<<metricbeat-metricset-kibana-status,status>>   
 |<<metricbeat-module-kubernetes,Kubernetes>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.22+| .22+|  |<<metricbeat-metricset-kubernetes-apiserver,apiserver>>   
+.23+| .23+|  |<<metricbeat-metricset-kubernetes-apiserver,apiserver>>   
 |<<metricbeat-metricset-kubernetes-container,container>>   
 |<<metricbeat-metricset-kubernetes-controllermanager,controllermanager>>   
 |<<metricbeat-metricset-kubernetes-event,event>>   
@@ -169,6 +169,7 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-kubernetes-state_cronjob,state_cronjob>>   
 |<<metricbeat-metricset-kubernetes-state_daemonset,state_daemonset>>   
 |<<metricbeat-metricset-kubernetes-state_deployment,state_deployment>>   
+|<<metricbeat-metricset-kubernetes-state_job,state_job>>   
 |<<metricbeat-metricset-kubernetes-state_node,state_node>>   
 |<<metricbeat-metricset-kubernetes-state_persistentvolumeclaim,state_persistentvolumeclaim>>   
 |<<metricbeat-metricset-kubernetes-state_pod,state_pod>>