Filebeat coredns module

CHANGELOG.next.asciidoc

@@ -290,6 +290,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change Suricata module pipeline to handle `destination.domain` being set if a reverse DNS processor is used. {issue}10510[10510]
 - Add the `network.community_id` flow identifier to field to the IPTables, Suricata, and Zeek modules. {pull}11005[11005]
 - Add support for loading custom NetFlow and IPFIX field definitions to netflow input. {pull}10945[10945]
+- New Filebeat coredns module to ingest coredns logs. It supports both the native coredns deployment and coredns deployment in kubernetes. {pull}11200[11200]
 
 *Heartbeat*
 

deploy/kubernetes/filebeat-autodiscover-k8s.yaml

@@ -0,0 +1,140 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ingest-config
+  namespace: kube-system
+  labels:
+    k8s-app: ingest
+data:
+  filebeat-k8s.yml: |-
+    logging.level: debug
+
+    filebeat.autodiscover:
+      providers:
+        - type: kubernetes
+          hints.enabled: true
+          default.disable: true
+
+    processors:
+      - add_kubernetes_metadata:
+          in_cluster: true
+
+    output.elasticsearch:
+     ilm.enabled: false
+     hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
+     username: ${ELASTICSEARCH_USERNAME}
+     password: ${ELASTICSEARCH_PASSWORD}
+
+    setup.kibana:
+      host: '${KIBANA_HOST:kibana}:${KIBANA_PORT:5601}'
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: ingest
+  namespace: kube-system
+  labels:
+    k8s-app: ingest
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: ingest
+    spec:
+      serviceAccountName: ingest
+      terminationGracePeriodSeconds: 30
+      containers:
+      - name: ingest
+        image: docker.elastic.co/beats/filebeat:8.0.0
+        args: [
+          "sh", "-c", "filebeat setup -e --modules coredns -c /etc/filebeat-k8s.yml && filebeat -e -c /etc/filebeat-k8s.yml"
+        ]
+        env:
+        # Edit these values to reflect your setup
+        - name: ELASTICSEARCH_HOST
+          value: 192.168.99.1
+        - name: ELASTICSEARCH_USERNAME
+          value: elastic
+        - name: ELASTICSEARCH_PASSWORD
+          value: changeme
+        - name: KIBANA_HOST
+          value: 192.168.99.1
+        - name: ELASTIC_CLOUD_ID
+          value:
+        - name: ELASTIC_CLOUD_AUTH
+          value:
+        securityContext:
+          runAsUser: 0
+          # If using Red Hat OpenShift uncomment this:
+          #privileged: true
+        resources:
+          limits:
+            memory: 256Mi
+          requests:
+            cpu: 100m
+            memory: 128Mi
+        volumeMounts:
+        - name: config
+          mountPath: /etc/filebeat-k8s.yml
+          readOnly: true
+          subPath: filebeat-k8s.yml
+        - name: data
+          mountPath: /usr/share/filebeat/data
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+        #- name: varlogcontainers
+        #  mountPath: /var/log/containers
+        #  readOnly: true
+      volumes:
+      - name: config
+        configMap:
+          defaultMode: 0600
+          name: ingest-config
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
+      - name: data
+        hostPath:
+          path: /var/lib/filebeat-data
+          type: DirectoryOrCreate
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: ingest
+subjects:
+- kind: ServiceAccount
+  name: ingest
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ingest
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: ingest
+  labels:
+    k8s-app: ingest
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources:
+  - namespaces
+  - pods
+  verbs:
+  - get
+  - watch
+  - list
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ingest
+  namespace: kube-system
+  labels:
+    k8s-app: ingest
+---
+

filebeat/docs/fields.asciidoc

@@ -16,6 +16,7 @@ grouped in the following categories:
 * <<exported-fields-auditd>>
 * <<exported-fields-beat>>
 * <<exported-fields-cloud>>
+* <<exported-fields-coredns>>
 * <<exported-fields-docker-processor>>
 * <<exported-fields-ecs>>
 * <<exported-fields-elasticsearch>>
@@ -1125,6 +1126,114 @@ type: alias
 
 alias to: cloud.region
 
+--
+
+[[exported-fields-coredns]]
+== Coredns fields
+
+Module for handling logs produced by coredns
+
+
+
+[float]
+== coredns fields
+
+coredns fields after normalization
+
+
+
+*`coredns.id`*::
++
+--
+type: keyword
+
+id of the DNS transaction
+
+
+--
+
+*`coredns.query.size`*::
++
+--
+type: integer
+
+format: bytes
+
+size of the DNS query
+
+
+--
+
+*`coredns.query.class`*::
++
+--
+type: keyword
+
+DNS query class
+
+
+--
+
+*`coredns.query.name`*::
++
+--
+type: keyword
+
+DNS query name
+
+
+--
+
+*`coredns.query.type`*::
++
+--
+type: keyword
+
+DNS query type
+
+
+--
+
+*`coredns.response.code`*::
++
+--
+type: keyword
+
+DNS response code
+
+
+--
+
+*`coredns.response.flags`*::
++
+--
+type: keyword
+
+DNS response flags
+
+
+--
+
+*`coredns.response.size`*::
++
+--
+type: integer
+
+format: bytes
+
+size of the DNS response
+
+
+--
+
+*`coredns.dnssec_ok`*::
++
+--
+type: boolean
+
+dnssec flag
+
+
 --
 
 [[exported-fields-docker-processor]]

filebeat/docs/modules/coredns.asciidoc

@@ -0,0 +1,37 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-coredns]]
+[role="xpack"]
+
+:modulename: coredns
+:has-dashboards: true
+
+== Coredns Module
+
+This is a filebeat module for coredns. It supports both standalone coredns deployment and 
+coredns deployment in Kubernetes. Standalone coredns deployment uses the log fileset, while
+Kubernetes coredns deployment uses the kubernetes fileset.
+
+[float]
+=== Compatibility
+
+Although this module has been developed against Kubernetes v1.13.x, it is expected to work
+with other versions of Kubernetes.
+
+[float]
+=== Example dashboard
+
+This module comes with a sample dashboard.
+
+[role="screenshot"]
+image::./images/Coredns_Overview_Dashboard.jpg[]
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-coredns,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -5,6 +5,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-modules-overview>>
   * <<filebeat-module-apache>>
   * <<filebeat-module-auditd>>
+  * <<filebeat-module-coredns>>
   * <<filebeat-module-elasticsearch>>
   * <<filebeat-module-haproxy>>
   * <<filebeat-module-icinga>>
@@ -33,6 +34,7 @@ This file is generated! See scripts/docs_collector.py
 include::modules-overview.asciidoc[]
 include::modules/apache.asciidoc[]
 include::modules/auditd.asciidoc[]
+include::modules/coredns.asciidoc[]
 include::modules/elasticsearch.asciidoc[]
 include::modules/haproxy.asciidoc[]
 include::modules/icinga.asciidoc[]

x-pack/filebeat/filebeat.reference.yml

@@ -82,6 +82,19 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+#------------------------------- Coredns Module -------------------------------
+- module: coredns
+  # Fileset for native deployment
+  log: 
+    enabled: true
+  # Fileset for Kubernetes deployment
+  kubernetes:
+    enabled: false
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 #---------------------------- Elasticsearch Module ----------------------------
 - module: elasticsearch
   # Server log