Cherry-pick #16533 to 7.x: [Filebeat] Improve ECS categorization field mapping in icinga

CHANGELOG.next.asciidoc

@@ -241,6 +241,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS categorization field mappings in iis module. {issue}16165[16165] {pull}16618[16618]
 - Improve the decode_cef processor by reducing the number of memory allocations. {pull}16587[16587]
 - Improve ECS categorization field mapping in kafka module. {issue}16167[16167] {pull}16645[16645]
+- Improve ECS categorization field mapping in icinga module. {issue}16164[16164] {pull}16533[16533]
 
 *Heartbeat*
 

filebeat/module/icinga/debug/ingest/pipeline.yml

@@ -0,0 +1,39 @@
+description: Pipeline for parsing icinga debug logs
+processors:
+- grok:
+    field: message
+    patterns:
+    - '\[%{TIMESTAMP:icinga.debug.timestamp}\] %{WORD:log.level}/%{WORD:icinga.debug.facility}:
+      %{GREEDYMULTILINE:message}'
+    ignore_missing: true
+    pattern_definitions:
+      TIMESTAMP: '%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}'
+      GREEDYMULTILINE: |-
+        (.|
+        )*
+- date:
+    field: icinga.debug.timestamp
+    target_field: '@timestamp'
+    formats:
+    - yyyy-MM-dd HH:mm:ss Z
+    ignore_failure: true
+- remove:
+    field: icinga.debug.timestamp
+- set:
+    field: event.kind
+    value: event
+- script:
+    lang: painless
+    source: >-
+      def errorLevels = ["warning", "critical"];
+      if (ctx?.log?.level != null) {
+        if (errorLevels.contains(ctx.log.level)) {
+          ctx.event.type = "error";
+        } else {
+          ctx.event.type = "info";
+        }
+      }
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

filebeat/module/icinga/debug/manifest.yml

@@ -9,5 +9,5 @@ var:
     os.windows:
       - c:/programdata/icinga2/var/log/icinga2/debug.log*
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/debug.yml

filebeat/module/icinga/debug/test/test.log-expected.json

@@ -2,7 +2,9 @@
     {
         "@timestamp": "2017-04-04T11:43:09.000Z",
         "event.dataset": "icinga.debug",
+        "event.kind": "event",
         "event.module": "icinga",
+        "event.type": "info",
         "fileset.name": "debug",
         "icinga.debug.facility": "GraphiteWriter",
         "input.type": "log",
@@ -14,7 +16,9 @@
     {
         "@timestamp": "2017-04-04T11:43:09.000Z",
         "event.dataset": "icinga.debug",
+        "event.kind": "event",
         "event.module": "icinga",
+        "event.type": "info",
         "fileset.name": "debug",
         "icinga.debug.facility": "IdoMysqlConnection",
         "input.type": "log",
@@ -26,7 +30,9 @@
     {
         "@timestamp": "2017-04-04T11:43:11.000Z",
         "event.dataset": "icinga.debug",
+        "event.kind": "event",
         "event.module": "icinga",
+        "event.type": "info",
         "fileset.name": "debug",
         "icinga.debug.facility": "Process",
         "input.type": "log",

filebeat/module/icinga/main/ingest/pipeline.yml

@@ -0,0 +1,39 @@
+description: Pipeline for parsing icinga main logs
+processors:
+- grok:
+    field: message
+    patterns:
+    - '\[%{TIMESTAMP:icinga.main.timestamp}\] %{WORD:log.level}/%{WORD:icinga.main.facility}:
+      %{GREEDYMULTILINE:message}'
+    ignore_missing: true
+    pattern_definitions:
+      TIMESTAMP: '%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}'
+      GREEDYMULTILINE: |-
+        (.|
+        )*
+- date:
+    field: icinga.main.timestamp
+    target_field: '@timestamp'
+    formats:
+    - yyyy-MM-dd HH:mm:ss Z
+    ignore_failure: true
+- remove:
+    field: icinga.main.timestamp
+- set:
+    field: event.kind
+    value: event
+- script:
+    lang: painless
+    source: >-
+      def errorLevels = ["warning", "critical"];
+      if (ctx?.log?.level != null) {
+        if (errorLevels.contains(ctx.log.level)) {
+          ctx.event.type = "error";
+        } else {
+          ctx.event.type = "info";
+        }
+      }
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

filebeat/module/icinga/main/manifest.yml

@@ -9,5 +9,5 @@ var:
     os.windows:
       - c:/programdata/icinga2/var/log/icinga2/icinga2.log*
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/main.yml

filebeat/module/icinga/main/test/test.log-expected.json

@@ -2,7 +2,9 @@
     {
         "@timestamp": "2017-04-04T09:16:34.000Z",
         "event.dataset": "icinga.main",
+        "event.kind": "event",
         "event.module": "icinga",
+        "event.type": "info",
         "fileset.name": "main",
         "icinga.main.facility": "Notification",
         "input.type": "log",
@@ -14,7 +16,9 @@
     {
         "@timestamp": "2017-04-04T09:16:34.000Z",
         "event.dataset": "icinga.main",
+        "event.kind": "event",
         "event.module": "icinga",
+        "event.type": "error",
         "fileset.name": "main",
         "icinga.main.facility": "PluginNotificationTask",
         "input.type": "log",
@@ -29,7 +33,9 @@
     {
         "@timestamp": "2017-04-04T09:16:48.000Z",
         "event.dataset": "icinga.main",
+        "event.kind": "event",
         "event.module": "icinga",
+        "event.type": "info",
         "fileset.name": "main",
         "icinga.main.facility": "IdoMysqlConnection",
         "input.type": "log",

filebeat/module/icinga/startup/ingest/pipeline.yml

@@ -0,0 +1,29 @@
+description: Pipeline for parsing icinga startup logs
+processors:
+- grok:
+    field: message
+    patterns:
+    - '%{WORD:log.level}/%{WORD:icinga.startup.facility}: %{GREEDYMULTILINE:message}'
+    ignore_missing: true
+    pattern_definitions:
+      GREEDYMULTILINE: |-
+        (.|
+        )*
+- set:
+    field: event.kind
+    value: event
+- script:
+    lang: painless
+    source: >-
+      def errorLevels = ["warning", "critical"];
+      if (ctx?.log?.level != null) {
+        if (errorLevels.contains(ctx.log.level)) {
+          ctx.event.type = "error";
+        } else {
+          ctx.event.type = "info";
+        }
+      }
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

filebeat/module/icinga/startup/manifest.yml

@@ -9,5 +9,5 @@ var:
     os.windows:
       - c:/programdata/icinga2/var/log/icinga2/startup.log
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/startup.yml

filebeat/module/icinga/startup/test/test.log-expected.json

@@ -1,7 +1,9 @@
 [
     {
         "event.dataset": "icinga.startup",
+        "event.kind": "event",
         "event.module": "icinga",
+        "event.type": "info",
         "fileset.name": "startup",
         "icinga.startup.facility": "cli",
         "input.type": "log",
@@ -12,7 +14,9 @@
     },
     {
         "event.dataset": "icinga.startup",
+        "event.kind": "event",
         "event.module": "icinga",
+        "event.type": "info",
         "fileset.name": "startup",
         "icinga.startup.facility": "cli",
         "input.type": "log",