[libbeat] New decode xml wineventlog processor (#25115) (#25165)

* Move enrich raw functionality to common package

* Enrich Raw fields when possible in decode_xml

* Add ECS mappings when decoding wineventlog xml

* Add decode_xml_wineventlog processor

* Add missing fields to config checks

* Change event.code type

* Fix PR number in changelog

* Fix test

* Remove document_id and make docs more clear

(cherry picked from commit 8cf8f51)

Co-authored-by: Marc Guasch <marc-gr@users.noreply.github.com>

CHANGELOG.next.asciidoc

@@ -373,6 +373,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Libbeat: report queue capacity, output batch size, and output client count to monitoring. {pull}24700[24700]
 - Add kubernetes.pod.ip field in kubernetes metadata. {pull}25037[25037]
 - Discover changes in Kubernetes namespace metadata as soon as they happen. {pull}25117[25117]
+- Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25115[25115]
 
 *Auditbeat*
 

libbeat/cmd/instance/imports_common.go

@@ -31,6 +31,7 @@ import (
 	_ "github.com/elastic/beats/v7/libbeat/processors/communityid"
 	_ "github.com/elastic/beats/v7/libbeat/processors/convert"
 	_ "github.com/elastic/beats/v7/libbeat/processors/decode_xml"
+	_ "github.com/elastic/beats/v7/libbeat/processors/decode_xml_wineventlog"
 	_ "github.com/elastic/beats/v7/libbeat/processors/dissect"
 	_ "github.com/elastic/beats/v7/libbeat/processors/dns"
 	_ "github.com/elastic/beats/v7/libbeat/processors/extract_array"

libbeat/docs/processors-list.asciidoc

@@ -63,7 +63,10 @@ ifndef::no_decode_json_fields_processor[]
 * <<decode-json-fields,`decode_json_fields`>>
 endif::[]
 ifndef::no_decode_xml_processor[]
-* <<decode_xml, `decode_xml`>>
+* <<decode-xml, `decode_xml`>>
+endif::[]
+ifndef::no_decode_xml_wineventlog_processor[]
+* <<decode-xml-wineventlog, `decode_xml_wineventlog`>>
 endif::[]
 ifndef::no_decompress_gzip_field_processor[]
 * <<decompress-gzip-field,`decompress_gzip_field`>>
@@ -183,6 +186,12 @@ endif::[]
 ifndef::no_decode_json_fields_processor[]
 include::{libbeat-processors-dir}/actions/docs/decode_json_fields.asciidoc[]
 endif::[]
+ifndef::no_decode_xml_processor[]
+include::{libbeat-processors-dir}/decode_xml/docs/decode_xml.asciidoc[]
+endif::[]
+ifndef::no_decode_xml_wineventlog_processor[]
+include::{libbeat-processors-dir}/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc[]
+endif::[]
 ifndef::no_decompress_gzip_field_processor[]
 include::{libbeat-processors-dir}/actions/docs/decompress_gzip_field.asciidoc[]
 endif::[]
@@ -234,8 +243,5 @@ endif::[]
 ifndef::no_urldecode_processor[]
 include::{libbeat-processors-dir}/urldecode/docs/urldecode.asciidoc[]
 endif::[]
-ifndef::no_decode_xml_processor[]
-include::{libbeat-processors-dir}/decode_xml/docs/decode_xml.asciidoc[]
-endif::[]
 
 //# end::processors-include[]

libbeat/processors/decode_xml/config.go

@@ -25,7 +25,6 @@ type decodeXMLConfig struct {
 	ToLower       bool    `config:"to_lower"`
 	IgnoreMissing bool    `config:"ignore_missing"`
 	IgnoreFailure bool    `config:"ignore_failure"`
-	Schema        string  `config:"schema"`
 }
 
 func defaultConfig() decodeXMLConfig {

libbeat/processors/decode_xml/decode_xml.go

@@ -18,13 +18,15 @@
 package decode_xml
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
+	"github.com/elastic/beats/v7/libbeat/common/encoding/xml"
 	"github.com/elastic/beats/v7/libbeat/common/jsontransform"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/processors"
@@ -35,8 +37,7 @@ import (
 type decodeXML struct {
 	decodeXMLConfig
 
-	decode decoder
-	log    *logp.Logger
+	log *logp.Logger
 }
 
 var (
@@ -56,10 +57,9 @@ func init() {
 				"field", "target_field",
 				"overwrite_keys", "document_id",
 				"to_lower", "ignore_missing",
-				"ignore_failure", "schema",
+				"ignore_failure",
 			)))
 	jsprocessor.RegisterPlugin(procName, New)
-	registerDecoders()
 }
 
 // New constructs a new decode_xml processor.
@@ -83,7 +83,6 @@ func newDecodeXML(config decodeXMLConfig) (processors.Processor, error) {
 
 	return &decodeXML{
 		decodeXMLConfig: config,
-		decode:          newDecoder(config),
 		log:             logp.NewLogger(logName),
 	}, nil
 }
@@ -135,9 +134,24 @@ func (x *decodeXML) run(event *beat.Event) error {
 	if id != "" {
 		event.SetID(id)
 	}
+
 	return nil
 }
 
+func (x *decodeXML) decode(p []byte) (common.MapStr, error) {
+	dec := xml.NewDecoder(bytes.NewReader(p))
+	if x.ToLower {
+		dec.LowercaseKeys()
+	}
+
+	out, err := dec.Decode()
+	if err != nil {
+		return nil, err
+	}
+
+	return common.MapStr(out), nil
+}
+
 func (x *decodeXML) String() string {
 	json, _ := json.Marshal(x.decodeXMLConfig)
 	return procName + "=" + string(json)

libbeat/processors/decode_xml/docs/decode_xml.asciidoc

@@ -1,4 +1,4 @@
-[[decode_xml]]
+[[decode-xml]]
 === Decode XML
 
 ++++
@@ -100,9 +100,6 @@ default value is `true`.
 `to_lower`:: (Optional) Converts all keys to lowercase. Accepts either true or
 false. The default value is `true`.
 
-`schema`:: (Optional) Specifies the schema of the message. Accepted schemas: `wineventlog`.
-If no schema is specified it defaults to using the regular XML to JSON conversion.
-
 `document_id`:: (Optional) XML key to use as the document ID. If configured, the
 field will be removed from the original XML document and stored in
 `@metadata._id`.
@@ -114,80 +111,3 @@ when a specified field does not exist. Defaults to `false`.
 Defaults to `false`.
 
 See <<conditions>> for a list of supported conditions.
-
-
-==== Schemas
-
-When a schema is defined, the specific decoder will parse the configured field.
-The ouput of the parsing will be specific to that schema.
-
-===== Wineventlog
-
-The `wineventlog` schema decodes Windows Events.
-
-The decoder will always output the fields formatted in the same way, the
-`to_lower` option will be ignored when using this schema decoder.
-The output fields will be the same as the
-{winlogbeat-ref}/exported-fields-winlog.html#_winlog[winlogbeat winlog fields].
-
-Example:
-
-[source,yaml]
--------------------------------------------------------------------------------
-processors:
-  - decode_xml:
-      field: event.original
-      target_field: winlog
-      to_lower: false
--------------------------------------------------------------------------------
-
-[source,json]
--------------------------------------------------------------------------------
-{
-  "event": {
-    "original": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Security-Auditing' Guid='{54849625-5478-4994-a5ba-3e3b0328c30d}'/><EventID>4672</EventID><Version>0</Version><Level>0</Level><Task>12548</Task><Opcode>0</Opcode><Keywords>0x8020000000000000</Keywords><TimeCreated SystemTime='2021-03-23T09:56:13.137310000Z'/><EventRecordID>11303</EventRecordID><Correlation ActivityID='{ffb23523-1f32-0000-c335-b2ff321fd701}'/><Execution ProcessID='652' ThreadID='4660'/><Channel>Security</Channel><Computer>vagrant</Computer><Security/></System><EventData><Data Name='SubjectUserSid'>S-1-5-18</Data><Data Name='SubjectUserName'>SYSTEM</Data><Data Name='SubjectDomainName'>NT AUTHORITY</Data><Data Name='SubjectLogonId'>0x3e7</Data><Data Name='PrivilegeList'>SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege</Data></EventData><RenderingInfo Culture='en-US'><Message>Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege</Message><Level>Information</Level><Task>Special Logon</Task><Opcode>Info</Opcode><Channel>Security</Channel><Provider>Microsoft Windows security auditing.</Provider><Keywords><Keyword>Audit Success</Keyword></Keywords></RenderingInfo></Event>"
-  }
-}
--------------------------------------------------------------------------------
-
-Will produce the following output:
-
-[source,json]
--------------------------------------------------------------------------------
-{
-  "event": {
-    "original": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Security-Auditing' Guid='{54849625-5478-4994-a5ba-3e3b0328c30d}'/><EventID>4672</EventID><Version>0</Version><Level>0</Level><Task>12548</Task><Opcode>0</Opcode><Keywords>0x8020000000000000</Keywords><TimeCreated SystemTime='2021-03-23T09:56:13.137310000Z'/><EventRecordID>11303</EventRecordID><Correlation ActivityID='{ffb23523-1f32-0000-c335-b2ff321fd701}'/><Execution ProcessID='652' ThreadID='4660'/><Channel>Security</Channel><Computer>vagrant</Computer><Security/></System><EventData><Data Name='SubjectUserSid'>S-1-5-18</Data><Data Name='SubjectUserName'>SYSTEM</Data><Data Name='SubjectDomainName'>NT AUTHORITY</Data><Data Name='SubjectLogonId'>0x3e7</Data><Data Name='PrivilegeList'>SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege</Data></EventData><RenderingInfo Culture='en-US'><Message>Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege</Message><Level>Information</Level><Task>Special Logon</Task><Opcode>Info</Opcode><Channel>Security</Channel><Provider>Microsoft Windows security auditing.</Provider><Keywords><Keyword>Audit Success</Keyword></Keywords></RenderingInfo></Event>"
-  },
-  "winlog": {
-    "channel": "Security",
-    "outcome": "success",
-    "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}",
-    "level": "information",
-    "event_id": 4672,
-    "provider_name": "Microsoft-Windows-Security-Auditing",
-    "record_id": 11303,
-    "computer_name": "vagrant",
-    "keywords_raw": 9232379236109516800,
-    "opcode": "Info",
-    "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}",
-    "event_data": {
-      "SubjectUserSid": "S-1-5-18",
-      "SubjectUserName": "SYSTEM",
-      "SubjectDomainName": "NT AUTHORITY",
-      "SubjectLogonId": "0x3e7",
-      "PrivilegeList": "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege"
-    },
-    "task": "Special Logon",
-    "keywords": [
-      "Audit Success"
-    ],
-    "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege",
-    "process": {
-      "pid": 652,
-      "thread": {
-        "id": 4660
-      }
-    }
-  }
-}
--------------------------------------------------------------------------------

libbeat/processors/decode_xml_wineventlog/config.go

@@ -0,0 +1,36 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package decode_xml_wineventlog
+
+type config struct {
+	Field         string `config:"field" validate:"required"`
+	Target        string `config:"target_field"`
+	OverwriteKeys bool   `config:"overwrite_keys"`
+	MapECSFields  bool   `config:"map_ecs_fields"`
+	IgnoreMissing bool   `config:"ignore_missing"`
+	IgnoreFailure bool   `config:"ignore_failure"`
+}
+
+func defaultConfig() config {
+	return config{
+		Field:         "message",
+		OverwriteKeys: true,
+		MapECSFields:  true,
+		Target:        "winlog",
+	}
+}