From 8f6b4ada060314cad4729a0e47d24f7b8cbbc4d4 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 20 Apr 2021 16:50:47 +0200 Subject: [PATCH] [libbeat] New decode xml wineventlog processor (#25115) (#25165) * Move enrich raw functionality to common package * Enrich Raw fields when possible in decode_xml * Add ECS mappings when decoding wineventlog xml * Add decode_xml_wineventlog processor * Add missing fields to config checks * Change event.code type * Fix PR number in changelog * Fix test * Remove document_id and make docs more clear (cherry picked from commit 8cf8f518ad51640633755e430714ca77aa0c9cff) Co-authored-by: Marc Guasch --- CHANGELOG.next.asciidoc | 1 + libbeat/cmd/instance/imports_common.go | 1 + libbeat/docs/processors-list.asciidoc | 14 +- libbeat/processors/decode_xml/config.go | 1 - libbeat/processors/decode_xml/decode_xml.go | 24 ++- .../decode_xml/docs/decode_xml.asciidoc | 82 +-------- libbeat/processors/decode_xml/schema.go | 95 ----------- .../decode_xml_wineventlog/config.go | 36 ++++ .../decode_xml_wineventlog/decoder.go | 41 +++++ .../decode_xml_wineventlog/decoder_windows.go | 94 +++++++++++ .../docs/decode_xml_wineventlog.asciidoc | 110 ++++++++++++ .../decode_xml_wineventlog/processor.go | 159 ++++++++++++++++++ .../processor_test.go} | 91 ++++++++-- winlogbeat/sys/winevent/event.go | 63 ++++++- winlogbeat/sys/winevent/maputil.go | 5 + .../sys/{wineventlog => winevent}/winmeta.go | 13 +- winlogbeat/sys/wineventlog/metadata_store.go | 80 ++++----- .../sys/wineventlog/metadata_store_test.go | 2 +- winlogbeat/sys/wineventlog/renderer.go | 62 ++----- 19 files changed, 678 insertions(+), 296 deletions(-) delete mode 100644 libbeat/processors/decode_xml/schema.go create mode 100644 libbeat/processors/decode_xml_wineventlog/config.go create mode 100644 libbeat/processors/decode_xml_wineventlog/decoder.go create mode 100644 libbeat/processors/decode_xml_wineventlog/decoder_windows.go create mode 100644 libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc create mode 100644 libbeat/processors/decode_xml_wineventlog/processor.go rename libbeat/processors/{decode_xml/schema_test.go => decode_xml_wineventlog/processor_test.go} (58%) rename winlogbeat/sys/{wineventlog => winevent}/winmeta.go (80%) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a3c43c84036..5ad46b29c08 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -373,6 +373,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Libbeat: report queue capacity, output batch size, and output client count to monitoring. {pull}24700[24700] - Add kubernetes.pod.ip field in kubernetes metadata. {pull}25037[25037] - Discover changes in Kubernetes namespace metadata as soon as they happen. {pull}25117[25117] +- Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25115[25115] *Auditbeat* diff --git a/libbeat/cmd/instance/imports_common.go b/libbeat/cmd/instance/imports_common.go index ac767a55964..724919364db 100644 --- a/libbeat/cmd/instance/imports_common.go +++ b/libbeat/cmd/instance/imports_common.go @@ -31,6 +31,7 @@ import ( _ "github.com/elastic/beats/v7/libbeat/processors/communityid" _ "github.com/elastic/beats/v7/libbeat/processors/convert" _ "github.com/elastic/beats/v7/libbeat/processors/decode_xml" + _ "github.com/elastic/beats/v7/libbeat/processors/decode_xml_wineventlog" _ "github.com/elastic/beats/v7/libbeat/processors/dissect" _ "github.com/elastic/beats/v7/libbeat/processors/dns" _ "github.com/elastic/beats/v7/libbeat/processors/extract_array" diff --git a/libbeat/docs/processors-list.asciidoc b/libbeat/docs/processors-list.asciidoc index 3900366708f..b063965ed38 100644 --- a/libbeat/docs/processors-list.asciidoc +++ b/libbeat/docs/processors-list.asciidoc @@ -63,7 +63,10 @@ ifndef::no_decode_json_fields_processor[] * <> endif::[] ifndef::no_decode_xml_processor[] -* <> +* <> +endif::[] +ifndef::no_decode_xml_wineventlog_processor[] +* <> endif::[] ifndef::no_decompress_gzip_field_processor[] * <> @@ -183,6 +186,12 @@ endif::[] ifndef::no_decode_json_fields_processor[] include::{libbeat-processors-dir}/actions/docs/decode_json_fields.asciidoc[] endif::[] +ifndef::no_decode_xml_processor[] +include::{libbeat-processors-dir}/decode_xml/docs/decode_xml.asciidoc[] +endif::[] +ifndef::no_decode_xml_wineventlog_processor[] +include::{libbeat-processors-dir}/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc[] +endif::[] ifndef::no_decompress_gzip_field_processor[] include::{libbeat-processors-dir}/actions/docs/decompress_gzip_field.asciidoc[] endif::[] @@ -234,8 +243,5 @@ endif::[] ifndef::no_urldecode_processor[] include::{libbeat-processors-dir}/urldecode/docs/urldecode.asciidoc[] endif::[] -ifndef::no_decode_xml_processor[] -include::{libbeat-processors-dir}/decode_xml/docs/decode_xml.asciidoc[] -endif::[] //# end::processors-include[] diff --git a/libbeat/processors/decode_xml/config.go b/libbeat/processors/decode_xml/config.go index 21bb426c5b2..289b2eaa0e9 100644 --- a/libbeat/processors/decode_xml/config.go +++ b/libbeat/processors/decode_xml/config.go @@ -25,7 +25,6 @@ type decodeXMLConfig struct { ToLower bool `config:"to_lower"` IgnoreMissing bool `config:"ignore_missing"` IgnoreFailure bool `config:"ignore_failure"` - Schema string `config:"schema"` } func defaultConfig() decodeXMLConfig { diff --git a/libbeat/processors/decode_xml/decode_xml.go b/libbeat/processors/decode_xml/decode_xml.go index 5d841a2b576..ced399c00fa 100644 --- a/libbeat/processors/decode_xml/decode_xml.go +++ b/libbeat/processors/decode_xml/decode_xml.go @@ -18,6 +18,7 @@ package decode_xml import ( + "bytes" "encoding/json" "errors" "fmt" @@ -25,6 +26,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/common/encoding/xml" "github.com/elastic/beats/v7/libbeat/common/jsontransform" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/processors" @@ -35,8 +37,7 @@ import ( type decodeXML struct { decodeXMLConfig - decode decoder - log *logp.Logger + log *logp.Logger } var ( @@ -56,10 +57,9 @@ func init() { "field", "target_field", "overwrite_keys", "document_id", "to_lower", "ignore_missing", - "ignore_failure", "schema", + "ignore_failure", ))) jsprocessor.RegisterPlugin(procName, New) - registerDecoders() } // New constructs a new decode_xml processor. @@ -83,7 +83,6 @@ func newDecodeXML(config decodeXMLConfig) (processors.Processor, error) { return &decodeXML{ decodeXMLConfig: config, - decode: newDecoder(config), log: logp.NewLogger(logName), }, nil } @@ -135,9 +134,24 @@ func (x *decodeXML) run(event *beat.Event) error { if id != "" { event.SetID(id) } + return nil } +func (x *decodeXML) decode(p []byte) (common.MapStr, error) { + dec := xml.NewDecoder(bytes.NewReader(p)) + if x.ToLower { + dec.LowercaseKeys() + } + + out, err := dec.Decode() + if err != nil { + return nil, err + } + + return common.MapStr(out), nil +} + func (x *decodeXML) String() string { json, _ := json.Marshal(x.decodeXMLConfig) return procName + "=" + string(json) diff --git a/libbeat/processors/decode_xml/docs/decode_xml.asciidoc b/libbeat/processors/decode_xml/docs/decode_xml.asciidoc index c21c25081fc..67d1c49111f 100644 --- a/libbeat/processors/decode_xml/docs/decode_xml.asciidoc +++ b/libbeat/processors/decode_xml/docs/decode_xml.asciidoc @@ -1,4 +1,4 @@ -[[decode_xml]] +[[decode-xml]] === Decode XML ++++ @@ -100,9 +100,6 @@ default value is `true`. `to_lower`:: (Optional) Converts all keys to lowercase. Accepts either true or false. The default value is `true`. -`schema`:: (Optional) Specifies the schema of the message. Accepted schemas: `wineventlog`. -If no schema is specified it defaults to using the regular XML to JSON conversion. - `document_id`:: (Optional) XML key to use as the document ID. If configured, the field will be removed from the original XML document and stored in `@metadata._id`. @@ -114,80 +111,3 @@ when a specified field does not exist. Defaults to `false`. Defaults to `false`. See <> for a list of supported conditions. - - -==== Schemas - -When a schema is defined, the specific decoder will parse the configured field. -The ouput of the parsing will be specific to that schema. - -===== Wineventlog - -The `wineventlog` schema decodes Windows Events. - -The decoder will always output the fields formatted in the same way, the -`to_lower` option will be ignored when using this schema decoder. -The output fields will be the same as the -{winlogbeat-ref}/exported-fields-winlog.html#_winlog[winlogbeat winlog fields]. - -Example: - -[source,yaml] -------------------------------------------------------------------------------- -processors: - - decode_xml: - field: event.original - target_field: winlog - to_lower: false -------------------------------------------------------------------------------- - -[source,json] -------------------------------------------------------------------------------- -{ - "event": { - "original": "4672001254800x802000000000000011303SecurityvagrantS-1-5-18SYSTEMNT AUTHORITY0x3e7SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeSpecial privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformationSpecial LogonInfoSecurityMicrosoft Windows security auditing.Audit Success" - } -} -------------------------------------------------------------------------------- - -Will produce the following output: - -[source,json] -------------------------------------------------------------------------------- -{ - "event": { - "original": "4672001254800x802000000000000011303SecurityvagrantS-1-5-18SYSTEMNT AUTHORITY0x3e7SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeSpecial privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformationSpecial LogonInfoSecurityMicrosoft Windows security auditing.Audit Success" - }, - "winlog": { - "channel": "Security", - "outcome": "success", - "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}", - "level": "information", - "event_id": 4672, - "provider_name": "Microsoft-Windows-Security-Auditing", - "record_id": 11303, - "computer_name": "vagrant", - "keywords_raw": 9232379236109516800, - "opcode": "Info", - "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", - "event_data": { - "SubjectUserSid": "S-1-5-18", - "SubjectUserName": "SYSTEM", - "SubjectDomainName": "NT AUTHORITY", - "SubjectLogonId": "0x3e7", - "PrivilegeList": "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege" - }, - "task": "Special Logon", - "keywords": [ - "Audit Success" - ], - "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", - "process": { - "pid": 652, - "thread": { - "id": 4660 - } - } - } -} -------------------------------------------------------------------------------- diff --git a/libbeat/processors/decode_xml/schema.go b/libbeat/processors/decode_xml/schema.go deleted file mode 100644 index 10043ec3d0b..00000000000 --- a/libbeat/processors/decode_xml/schema.go +++ /dev/null @@ -1,95 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package decode_xml - -import ( - "bytes" - "errors" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/encoding/xml" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/winlogbeat/sys/winevent" -) - -const wineventlogSchema = "wineventlog" - -type newDecoderFunc func(cfg decodeXMLConfig) decoder -type decoder func(p []byte) (common.MapStr, error) - -var ( - registeredDecoders = map[string]newDecoderFunc{} - newDefaultDecoder newDecoderFunc = newSchemaLessDecoder -) - -func registerDecoder(schema string, dec newDecoderFunc) error { - if schema == "" { - return errors.New("schema can't be empty") - } - - if dec == nil { - return errors.New("decoder can't be nil") - } - - if _, found := registeredDecoders[schema]; found { - return errors.New("already registered") - } - - registeredDecoders[schema] = dec - - return nil -} - -func newDecoder(cfg decodeXMLConfig) decoder { - newDec, found := registeredDecoders[cfg.Schema] - if !found { - return newDefaultDecoder(cfg) - } - return newDec(cfg) -} - -func registerDecoders() { - log := logp.L().Named(logName) - log.Debug(registerDecoder(wineventlogSchema, newWineventlogDecoder)) -} - -func newSchemaLessDecoder(cfg decodeXMLConfig) decoder { - return func(p []byte) (common.MapStr, error) { - dec := xml.NewDecoder(bytes.NewReader(p)) - if cfg.ToLower { - dec.LowercaseKeys() - } - - out, err := dec.Decode() - if err != nil { - return nil, err - } - - return common.MapStr(out), nil - } -} - -func newWineventlogDecoder(decodeXMLConfig) decoder { - return func(p []byte) (common.MapStr, error) { - evt, err := winevent.UnmarshalXML(p) - if err != nil { - return nil, err - } - return evt.Fields(), nil - } -} diff --git a/libbeat/processors/decode_xml_wineventlog/config.go b/libbeat/processors/decode_xml_wineventlog/config.go new file mode 100644 index 00000000000..061a4d0a52f --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/config.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package decode_xml_wineventlog + +type config struct { + Field string `config:"field" validate:"required"` + Target string `config:"target_field"` + OverwriteKeys bool `config:"overwrite_keys"` + MapECSFields bool `config:"map_ecs_fields"` + IgnoreMissing bool `config:"ignore_missing"` + IgnoreFailure bool `config:"ignore_failure"` +} + +func defaultConfig() config { + return config{ + Field: "message", + OverwriteKeys: true, + MapECSFields: true, + Target: "winlog", + } +} diff --git a/libbeat/processors/decode_xml_wineventlog/decoder.go b/libbeat/processors/decode_xml_wineventlog/decoder.go new file mode 100644 index 00000000000..c1b7843b18d --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/decoder.go @@ -0,0 +1,41 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !windows + +package decode_xml_wineventlog + +import ( + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/winlogbeat/sys/winevent" +) + +type nonWinDecoder struct{} + +func newDecoder() decoder { + return nonWinDecoder{} +} + +func (nonWinDecoder) decode(data []byte) (common.MapStr, common.MapStr, error) { + evt, err := winevent.UnmarshalXML(data) + if err != nil { + return nil, nil, err + } + winevent.EnrichRawValuesWithNames(nil, &evt) + win, ecs := fields(evt) + return win, ecs, nil +} diff --git a/libbeat/processors/decode_xml_wineventlog/decoder_windows.go b/libbeat/processors/decode_xml_wineventlog/decoder_windows.go new file mode 100644 index 00000000000..3b3f7bf3599 --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/decoder_windows.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package decode_xml_wineventlog + +import ( + "sync" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/winlogbeat/sys/winevent" + "github.com/elastic/beats/v7/winlogbeat/sys/wineventlog" +) + +type winDecoder struct { + cache *metadataCache +} + +func newDecoder() decoder { + return &winDecoder{ + cache: &metadataCache{ + store: map[string]*winevent.WinMeta{}, + log: logp.NewLogger(logName), + }, + } +} + +func (dec *winDecoder) decode(data []byte) (common.MapStr, common.MapStr, error) { + evt, err := winevent.UnmarshalXML(data) + if err != nil { + return nil, nil, err + } + md := dec.cache.getPublisherMetadata(evt.Provider.Name) + winevent.EnrichRawValuesWithNames(md, &evt) + win, ecs := fields(evt) + return win, ecs, nil +} + +type metadataCache struct { + store map[string]*winevent.WinMeta + mutex sync.RWMutex + + log *logp.Logger +} + +func (c *metadataCache) getPublisherMetadata(publisher string) *winevent.WinMeta { + // NOTE: This code uses double-check locking to elevate to a write-lock + // when a cache value needs initialized. + c.mutex.RLock() + + // Lookup cached value. + md, found := c.store[publisher] + if !found { + // Elevate to write lock. + c.mutex.RUnlock() + c.mutex.Lock() + defer c.mutex.Unlock() + + // Double-check if the condition changed while upgrading the lock. + md, found = c.store[publisher] + if found { + return md + } + + // Load metadata from the publisher. + md, err := wineventlog.NewPublisherMetadataStore(wineventlog.NilHandle, publisher, c.log) + if err != nil { + // Return an empty store on error (can happen in cases where the + // log was forwarded and the provider doesn't exist on collector). + md = wineventlog.NewEmptyPublisherMetadataStore(publisher, c.log) + } + c.store[publisher] = &md.WinMeta + } else { + c.mutex.RUnlock() + } + + return md +} diff --git a/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc b/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc new file mode 100644 index 00000000000..1df47d3f955 --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc @@ -0,0 +1,110 @@ +[[decode-xml-wineventlog]] +=== Decode XML Wineventlog + +++++ +decode_xml_wineventlog +++++ + +experimental[] + +The `decode_xml_wineventlog` processor decodes Windows Event Log data in XML format that is stored under the `field` +key. It outputs the result into the `target_field`. + +The output fields will be the same as the +{winlogbeat-ref}/exported-fields-winlog.html#_winlog[winlogbeat winlog fields]. + +The supported configuration options are: + +`field`:: (Required) Source field containing the XML. Defaults to `message`. + +`target_field`:: (Required) The field under which the decoded XML will be +written. To merge the decoded XML fields into the root of the event specify +`target_field` with an empty string (`target_field: ""`). The default value is +`winlog`. + +`overwrite_keys`:: (Optional) A boolean that specifies whether keys that already +exist in the event are overwritten by keys from the decoded XML object. The +default value is `true`. + +`map_ecs_fields`:: (Optional) A boolean that specifies whether to map additional ECS +fields when possible. Note that ECS field keys are placed outside of `target_field`. The default value is `true`. + +`ignore_missing`:: (Optional) If `true` the processor will not return an error +when a specified field does not exist. Defaults to `false`. + +`ignore_failure`:: (Optional) Ignore all errors produced by the processor. +Defaults to `false`. + +Example: + +[source,yaml] +------------------------------------------------------------------------------- +processors: + - decode_xml_wineventlog: + field: event.original + target_field: winlog +------------------------------------------------------------------------------- + +[source,json] +------------------------------------------------------------------------------- +{ + "event": { + "original": "4672001254800x802000000000000011303SecurityvagrantS-1-5-18SYSTEMNT AUTHORITY0x3e7SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeSpecial privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformationSpecial LogonInfoSecurityMicrosoft Windows security auditing.Audit Success" + } +} +------------------------------------------------------------------------------- + +Will produce the following output: + +[source,json] +------------------------------------------------------------------------------- +{ + "event": { + "original": "4672001254800x802000000000000011303SecurityvagrantS-1-5-18SYSTEMNT AUTHORITY0x3e7SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeSpecial privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformationSpecial LogonInfoSecurityMicrosoft Windows security auditing.Audit Success", + "action": "Special Logon", + "code": "4672", + "kind": "event", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + }, + "host": { + "name": "vagrant", + }, + "log": { + "level": "information", + }, + "winlog": { + "channel": "Security", + "outcome": "success", + "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}", + "level": "information", + "event_id": 4672, + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 11303, + "computer_name": "vagrant", + "keywords_raw": 9232379236109516800, + "opcode": "Info", + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "event_data": { + "SubjectUserSid": "S-1-5-18", + "SubjectUserName": "SYSTEM", + "SubjectDomainName": "NT AUTHORITY", + "SubjectLogonId": "0x3e7", + "PrivilegeList": "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege" + }, + "task": "Special Logon", + "keywords": [ + "Audit Success" + ], + "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", + "process": { + "pid": 652, + "thread": { + "id": 4660 + } + } + } +} +------------------------------------------------------------------------------- + +See <> for a list of supported conditions. diff --git a/libbeat/processors/decode_xml_wineventlog/processor.go b/libbeat/processors/decode_xml_wineventlog/processor.go new file mode 100644 index 00000000000..b5487764a2b --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/processor.go @@ -0,0 +1,159 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package decode_xml_wineventlog + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/common/jsontransform" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/beats/v7/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" + "github.com/elastic/beats/v7/winlogbeat/sys/winevent" +) + +var ( + errFieldIsNotString = errors.New("field value is not a string") +) + +const ( + procName = "decode_xml_wineventlog" + logName = "processor." + procName +) + +func init() { + processors.RegisterPlugin(procName, + checks.ConfigChecked(New, + checks.RequireFields("field", "target_field"), + checks.AllowedFields( + "field", "target_field", + "overwrite_keys", "map_ecs_fields", + "ignore_missing", "ignore_failure", + ))) + jsprocessor.RegisterPlugin(procName, New) +} + +type processor struct { + config + + decoder decoder + log *logp.Logger +} + +type decoder interface { + decode(data []byte) (win, ecs common.MapStr, err error) +} + +// New constructs a new decode_xml processor. +func New(c *common.Config) (processors.Processor, error) { + config := defaultConfig() + + if err := c.Unpack(&config); err != nil { + return nil, fmt.Errorf("fail to unpack the "+procName+" processor configuration: %s", err) + } + + return newProcessor(config) +} + +func newProcessor(config config) (processors.Processor, error) { + cfgwarn.Experimental("The " + procName + " processor is experimental.") + + return &processor{ + config: config, + decoder: newDecoder(), + log: logp.NewLogger(logName), + }, nil +} + +func (p *processor) Run(event *beat.Event) (*beat.Event, error) { + if err := p.run(event); err != nil && !p.IgnoreFailure { + err = fmt.Errorf("failed in decode_xml_wineventlog on the %q field: %w", p.Field, err) + event.PutValue("error.message", err.Error()) + return event, err + } + return event, nil +} + +func (p *processor) run(event *beat.Event) error { + data, err := event.GetValue(p.Field) + if err != nil { + if p.IgnoreMissing && err == common.ErrKeyNotFound { + return nil + } + return err + } + + text, ok := data.(string) + if !ok { + return errFieldIsNotString + } + + win, ecs, err := p.decoder.decode([]byte(text)) + if err != nil { + return fmt.Errorf("error decoding XML field: %w", err) + } + + if p.Target != "" { + if _, err = event.PutValue(p.Target, win); err != nil { + return fmt.Errorf("failed to put value %v into field %q: %w", win, p.Target, err) + } + } else { + jsontransform.WriteJSONKeys(event, win, false, p.OverwriteKeys, !p.IgnoreFailure) + } + + if p.MapECSFields { + jsontransform.WriteJSONKeys(event, ecs, false, p.OverwriteKeys, !p.IgnoreFailure) + } + + return nil +} + +func (p *processor) String() string { + json, _ := json.Marshal(p.config) + return procName + "=" + string(json) +} + +func fields(evt winevent.Event) (common.MapStr, common.MapStr) { + win := evt.Fields() + + ecs := common.MapStr{} + + ecs.Put("event.kind", "event") + ecs.Put("event.code", evt.EventIdentifier.ID) + ecs.Put("event.provider", evt.Provider.Name) + winevent.AddOptional(ecs, "event.action", evt.Task) + winevent.AddOptional(ecs, "host.name", evt.Computer) + winevent.AddOptional(ecs, "event.outcome", getValue(win, "outcome")) + winevent.AddOptional(ecs, "log.level", getValue(win, "level")) + winevent.AddOptional(ecs, "message", getValue(win, "message")) + winevent.AddOptional(ecs, "error.code", getValue(win, "error.code")) + winevent.AddOptional(ecs, "error.message", getValue(win, "error.message")) + + return win, ecs +} + +func getValue(m common.MapStr, key string) interface{} { + v, _ := m.GetValue(key) + return v +} diff --git a/libbeat/processors/decode_xml/schema_test.go b/libbeat/processors/decode_xml_wineventlog/processor_test.go similarity index 58% rename from libbeat/processors/decode_xml/schema_test.go rename to libbeat/processors/decode_xml_wineventlog/processor_test.go index 655c5156fe2..4e31a013861 100644 --- a/libbeat/processors/decode_xml/schema_test.go +++ b/libbeat/processors/decode_xml_wineventlog/processor_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package decode_xml +package decode_xml_wineventlog import ( "testing" @@ -28,21 +28,88 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) -func TestDecodeSchemas(t *testing.T) { +func TestProcessor(t *testing.T) { var testCases = []struct { - schema string - config decodeXMLConfig + description string + config config Input common.MapStr Output common.MapStr error bool errorMessage string }{ { - schema: wineventlogSchema, - config: decodeXMLConfig{ - Field: "message", - Target: &testXMLTargetField, - Schema: wineventlogSchema, + description: "Decodes properly with default config", + config: defaultConfig(), + Input: common.MapStr{ + "message": "" + + "4672001254800x8020000000000000" + + "11303Securityvagrant" + + "S-1-5-18SYSTEMNT AUTHORITY0x3e7" + + "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\t" + + "SeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege" + + "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\n" + + "Privileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\t" + + "SeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformation" + + "Special LogonInfoSecurityMicrosoft Windows security auditing.Audit Success", + }, + Output: common.MapStr{ + "event": common.MapStr{ + "action": "Special Logon", + "code": uint32(4672), + "kind": "event", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + }, + "host": common.MapStr{ + "name": "vagrant", + }, + "log": common.MapStr{ + "level": "information", + }, + "winlog": common.MapStr{ + "channel": "Security", + "outcome": "success", + "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}", + "level": "information", + "event_id": uint32(4672), + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": uint64(11303), + "computer_name": "vagrant", + "time_created": func() time.Time { + t, _ := time.Parse(time.RFC3339Nano, "2021-03-23T09:56:13.137310000Z") + return t + }(), + "opcode": "Info", + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "event_data": common.MapStr{ + "SubjectUserSid": "S-1-5-18", + "SubjectUserName": "SYSTEM", + "SubjectDomainName": "NT AUTHORITY", + "SubjectLogonId": "0x3e7", + "PrivilegeList": "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", + }, + "task": "Special Logon", + "keywords": []string{ + "Audit Success", + }, + "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", + "process": common.MapStr{ + "pid": uint32(652), + "thread": common.MapStr{ + "id": uint32(4660), + }, + }, + }, + "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", + }, + }, + { + description: "Decodes without ECS", + config: config{ + Field: "message", + OverwriteKeys: true, + MapECSFields: false, + Target: "winlog", }, Input: common.MapStr{ "message": "" + @@ -57,7 +124,7 @@ func TestDecodeSchemas(t *testing.T) { "Special LogonInfoSecurityMicrosoft Windows security auditing.Audit Success", }, Output: common.MapStr{ - "xml": common.MapStr{ + "winlog": common.MapStr{ "channel": "Security", "outcome": "success", "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}", @@ -107,10 +174,10 @@ func TestDecodeSchemas(t *testing.T) { for _, test := range testCases { test := test - t.Run(test.schema, func(t *testing.T) { + t.Run(test.description, func(t *testing.T) { t.Parallel() - f, err := newDecodeXML(test.config) + f, err := newProcessor(test.config) require.NoError(t, err) event := &beat.Event{ diff --git a/winlogbeat/sys/winevent/event.go b/winlogbeat/sys/winevent/event.go index 53c6b49abd5..8af8d0c9a75 100644 --- a/winlogbeat/sys/winevent/event.go +++ b/winlogbeat/sys/winevent/event.go @@ -44,6 +44,11 @@ var ( const ( keywordAuditFailure = 0x10000000000000 keywordAuditSuccess = 0x20000000000000 + + // keywordClassic indicates the log was published with the "classic" event + // logging API. + // https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.eventing.reader.standardeventkeywords?view=netframework-4.8 + keywordClassic = 0x80000000000000 ) // UnmarshalXML unmarshals the given XML into a new Event. @@ -101,7 +106,6 @@ func (e Event) Fields() common.MapStr { AddOptional(win, "keywords", e.Keywords) AddOptional(win, "opcode", e.Opcode) AddOptional(win, "provider_guid", e.Provider.GUID) - AddOptional(win, "task", e.Task) AddOptional(win, "version", e.Version) AddOptional(win, "time_created", e.TimeCreated.SystemTime) @@ -330,3 +334,60 @@ func (v *HexInt64) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { *v = HexInt64(num) return nil } + +// EnrichRawValuesWithNames adds the names associated with the raw system +// property values. It enriches the event with keywords, opcode, level, and +// task. The search order is defined in the EvtFormatMessage documentation. +func EnrichRawValuesWithNames(publisherMeta *WinMeta, event *Event) { + // Keywords. Each bit in the value can represent a keyword. + rawKeyword := int64(event.KeywordsRaw) + isClassic := keywordClassic&rawKeyword > 0 + + if len(event.Keywords) == 0 { + for mask, keyword := range defaultWinMeta.Keywords { + if rawKeyword&mask > 0 { + event.Keywords = append(event.Keywords, keyword) + rawKeyword -= mask + } + } + if publisherMeta != nil { + for mask, keyword := range publisherMeta.Keywords { + if rawKeyword&mask > 0 { + event.Keywords = append(event.Keywords, keyword) + rawKeyword -= mask + } + } + } + } + + var found bool + if event.Opcode == "" { + // Opcode (search in defaultWinMeta first). + if !isClassic { + event.Opcode, found = defaultWinMeta.Opcodes[event.OpcodeRaw] + if !found && publisherMeta != nil { + event.Opcode = publisherMeta.Opcodes[event.OpcodeRaw] + } + } + } + + if event.Level == "" { + // Level (search in defaultWinMeta first). + event.Level, found = defaultWinMeta.Levels[event.LevelRaw] + if !found && publisherMeta != nil { + event.Level = publisherMeta.Levels[event.LevelRaw] + } + } + + if event.Task == "" { + if publisherMeta != nil { + // Task (fall-back to defaultWinMeta if not found). + event.Task, found = publisherMeta.Tasks[event.TaskRaw] + if !found { + event.Task = defaultWinMeta.Tasks[event.TaskRaw] + } + } else { + event.Task = defaultWinMeta.Tasks[event.TaskRaw] + } + } +} diff --git a/winlogbeat/sys/winevent/maputil.go b/winlogbeat/sys/winevent/maputil.go index 41fe694c88e..45a265ae8c6 100644 --- a/winlogbeat/sys/winevent/maputil.go +++ b/winlogbeat/sys/winevent/maputil.go @@ -80,6 +80,10 @@ func AddPairs(m common.MapStr, key string, pairs []KeyValue) common.MapStr { // isZero return true if the given value is the zero value for its type. func isZero(i interface{}) bool { + if i == nil { + return true + } + v := reflect.ValueOf(i) switch v.Kind() { case reflect.Array, reflect.String: @@ -95,5 +99,6 @@ func isZero(i interface{}) bool { case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: return v.IsNil() } + return false } diff --git a/winlogbeat/sys/wineventlog/winmeta.go b/winlogbeat/sys/winevent/winmeta.go similarity index 80% rename from winlogbeat/sys/wineventlog/winmeta.go rename to winlogbeat/sys/winevent/winmeta.go index 140b1c00066..1d7ac8f5c68 100644 --- a/winlogbeat/sys/wineventlog/winmeta.go +++ b/winlogbeat/sys/winevent/winmeta.go @@ -15,13 +15,18 @@ // specific language governing permissions and limitations // under the License. -// +build windows +package winevent -package wineventlog +type WinMeta struct { + Keywords map[int64]string // Keyword bit mask to keyword name. + Opcodes map[uint8]string // Opcode value to name. + Levels map[uint8]string // Level value to name. + Tasks map[uint16]string // Task value to name. +} -// winMeta contains the static values that are a common across Windows. These +// defaultWinMeta contains the static values that are a common across Windows. These // values are from winmeta.xml inside the Windows SDK. -var winMeta = &publisherMetadataStore{ +var defaultWinMeta = &WinMeta{ Keywords: map[int64]string{ 0: "AnyKeyword", 0x1000000000000: "Response Time", diff --git a/winlogbeat/sys/wineventlog/metadata_store.go b/winlogbeat/sys/wineventlog/metadata_store.go index fe94c03e168..a02da985b4f 100644 --- a/winlogbeat/sys/wineventlog/metadata_store.go +++ b/winlogbeat/sys/wineventlog/metadata_store.go @@ -43,32 +43,30 @@ var ( } ) -// publisherMetadataStore stores metadata from a publisher. -type publisherMetadataStore struct { +// PublisherMetadataStore stores metadata from a publisher. +type PublisherMetadataStore struct { Metadata *PublisherMetadata // Handle to the publisher metadata. May be nil. - Keywords map[int64]string // Keyword bit mask to keyword name. - Opcodes map[uint8]string // Opcode value to name. - Levels map[uint8]string // Level value to name. - Tasks map[uint16]string // Task value to name. + + winevent.WinMeta // Event ID to event metadata (message and event data param names). - Events map[uint16]*eventMetadata + Events map[uint16]*EventMetadata // Event ID to map of fingerprints to event metadata. The fingerprint value // is hash of the event data parameters count and types. - EventFingerprints map[uint16]map[uint64]*eventMetadata + EventFingerprints map[uint16]map[uint64]*EventMetadata mutex sync.RWMutex log *logp.Logger } -func newPublisherMetadataStore(session EvtHandle, provider string, log *logp.Logger) (*publisherMetadataStore, error) { +func NewPublisherMetadataStore(session EvtHandle, provider string, log *logp.Logger) (*PublisherMetadataStore, error) { md, err := NewPublisherMetadata(session, provider) if err != nil { return nil, err } - store := &publisherMetadataStore{ + store := &PublisherMetadataStore{ Metadata: md, - EventFingerprints: map[uint16]map[uint64]*eventMetadata{}, + EventFingerprints: map[uint16]map[uint64]*EventMetadata{}, log: log.With("publisher", provider), } @@ -88,21 +86,23 @@ func newPublisherMetadataStore(session EvtHandle, provider string, log *logp.Log return store, nil } -// newEmptyPublisherMetadataStore creates an empty metadata store for cases +// NewEmptyPublisherMetadataStore creates an empty metadata store for cases // where no local publisher metadata exists. -func newEmptyPublisherMetadataStore(provider string, log *logp.Logger) *publisherMetadataStore { - return &publisherMetadataStore{ - Keywords: map[int64]string{}, - Opcodes: map[uint8]string{}, - Levels: map[uint8]string{}, - Tasks: map[uint16]string{}, - Events: map[uint16]*eventMetadata{}, - EventFingerprints: map[uint16]map[uint64]*eventMetadata{}, +func NewEmptyPublisherMetadataStore(provider string, log *logp.Logger) *PublisherMetadataStore { + return &PublisherMetadataStore{ + WinMeta: winevent.WinMeta{ + Keywords: map[int64]string{}, + Opcodes: map[uint8]string{}, + Levels: map[uint8]string{}, + Tasks: map[uint16]string{}, + }, + Events: map[uint16]*EventMetadata{}, + EventFingerprints: map[uint16]map[uint64]*EventMetadata{}, log: log.With("publisher", provider, "empty", true), } } -func (s *publisherMetadataStore) initKeywords() error { +func (s *PublisherMetadataStore) initKeywords() error { keywords, err := s.Metadata.Keywords() if err != nil { return err @@ -119,7 +119,7 @@ func (s *publisherMetadataStore) initKeywords() error { return nil } -func (s *publisherMetadataStore) initOpcodes() error { +func (s *PublisherMetadataStore) initOpcodes() error { opcodes, err := s.Metadata.Opcodes() if err != nil { return err @@ -135,7 +135,7 @@ func (s *publisherMetadataStore) initOpcodes() error { return nil } -func (s *publisherMetadataStore) initLevels() error { +func (s *PublisherMetadataStore) initLevels() error { levels, err := s.Metadata.Levels() if err != nil { return err @@ -152,7 +152,7 @@ func (s *publisherMetadataStore) initLevels() error { return nil } -func (s *publisherMetadataStore) initTasks() error { +func (s *PublisherMetadataStore) initTasks() error { tasks, err := s.Metadata.Tasks() if err != nil { return err @@ -168,14 +168,14 @@ func (s *publisherMetadataStore) initTasks() error { return nil } -func (s *publisherMetadataStore) initEvents() error { +func (s *PublisherMetadataStore) initEvents() error { itr, err := s.Metadata.EventMetadataIterator() if err != nil { return err } defer itr.Close() - s.Events = map[uint16]*eventMetadata{} + s.Events = map[uint16]*EventMetadata{} for itr.Next() { evt, err := newEventMetadataFromPublisherMetadata(itr, s.Metadata) if err != nil { @@ -188,7 +188,7 @@ func (s *publisherMetadataStore) initEvents() error { return itr.Err() } -func (s *publisherMetadataStore) getEventMetadata(eventID uint16, eventDataFingerprint uint64, eventHandle EvtHandle) *eventMetadata { +func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFingerprint uint64, eventHandle EvtHandle) *EventMetadata { // Use a read lock to get a cached value. s.mutex.RLock() fingerprints, found := s.EventFingerprints[eventID] @@ -207,7 +207,7 @@ func (s *publisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge fingerprints, found = s.EventFingerprints[eventID] if !found { - fingerprints = map[uint64]*eventMetadata{} + fingerprints = map[uint64]*EventMetadata{} s.EventFingerprints[eventID] = fingerprints } @@ -268,7 +268,7 @@ func (s *publisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge return em } -func (s *publisherMetadataStore) Close() error { +func (s *PublisherMetadataStore) Close() error { if s.Metadata != nil { s.mutex.Lock() defer s.mutex.Unlock() @@ -278,7 +278,7 @@ func (s *publisherMetadataStore) Close() error { return nil } -type eventMetadata struct { +type EventMetadata struct { EventID uint16 // Event ID. Version uint8 // Event format version. MsgStatic string // Used when the message has no parameters. @@ -288,7 +288,7 @@ type eventMetadata struct { // newEventMetadataFromEventHandle collects metadata about an event type using // the handle of an event. -func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle EvtHandle) (*eventMetadata, error) { +func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle EvtHandle) (*EventMetadata, error) { xml, err := getEventXML(publisher, eventHandle) if err != nil { return nil, err @@ -301,7 +301,7 @@ func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle E return nil, errors.Wrap(err, "failed to unmarshal XML") } - em := &eventMetadata{ + em := &EventMetadata{ EventID: uint16(event.EventIdentifier.ID), Version: uint8(event.Version), } @@ -335,8 +335,8 @@ func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle E // newEventMetadataFromPublisherMetadata collects metadata about an event type // using the publisher metadata. -func newEventMetadataFromPublisherMetadata(itr *EventMetadataIterator, publisher *PublisherMetadata) (*eventMetadata, error) { - em := &eventMetadata{} +func newEventMetadataFromPublisherMetadata(itr *EventMetadataIterator, publisher *PublisherMetadata) (*EventMetadata, error) { + em := &EventMetadata{} err := multierr.Combine( em.initEventID(itr), em.initVersion(itr), @@ -349,7 +349,7 @@ func newEventMetadataFromPublisherMetadata(itr *EventMetadataIterator, publisher return em, nil } -func (em *eventMetadata) initEventID(itr *EventMetadataIterator) error { +func (em *EventMetadata) initEventID(itr *EventMetadataIterator) error { id, err := itr.EventID() if err != nil { return err @@ -359,7 +359,7 @@ func (em *eventMetadata) initEventID(itr *EventMetadataIterator) error { return nil } -func (em *eventMetadata) initVersion(itr *EventMetadataIterator) error { +func (em *EventMetadata) initVersion(itr *EventMetadataIterator) error { version, err := itr.Version() if err != nil { return err @@ -368,7 +368,7 @@ func (em *eventMetadata) initVersion(itr *EventMetadataIterator) error { return nil } -func (em *eventMetadata) initEventDataTemplate(itr *EventMetadataIterator) error { +func (em *EventMetadata) initEventDataTemplate(itr *EventMetadataIterator) error { xml, err := itr.Template() if err != nil { return err @@ -391,7 +391,7 @@ func (em *eventMetadata) initEventDataTemplate(itr *EventMetadataIterator) error return nil } -func (em *eventMetadata) initEventMessage(itr *EventMetadataIterator, publisher *PublisherMetadata) error { +func (em *EventMetadata) initEventMessage(itr *EventMetadataIterator, publisher *PublisherMetadata) error { messageID, err := itr.MessageID() if err != nil { return err @@ -405,7 +405,7 @@ func (em *eventMetadata) initEventMessage(itr *EventMetadataIterator, publisher return em.setMessage(msg) } -func (em *eventMetadata) setMessage(msg string) error { +func (em *EventMetadata) setMessage(msg string) error { msg = sys.RemoveWindowsLineEndings(msg) tmplID := strconv.Itoa(int(em.EventID)) @@ -424,7 +424,7 @@ func (em *eventMetadata) setMessage(msg string) error { return nil } -func (em *eventMetadata) equal(other *eventMetadata) bool { +func (em *EventMetadata) equal(other *EventMetadata) bool { if em == other { return true } diff --git a/winlogbeat/sys/wineventlog/metadata_store_test.go b/winlogbeat/sys/wineventlog/metadata_store_test.go index 0c89251bb7a..205f703105d 100644 --- a/winlogbeat/sys/wineventlog/metadata_store_test.go +++ b/winlogbeat/sys/wineventlog/metadata_store_test.go @@ -30,7 +30,7 @@ import ( func TestPublisherMetadataStore(t *testing.T) { logp.TestingSetup() - s, err := newPublisherMetadataStore( + s, err := NewPublisherMetadataStore( NilHandle, "Microsoft-Windows-Security-Auditing", logp.NewLogger("metadata")) diff --git a/winlogbeat/sys/wineventlog/renderer.go b/winlogbeat/sys/wineventlog/renderer.go index d52f4399fa2..40392abd15d 100644 --- a/winlogbeat/sys/wineventlog/renderer.go +++ b/winlogbeat/sys/wineventlog/renderer.go @@ -48,7 +48,7 @@ const ( // Renderer is used for converting event log handles into complete events. type Renderer struct { // Cache of publisher metadata. Maps publisher names to stored metadata. - metadataCache map[string]*publisherMetadataStore + metadataCache map[string]*PublisherMetadataStore // Mutex to guard the metadataCache. The other members are immutable. mutex sync.RWMutex @@ -71,7 +71,7 @@ func NewRenderer(session EvtHandle, log *logp.Logger) (*Renderer, error) { } return &Renderer{ - metadataCache: map[string]*publisherMetadataStore{}, + metadataCache: map[string]*PublisherMetadataStore{}, session: session, systemContext: systemContext, userContext: userContext, @@ -112,7 +112,7 @@ func (r *Renderer) Render(handle EvtHandle) (*winevent.Event, error) { } // Associate raw system properties to names (e.g. level=2 to Error). - enrichRawValuesWithNames(md, event) + winevent.EnrichRawValuesWithNames(&md.WinMeta, event) eventData, fingerprint, err := r.renderUser(handle, event) if err != nil { @@ -135,9 +135,9 @@ func (r *Renderer) Render(handle EvtHandle) (*winevent.Event, error) { return event, nil } -// getPublisherMetadata return a publisherMetadataStore for the provider. It +// getPublisherMetadata return a PublisherMetadataStore for the provider. It // never returns nil, but may return an error if it couldn't open a publisher. -func (r *Renderer) getPublisherMetadata(publisher string) (*publisherMetadataStore, error) { +func (r *Renderer) getPublisherMetadata(publisher string) (*PublisherMetadataStore, error) { var err error // NOTE: This code uses double-check locking to elevate to a write-lock @@ -159,11 +159,11 @@ func (r *Renderer) getPublisherMetadata(publisher string) (*publisherMetadataSto } // Load metadata from the publisher. - md, err = newPublisherMetadataStore(r.session, publisher, r.log) + md, err = NewPublisherMetadataStore(r.session, publisher, r.log) if err != nil { // Return an empty store on error (can happen in cases where the // log was forwarded and the provider doesn't exist on collector). - md = newEmptyPublisherMetadataStore(publisher, r.log) + md = NewEmptyPublisherMetadataStore(publisher, r.log) err = errors.Wrapf(err, "failed to load publisher metadata for %v "+ "(returning an empty metadata store)", publisher) } @@ -307,7 +307,7 @@ func (r *Renderer) render(context EvtHandle, eventHandle EvtHandle) (*sys.Pooled } // addEventData adds the event/user data values to the event. -func (r *Renderer) addEventData(evtMeta *eventMetadata, values []interface{}, event *winevent.Event) { +func (r *Renderer) addEventData(evtMeta *EventMetadata, values []interface{}, event *winevent.Event) { if len(values) == 0 { return } @@ -361,8 +361,8 @@ func (r *Renderer) addEventData(evtMeta *eventMetadata, values []interface{}, ev } // formatMessage adds the message to the event. -func (r *Renderer) formatMessage(publisherMeta *publisherMetadataStore, - eventMeta *eventMetadata, eventHandle EvtHandle, values []interface{}, +func (r *Renderer) formatMessage(publisherMeta *PublisherMetadataStore, + eventMeta *EventMetadata, eventHandle EvtHandle, values []interface{}, eventID uint16) (string, error) { if eventMeta != nil { @@ -394,45 +394,3 @@ func (r *Renderer) formatMessageFromTemplate(msgTmpl *template.Template, values return string(bb.Bytes()), nil } - -// enrichRawValuesWithNames adds the names associated with the raw system -// property values. It enriches the event with keywords, opcode, level, and -// task. The search order is defined in the EvtFormatMessage documentation. -func enrichRawValuesWithNames(publisherMeta *publisherMetadataStore, event *winevent.Event) { - // Keywords. Each bit in the value can represent a keyword. - rawKeyword := int64(event.KeywordsRaw) - isClassic := keywordClassic&rawKeyword > 0 - for mask, keyword := range winMeta.Keywords { - if rawKeyword&mask > 0 { - event.Keywords = append(event.Keywords, keyword) - rawKeyword -= mask - } - } - for mask, keyword := range publisherMeta.Keywords { - if rawKeyword&mask > 0 { - event.Keywords = append(event.Keywords, keyword) - rawKeyword -= mask - } - } - - // Opcode (search in winmeta first). - var found bool - if !isClassic { - event.Opcode, found = winMeta.Opcodes[event.OpcodeRaw] - if !found { - event.Opcode = publisherMeta.Opcodes[event.OpcodeRaw] - } - } - - // Level (search in winmeta first). - event.Level, found = winMeta.Levels[event.LevelRaw] - if !found { - event.Level = publisherMeta.Levels[event.LevelRaw] - } - - // Task (fall-back to winmeta if not found). - event.Task, found = publisherMeta.Tasks[event.TaskRaw] - if !found { - event.Task = winMeta.Tasks[event.TaskRaw] - } -}