From ea2a84f391bc47fab551b4784960e848b1d54307 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 15 Apr 2021 11:33:33 +0200 Subject: [PATCH 1/9] Move enrich raw functionality to common package --- winlogbeat/sys/winevent/event.go | 63 ++++++++++++++- .../sys/{wineventlog => winevent}/winmeta.go | 13 ++- winlogbeat/sys/wineventlog/metadata_store.go | 80 +++++++++---------- .../sys/wineventlog/metadata_store_test.go | 2 +- winlogbeat/sys/wineventlog/renderer.go | 62 +++----------- 5 files changed, 122 insertions(+), 98 deletions(-) rename winlogbeat/sys/{wineventlog => winevent}/winmeta.go (80%) diff --git a/winlogbeat/sys/winevent/event.go b/winlogbeat/sys/winevent/event.go index 53c6b49abd5..8af8d0c9a75 100644 --- a/winlogbeat/sys/winevent/event.go +++ b/winlogbeat/sys/winevent/event.go @@ -44,6 +44,11 @@ var ( const ( keywordAuditFailure = 0x10000000000000 keywordAuditSuccess = 0x20000000000000 + + // keywordClassic indicates the log was published with the "classic" event + // logging API. + // https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.eventing.reader.standardeventkeywords?view=netframework-4.8 + keywordClassic = 0x80000000000000 ) // UnmarshalXML unmarshals the given XML into a new Event. @@ -101,7 +106,6 @@ func (e Event) Fields() common.MapStr { AddOptional(win, "keywords", e.Keywords) AddOptional(win, "opcode", e.Opcode) AddOptional(win, "provider_guid", e.Provider.GUID) - AddOptional(win, "task", e.Task) AddOptional(win, "version", e.Version) AddOptional(win, "time_created", e.TimeCreated.SystemTime) @@ -330,3 +334,60 @@ func (v *HexInt64) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { *v = HexInt64(num) return nil } + +// EnrichRawValuesWithNames adds the names associated with the raw system +// property values. It enriches the event with keywords, opcode, level, and +// task. The search order is defined in the EvtFormatMessage documentation. +func EnrichRawValuesWithNames(publisherMeta *WinMeta, event *Event) { + // Keywords. Each bit in the value can represent a keyword. + rawKeyword := int64(event.KeywordsRaw) + isClassic := keywordClassic&rawKeyword > 0 + + if len(event.Keywords) == 0 { + for mask, keyword := range defaultWinMeta.Keywords { + if rawKeyword&mask > 0 { + event.Keywords = append(event.Keywords, keyword) + rawKeyword -= mask + } + } + if publisherMeta != nil { + for mask, keyword := range publisherMeta.Keywords { + if rawKeyword&mask > 0 { + event.Keywords = append(event.Keywords, keyword) + rawKeyword -= mask + } + } + } + } + + var found bool + if event.Opcode == "" { + // Opcode (search in defaultWinMeta first). + if !isClassic { + event.Opcode, found = defaultWinMeta.Opcodes[event.OpcodeRaw] + if !found && publisherMeta != nil { + event.Opcode = publisherMeta.Opcodes[event.OpcodeRaw] + } + } + } + + if event.Level == "" { + // Level (search in defaultWinMeta first). + event.Level, found = defaultWinMeta.Levels[event.LevelRaw] + if !found && publisherMeta != nil { + event.Level = publisherMeta.Levels[event.LevelRaw] + } + } + + if event.Task == "" { + if publisherMeta != nil { + // Task (fall-back to defaultWinMeta if not found). + event.Task, found = publisherMeta.Tasks[event.TaskRaw] + if !found { + event.Task = defaultWinMeta.Tasks[event.TaskRaw] + } + } else { + event.Task = defaultWinMeta.Tasks[event.TaskRaw] + } + } +} diff --git a/winlogbeat/sys/wineventlog/winmeta.go b/winlogbeat/sys/winevent/winmeta.go similarity index 80% rename from winlogbeat/sys/wineventlog/winmeta.go rename to winlogbeat/sys/winevent/winmeta.go index 140b1c00066..1d7ac8f5c68 100644 --- a/winlogbeat/sys/wineventlog/winmeta.go +++ b/winlogbeat/sys/winevent/winmeta.go @@ -15,13 +15,18 @@ // specific language governing permissions and limitations // under the License. -// +build windows +package winevent -package wineventlog +type WinMeta struct { + Keywords map[int64]string // Keyword bit mask to keyword name. + Opcodes map[uint8]string // Opcode value to name. + Levels map[uint8]string // Level value to name. + Tasks map[uint16]string // Task value to name. +} -// winMeta contains the static values that are a common across Windows. These +// defaultWinMeta contains the static values that are a common across Windows. These // values are from winmeta.xml inside the Windows SDK. -var winMeta = &publisherMetadataStore{ +var defaultWinMeta = &WinMeta{ Keywords: map[int64]string{ 0: "AnyKeyword", 0x1000000000000: "Response Time", diff --git a/winlogbeat/sys/wineventlog/metadata_store.go b/winlogbeat/sys/wineventlog/metadata_store.go index fe94c03e168..a02da985b4f 100644 --- a/winlogbeat/sys/wineventlog/metadata_store.go +++ b/winlogbeat/sys/wineventlog/metadata_store.go @@ -43,32 +43,30 @@ var ( } ) -// publisherMetadataStore stores metadata from a publisher. -type publisherMetadataStore struct { +// PublisherMetadataStore stores metadata from a publisher. +type PublisherMetadataStore struct { Metadata *PublisherMetadata // Handle to the publisher metadata. May be nil. - Keywords map[int64]string // Keyword bit mask to keyword name. - Opcodes map[uint8]string // Opcode value to name. - Levels map[uint8]string // Level value to name. - Tasks map[uint16]string // Task value to name. + + winevent.WinMeta // Event ID to event metadata (message and event data param names). - Events map[uint16]*eventMetadata + Events map[uint16]*EventMetadata // Event ID to map of fingerprints to event metadata. The fingerprint value // is hash of the event data parameters count and types. - EventFingerprints map[uint16]map[uint64]*eventMetadata + EventFingerprints map[uint16]map[uint64]*EventMetadata mutex sync.RWMutex log *logp.Logger } -func newPublisherMetadataStore(session EvtHandle, provider string, log *logp.Logger) (*publisherMetadataStore, error) { +func NewPublisherMetadataStore(session EvtHandle, provider string, log *logp.Logger) (*PublisherMetadataStore, error) { md, err := NewPublisherMetadata(session, provider) if err != nil { return nil, err } - store := &publisherMetadataStore{ + store := &PublisherMetadataStore{ Metadata: md, - EventFingerprints: map[uint16]map[uint64]*eventMetadata{}, + EventFingerprints: map[uint16]map[uint64]*EventMetadata{}, log: log.With("publisher", provider), } @@ -88,21 +86,23 @@ func newPublisherMetadataStore(session EvtHandle, provider string, log *logp.Log return store, nil } -// newEmptyPublisherMetadataStore creates an empty metadata store for cases +// NewEmptyPublisherMetadataStore creates an empty metadata store for cases // where no local publisher metadata exists. -func newEmptyPublisherMetadataStore(provider string, log *logp.Logger) *publisherMetadataStore { - return &publisherMetadataStore{ - Keywords: map[int64]string{}, - Opcodes: map[uint8]string{}, - Levels: map[uint8]string{}, - Tasks: map[uint16]string{}, - Events: map[uint16]*eventMetadata{}, - EventFingerprints: map[uint16]map[uint64]*eventMetadata{}, +func NewEmptyPublisherMetadataStore(provider string, log *logp.Logger) *PublisherMetadataStore { + return &PublisherMetadataStore{ + WinMeta: winevent.WinMeta{ + Keywords: map[int64]string{}, + Opcodes: map[uint8]string{}, + Levels: map[uint8]string{}, + Tasks: map[uint16]string{}, + }, + Events: map[uint16]*EventMetadata{}, + EventFingerprints: map[uint16]map[uint64]*EventMetadata{}, log: log.With("publisher", provider, "empty", true), } } -func (s *publisherMetadataStore) initKeywords() error { +func (s *PublisherMetadataStore) initKeywords() error { keywords, err := s.Metadata.Keywords() if err != nil { return err @@ -119,7 +119,7 @@ func (s *publisherMetadataStore) initKeywords() error { return nil } -func (s *publisherMetadataStore) initOpcodes() error { +func (s *PublisherMetadataStore) initOpcodes() error { opcodes, err := s.Metadata.Opcodes() if err != nil { return err @@ -135,7 +135,7 @@ func (s *publisherMetadataStore) initOpcodes() error { return nil } -func (s *publisherMetadataStore) initLevels() error { +func (s *PublisherMetadataStore) initLevels() error { levels, err := s.Metadata.Levels() if err != nil { return err @@ -152,7 +152,7 @@ func (s *publisherMetadataStore) initLevels() error { return nil } -func (s *publisherMetadataStore) initTasks() error { +func (s *PublisherMetadataStore) initTasks() error { tasks, err := s.Metadata.Tasks() if err != nil { return err @@ -168,14 +168,14 @@ func (s *publisherMetadataStore) initTasks() error { return nil } -func (s *publisherMetadataStore) initEvents() error { +func (s *PublisherMetadataStore) initEvents() error { itr, err := s.Metadata.EventMetadataIterator() if err != nil { return err } defer itr.Close() - s.Events = map[uint16]*eventMetadata{} + s.Events = map[uint16]*EventMetadata{} for itr.Next() { evt, err := newEventMetadataFromPublisherMetadata(itr, s.Metadata) if err != nil { @@ -188,7 +188,7 @@ func (s *publisherMetadataStore) initEvents() error { return itr.Err() } -func (s *publisherMetadataStore) getEventMetadata(eventID uint16, eventDataFingerprint uint64, eventHandle EvtHandle) *eventMetadata { +func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFingerprint uint64, eventHandle EvtHandle) *EventMetadata { // Use a read lock to get a cached value. s.mutex.RLock() fingerprints, found := s.EventFingerprints[eventID] @@ -207,7 +207,7 @@ func (s *publisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge fingerprints, found = s.EventFingerprints[eventID] if !found { - fingerprints = map[uint64]*eventMetadata{} + fingerprints = map[uint64]*EventMetadata{} s.EventFingerprints[eventID] = fingerprints } @@ -268,7 +268,7 @@ func (s *publisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge return em } -func (s *publisherMetadataStore) Close() error { +func (s *PublisherMetadataStore) Close() error { if s.Metadata != nil { s.mutex.Lock() defer s.mutex.Unlock() @@ -278,7 +278,7 @@ func (s *publisherMetadataStore) Close() error { return nil } -type eventMetadata struct { +type EventMetadata struct { EventID uint16 // Event ID. Version uint8 // Event format version. MsgStatic string // Used when the message has no parameters. @@ -288,7 +288,7 @@ type eventMetadata struct { // newEventMetadataFromEventHandle collects metadata about an event type using // the handle of an event. -func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle EvtHandle) (*eventMetadata, error) { +func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle EvtHandle) (*EventMetadata, error) { xml, err := getEventXML(publisher, eventHandle) if err != nil { return nil, err @@ -301,7 +301,7 @@ func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle E return nil, errors.Wrap(err, "failed to unmarshal XML") } - em := &eventMetadata{ + em := &EventMetadata{ EventID: uint16(event.EventIdentifier.ID), Version: uint8(event.Version), } @@ -335,8 +335,8 @@ func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle E // newEventMetadataFromPublisherMetadata collects metadata about an event type // using the publisher metadata. -func newEventMetadataFromPublisherMetadata(itr *EventMetadataIterator, publisher *PublisherMetadata) (*eventMetadata, error) { - em := &eventMetadata{} +func newEventMetadataFromPublisherMetadata(itr *EventMetadataIterator, publisher *PublisherMetadata) (*EventMetadata, error) { + em := &EventMetadata{} err := multierr.Combine( em.initEventID(itr), em.initVersion(itr), @@ -349,7 +349,7 @@ func newEventMetadataFromPublisherMetadata(itr *EventMetadataIterator, publisher return em, nil } -func (em *eventMetadata) initEventID(itr *EventMetadataIterator) error { +func (em *EventMetadata) initEventID(itr *EventMetadataIterator) error { id, err := itr.EventID() if err != nil { return err @@ -359,7 +359,7 @@ func (em *eventMetadata) initEventID(itr *EventMetadataIterator) error { return nil } -func (em *eventMetadata) initVersion(itr *EventMetadataIterator) error { +func (em *EventMetadata) initVersion(itr *EventMetadataIterator) error { version, err := itr.Version() if err != nil { return err @@ -368,7 +368,7 @@ func (em *eventMetadata) initVersion(itr *EventMetadataIterator) error { return nil } -func (em *eventMetadata) initEventDataTemplate(itr *EventMetadataIterator) error { +func (em *EventMetadata) initEventDataTemplate(itr *EventMetadataIterator) error { xml, err := itr.Template() if err != nil { return err @@ -391,7 +391,7 @@ func (em *eventMetadata) initEventDataTemplate(itr *EventMetadataIterator) error return nil } -func (em *eventMetadata) initEventMessage(itr *EventMetadataIterator, publisher *PublisherMetadata) error { +func (em *EventMetadata) initEventMessage(itr *EventMetadataIterator, publisher *PublisherMetadata) error { messageID, err := itr.MessageID() if err != nil { return err @@ -405,7 +405,7 @@ func (em *eventMetadata) initEventMessage(itr *EventMetadataIterator, publisher return em.setMessage(msg) } -func (em *eventMetadata) setMessage(msg string) error { +func (em *EventMetadata) setMessage(msg string) error { msg = sys.RemoveWindowsLineEndings(msg) tmplID := strconv.Itoa(int(em.EventID)) @@ -424,7 +424,7 @@ func (em *eventMetadata) setMessage(msg string) error { return nil } -func (em *eventMetadata) equal(other *eventMetadata) bool { +func (em *EventMetadata) equal(other *EventMetadata) bool { if em == other { return true } diff --git a/winlogbeat/sys/wineventlog/metadata_store_test.go b/winlogbeat/sys/wineventlog/metadata_store_test.go index 0c89251bb7a..205f703105d 100644 --- a/winlogbeat/sys/wineventlog/metadata_store_test.go +++ b/winlogbeat/sys/wineventlog/metadata_store_test.go @@ -30,7 +30,7 @@ import ( func TestPublisherMetadataStore(t *testing.T) { logp.TestingSetup() - s, err := newPublisherMetadataStore( + s, err := NewPublisherMetadataStore( NilHandle, "Microsoft-Windows-Security-Auditing", logp.NewLogger("metadata")) diff --git a/winlogbeat/sys/wineventlog/renderer.go b/winlogbeat/sys/wineventlog/renderer.go index d52f4399fa2..40392abd15d 100644 --- a/winlogbeat/sys/wineventlog/renderer.go +++ b/winlogbeat/sys/wineventlog/renderer.go @@ -48,7 +48,7 @@ const ( // Renderer is used for converting event log handles into complete events. type Renderer struct { // Cache of publisher metadata. Maps publisher names to stored metadata. - metadataCache map[string]*publisherMetadataStore + metadataCache map[string]*PublisherMetadataStore // Mutex to guard the metadataCache. The other members are immutable. mutex sync.RWMutex @@ -71,7 +71,7 @@ func NewRenderer(session EvtHandle, log *logp.Logger) (*Renderer, error) { } return &Renderer{ - metadataCache: map[string]*publisherMetadataStore{}, + metadataCache: map[string]*PublisherMetadataStore{}, session: session, systemContext: systemContext, userContext: userContext, @@ -112,7 +112,7 @@ func (r *Renderer) Render(handle EvtHandle) (*winevent.Event, error) { } // Associate raw system properties to names (e.g. level=2 to Error). - enrichRawValuesWithNames(md, event) + winevent.EnrichRawValuesWithNames(&md.WinMeta, event) eventData, fingerprint, err := r.renderUser(handle, event) if err != nil { @@ -135,9 +135,9 @@ func (r *Renderer) Render(handle EvtHandle) (*winevent.Event, error) { return event, nil } -// getPublisherMetadata return a publisherMetadataStore for the provider. It +// getPublisherMetadata return a PublisherMetadataStore for the provider. It // never returns nil, but may return an error if it couldn't open a publisher. -func (r *Renderer) getPublisherMetadata(publisher string) (*publisherMetadataStore, error) { +func (r *Renderer) getPublisherMetadata(publisher string) (*PublisherMetadataStore, error) { var err error // NOTE: This code uses double-check locking to elevate to a write-lock @@ -159,11 +159,11 @@ func (r *Renderer) getPublisherMetadata(publisher string) (*publisherMetadataSto } // Load metadata from the publisher. - md, err = newPublisherMetadataStore(r.session, publisher, r.log) + md, err = NewPublisherMetadataStore(r.session, publisher, r.log) if err != nil { // Return an empty store on error (can happen in cases where the // log was forwarded and the provider doesn't exist on collector). - md = newEmptyPublisherMetadataStore(publisher, r.log) + md = NewEmptyPublisherMetadataStore(publisher, r.log) err = errors.Wrapf(err, "failed to load publisher metadata for %v "+ "(returning an empty metadata store)", publisher) } @@ -307,7 +307,7 @@ func (r *Renderer) render(context EvtHandle, eventHandle EvtHandle) (*sys.Pooled } // addEventData adds the event/user data values to the event. -func (r *Renderer) addEventData(evtMeta *eventMetadata, values []interface{}, event *winevent.Event) { +func (r *Renderer) addEventData(evtMeta *EventMetadata, values []interface{}, event *winevent.Event) { if len(values) == 0 { return } @@ -361,8 +361,8 @@ func (r *Renderer) addEventData(evtMeta *eventMetadata, values []interface{}, ev } // formatMessage adds the message to the event. -func (r *Renderer) formatMessage(publisherMeta *publisherMetadataStore, - eventMeta *eventMetadata, eventHandle EvtHandle, values []interface{}, +func (r *Renderer) formatMessage(publisherMeta *PublisherMetadataStore, + eventMeta *EventMetadata, eventHandle EvtHandle, values []interface{}, eventID uint16) (string, error) { if eventMeta != nil { @@ -394,45 +394,3 @@ func (r *Renderer) formatMessageFromTemplate(msgTmpl *template.Template, values return string(bb.Bytes()), nil } - -// enrichRawValuesWithNames adds the names associated with the raw system -// property values. It enriches the event with keywords, opcode, level, and -// task. The search order is defined in the EvtFormatMessage documentation. -func enrichRawValuesWithNames(publisherMeta *publisherMetadataStore, event *winevent.Event) { - // Keywords. Each bit in the value can represent a keyword. - rawKeyword := int64(event.KeywordsRaw) - isClassic := keywordClassic&rawKeyword > 0 - for mask, keyword := range winMeta.Keywords { - if rawKeyword&mask > 0 { - event.Keywords = append(event.Keywords, keyword) - rawKeyword -= mask - } - } - for mask, keyword := range publisherMeta.Keywords { - if rawKeyword&mask > 0 { - event.Keywords = append(event.Keywords, keyword) - rawKeyword -= mask - } - } - - // Opcode (search in winmeta first). - var found bool - if !isClassic { - event.Opcode, found = winMeta.Opcodes[event.OpcodeRaw] - if !found { - event.Opcode = publisherMeta.Opcodes[event.OpcodeRaw] - } - } - - // Level (search in winmeta first). - event.Level, found = winMeta.Levels[event.LevelRaw] - if !found { - event.Level = publisherMeta.Levels[event.LevelRaw] - } - - // Task (fall-back to winmeta if not found). - event.Task, found = publisherMeta.Tasks[event.TaskRaw] - if !found { - event.Task = winMeta.Tasks[event.TaskRaw] - } -} From 9987efe27b2d8eb756af2cd90ee99f34b5da9595 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 15 Apr 2021 11:34:14 +0200 Subject: [PATCH 2/9] Enrich Raw fields when possible in decode_xml --- libbeat/processors/decode_xml/schema.go | 11 --- .../decode_xml/wineventlog_schema.go | 36 ++++++++ .../decode_xml/wineventlog_schema_windows.go | 83 +++++++++++++++++++ 3 files changed, 119 insertions(+), 11 deletions(-) create mode 100644 libbeat/processors/decode_xml/wineventlog_schema.go create mode 100644 libbeat/processors/decode_xml/wineventlog_schema_windows.go diff --git a/libbeat/processors/decode_xml/schema.go b/libbeat/processors/decode_xml/schema.go index 10043ec3d0b..2681142fdb9 100644 --- a/libbeat/processors/decode_xml/schema.go +++ b/libbeat/processors/decode_xml/schema.go @@ -24,7 +24,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/encoding/xml" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/winlogbeat/sys/winevent" ) const wineventlogSchema = "wineventlog" @@ -83,13 +82,3 @@ func newSchemaLessDecoder(cfg decodeXMLConfig) decoder { return common.MapStr(out), nil } } - -func newWineventlogDecoder(decodeXMLConfig) decoder { - return func(p []byte) (common.MapStr, error) { - evt, err := winevent.UnmarshalXML(p) - if err != nil { - return nil, err - } - return evt.Fields(), nil - } -} diff --git a/libbeat/processors/decode_xml/wineventlog_schema.go b/libbeat/processors/decode_xml/wineventlog_schema.go new file mode 100644 index 00000000000..64e42e043d1 --- /dev/null +++ b/libbeat/processors/decode_xml/wineventlog_schema.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !windows + +package decode_xml + +import ( + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/winlogbeat/sys/winevent" +) + +func newWineventlogDecoder(decodeXMLConfig) decoder { + return func(p []byte) (common.MapStr, error) { + evt, err := winevent.UnmarshalXML(p) + if err != nil { + return nil, err + } + winevent.EnrichRawValuesWithNames(nil, &evt) + return evt.Fields(), nil + } +} diff --git a/libbeat/processors/decode_xml/wineventlog_schema_windows.go b/libbeat/processors/decode_xml/wineventlog_schema_windows.go new file mode 100644 index 00000000000..d42250b8de8 --- /dev/null +++ b/libbeat/processors/decode_xml/wineventlog_schema_windows.go @@ -0,0 +1,83 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package decode_xml + +import ( + "sync" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/winlogbeat/sys/winevent" + "github.com/elastic/beats/v7/winlogbeat/sys/wineventlog" +) + +func newWineventlogDecoder(decodeXMLConfig) decoder { + cache := &metadataCache{ + store: map[string]*wineventlog.PublisherMetadataStore{}, + } + return func(p []byte) (common.MapStr, error) { + evt, err := winevent.UnmarshalXML(p) + if err != nil { + return nil, err + } + md := cache.getPublisherMetadata(evt.Provider.Name) + winevent.EnrichRawValuesWithNames(md, &evt) + return evt.Fields(), nil + } +} + +type metadataCache struct { + store map[string]*winevent.WinMeta + mutex sync.RWMutex +} + +func (c *metadataCache) getPublisherMetadata(publisher string) *winevent.WinMeta { + // NOTE: This code uses double-check locking to elevate to a write-lock + // when a cache value needs initialized. + c.mutex.RLock() + + // Lookup cached value. + md, found := c.store[publisher] + if !found { + // Elevate to write lock. + c.mutex.RUnlock() + c.mutex.Lock() + defer c.mutex.Unlock() + + // Double-check if the condition changed while upgrading the lock. + md, found = c.store[publisher] + if found { + return md + } + + // Load metadata from the publisher. + md, err := wineventlog.NewPublisherMetadataStore(wineventlog.NilHandle, publisher, r.log) + if err != nil { + // Return an empty store on error (can happen in cases where the + // log was forwarded and the provider doesn't exist on collector). + md = wineventlog.NewEmptyPublisherMetadataStore(publisher, logp.L().Named(logName)) + } + c.store[publisher] = &md.WinMeta + } else { + c.mutex.RUnlock() + } + + return md +} From 16155f31f97f2661110a724a7193903e164e8cb5 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 15 Apr 2021 12:35:15 +0200 Subject: [PATCH 3/9] Add ECS mappings when decoding wineventlog xml --- libbeat/processors/decode_xml/decode_xml.go | 6 ++- .../decode_xml/docs/decode_xml.asciidoc | 1 + libbeat/processors/decode_xml/schema.go | 11 +++-- libbeat/processors/decode_xml/schema_test.go | 31 +++++++----- ...entlog_schema.go => schema_wineventlog.go} | 7 +-- .../decode_xml/schema_wineventlog_fields.go | 49 +++++++++++++++++++ ...ndows.go => schema_wineventlog_windows.go} | 14 +++--- winlogbeat/sys/winevent/maputil.go | 5 ++ 8 files changed, 97 insertions(+), 27 deletions(-) rename libbeat/processors/decode_xml/{wineventlog_schema.go => schema_wineventlog.go} (87%) create mode 100644 libbeat/processors/decode_xml/schema_wineventlog_fields.go rename libbeat/processors/decode_xml/{wineventlog_schema_windows.go => schema_wineventlog_windows.go} (90%) diff --git a/libbeat/processors/decode_xml/decode_xml.go b/libbeat/processors/decode_xml/decode_xml.go index 5d841a2b576..1bce1a5eb00 100644 --- a/libbeat/processors/decode_xml/decode_xml.go +++ b/libbeat/processors/decode_xml/decode_xml.go @@ -111,7 +111,7 @@ func (x *decodeXML) run(event *beat.Event) error { return errFieldIsNotString } - xmlOutput, err := x.decode([]byte(text)) + xmlOutput, ecsFields, err := x.decode([]byte(text)) if err != nil { return fmt.Errorf("error decoding XML field: %w", err) } @@ -132,6 +132,10 @@ func (x *decodeXML) run(event *beat.Event) error { jsontransform.WriteJSONKeys(event, xmlOutput, false, x.OverwriteKeys, !x.IgnoreFailure) } + if len(ecsFields) > 0 { + jsontransform.WriteJSONKeys(event, ecsFields, false, x.OverwriteKeys, !x.IgnoreFailure) + } + if id != "" { event.SetID(id) } diff --git a/libbeat/processors/decode_xml/docs/decode_xml.asciidoc b/libbeat/processors/decode_xml/docs/decode_xml.asciidoc index c21c25081fc..8461f671321 100644 --- a/libbeat/processors/decode_xml/docs/decode_xml.asciidoc +++ b/libbeat/processors/decode_xml/docs/decode_xml.asciidoc @@ -129,6 +129,7 @@ The decoder will always output the fields formatted in the same way, the `to_lower` option will be ignored when using this schema decoder. The output fields will be the same as the {winlogbeat-ref}/exported-fields-winlog.html#_winlog[winlogbeat winlog fields]. +Additionally, ECS mappings are going to be added when possible. Example: diff --git a/libbeat/processors/decode_xml/schema.go b/libbeat/processors/decode_xml/schema.go index 2681142fdb9..b8c7ac1349b 100644 --- a/libbeat/processors/decode_xml/schema.go +++ b/libbeat/processors/decode_xml/schema.go @@ -29,7 +29,10 @@ import ( const wineventlogSchema = "wineventlog" type newDecoderFunc func(cfg decodeXMLConfig) decoder -type decoder func(p []byte) (common.MapStr, error) + +// a decoder will decode the raw XML into a set of fields, +// optionally will return additional ECS mappings when possible +type decoder func(p []byte) (decodedXML, ecsFields common.MapStr, err error) var ( registeredDecoders = map[string]newDecoderFunc{} @@ -68,7 +71,7 @@ func registerDecoders() { } func newSchemaLessDecoder(cfg decodeXMLConfig) decoder { - return func(p []byte) (common.MapStr, error) { + return func(p []byte) (common.MapStr, common.MapStr, error) { dec := xml.NewDecoder(bytes.NewReader(p)) if cfg.ToLower { dec.LowercaseKeys() @@ -76,9 +79,9 @@ func newSchemaLessDecoder(cfg decodeXMLConfig) decoder { out, err := dec.Decode() if err != nil { - return nil, err + return nil, nil, err } - return common.MapStr(out), nil + return common.MapStr(out), nil, nil } } diff --git a/libbeat/processors/decode_xml/schema_test.go b/libbeat/processors/decode_xml/schema_test.go index 655c5156fe2..aca09aebe16 100644 --- a/libbeat/processors/decode_xml/schema_test.go +++ b/libbeat/processors/decode_xml/schema_test.go @@ -40,9 +40,10 @@ func TestDecodeSchemas(t *testing.T) { { schema: wineventlogSchema, config: decodeXMLConfig{ - Field: "message", - Target: &testXMLTargetField, - Schema: wineventlogSchema, + Field: "message", + Target: &testXMLTargetField, + Schema: wineventlogSchema, + OverwriteKeys: true, }, Input: common.MapStr{ "message": "" + @@ -57,6 +58,19 @@ func TestDecodeSchemas(t *testing.T) { "Special LogonInfoSecurityMicrosoft Windows security auditing.Audit Success", }, Output: common.MapStr{ + "event": common.MapStr{ + "action": "Special Logon", + "code": "4672", + "kind": "event", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + }, + "host": common.MapStr{ + "name": "vagrant", + }, + "log": common.MapStr{ + "level": "information", + }, "xml": common.MapStr{ "channel": "Security", "outcome": "success", @@ -91,16 +105,7 @@ func TestDecodeSchemas(t *testing.T) { }, }, }, - "message": "" + - "4672001254800x8020000000000000" + - "11303Securityvagrant" + - "S-1-5-18SYSTEMNT AUTHORITY0x3e7" + - "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\t" + - "SeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege" + - "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\n" + - "Privileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\t" + - "SeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformation" + - "Special LogonInfoSecurityMicrosoft Windows security auditing.Audit Success", + "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", }, }, } diff --git a/libbeat/processors/decode_xml/wineventlog_schema.go b/libbeat/processors/decode_xml/schema_wineventlog.go similarity index 87% rename from libbeat/processors/decode_xml/wineventlog_schema.go rename to libbeat/processors/decode_xml/schema_wineventlog.go index 64e42e043d1..86530f165b4 100644 --- a/libbeat/processors/decode_xml/wineventlog_schema.go +++ b/libbeat/processors/decode_xml/schema_wineventlog.go @@ -25,12 +25,13 @@ import ( ) func newWineventlogDecoder(decodeXMLConfig) decoder { - return func(p []byte) (common.MapStr, error) { + return func(p []byte) (common.MapStr, common.MapStr, error) { evt, err := winevent.UnmarshalXML(p) if err != nil { - return nil, err + return nil, nil, err } winevent.EnrichRawValuesWithNames(nil, &evt) - return evt.Fields(), nil + fields, ecs := wineventlogFields(evt) + return fields, ecs, nil } } diff --git a/libbeat/processors/decode_xml/schema_wineventlog_fields.go b/libbeat/processors/decode_xml/schema_wineventlog_fields.go new file mode 100644 index 00000000000..c2ba9bea9d1 --- /dev/null +++ b/libbeat/processors/decode_xml/schema_wineventlog_fields.go @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package decode_xml + +import ( + "fmt" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/winlogbeat/sys/winevent" +) + +func wineventlogFields(evt winevent.Event) (common.MapStr, common.MapStr) { + win := evt.Fields() + + ecs := common.MapStr{} + + ecs.Put("event.kind", "event") + ecs.Put("event.code", fmt.Sprint(evt.EventIdentifier.ID)) + ecs.Put("event.provider", evt.Provider.Name) + winevent.AddOptional(ecs, "event.action", evt.Task) + winevent.AddOptional(ecs, "host.name", evt.Computer) + winevent.AddOptional(ecs, "event.outcome", getValue(win, "outcome")) + winevent.AddOptional(ecs, "log.level", getValue(win, "level")) + winevent.AddOptional(ecs, "message", getValue(win, "message")) + winevent.AddOptional(ecs, "error.code", getValue(win, "error.code")) + winevent.AddOptional(ecs, "error.message", getValue(win, "error.message")) + + return win, ecs +} + +func getValue(m common.MapStr, key string) interface{} { + v, _ := m.GetValue(key) + return v +} diff --git a/libbeat/processors/decode_xml/wineventlog_schema_windows.go b/libbeat/processors/decode_xml/schema_wineventlog_windows.go similarity index 90% rename from libbeat/processors/decode_xml/wineventlog_schema_windows.go rename to libbeat/processors/decode_xml/schema_wineventlog_windows.go index d42250b8de8..37f5a89daa2 100644 --- a/libbeat/processors/decode_xml/wineventlog_schema_windows.go +++ b/libbeat/processors/decode_xml/schema_wineventlog_windows.go @@ -30,16 +30,17 @@ import ( func newWineventlogDecoder(decodeXMLConfig) decoder { cache := &metadataCache{ - store: map[string]*wineventlog.PublisherMetadataStore{}, + store: map[string]*winevent.WinMeta{}, } - return func(p []byte) (common.MapStr, error) { + return func(p []byte) (common.MapStr, common.MapStr, error) { evt, err := winevent.UnmarshalXML(p) if err != nil { - return nil, err + return nil, nil, err } md := cache.getPublisherMetadata(evt.Provider.Name) winevent.EnrichRawValuesWithNames(md, &evt) - return evt.Fields(), nil + fields, ecs := wineventlogFields(evt) + return fields, ecs, nil } } @@ -53,6 +54,7 @@ func (c *metadataCache) getPublisherMetadata(publisher string) *winevent.WinMeta // when a cache value needs initialized. c.mutex.RLock() + log := logp.L().Named(logName) // Lookup cached value. md, found := c.store[publisher] if !found { @@ -68,11 +70,11 @@ func (c *metadataCache) getPublisherMetadata(publisher string) *winevent.WinMeta } // Load metadata from the publisher. - md, err := wineventlog.NewPublisherMetadataStore(wineventlog.NilHandle, publisher, r.log) + md, err := wineventlog.NewPublisherMetadataStore(wineventlog.NilHandle, publisher, log) if err != nil { // Return an empty store on error (can happen in cases where the // log was forwarded and the provider doesn't exist on collector). - md = wineventlog.NewEmptyPublisherMetadataStore(publisher, logp.L().Named(logName)) + md = wineventlog.NewEmptyPublisherMetadataStore(publisher, log) } c.store[publisher] = &md.WinMeta } else { diff --git a/winlogbeat/sys/winevent/maputil.go b/winlogbeat/sys/winevent/maputil.go index 41fe694c88e..45a265ae8c6 100644 --- a/winlogbeat/sys/winevent/maputil.go +++ b/winlogbeat/sys/winevent/maputil.go @@ -80,6 +80,10 @@ func AddPairs(m common.MapStr, key string, pairs []KeyValue) common.MapStr { // isZero return true if the given value is the zero value for its type. func isZero(i interface{}) bool { + if i == nil { + return true + } + v := reflect.ValueOf(i) switch v.Kind() { case reflect.Array, reflect.String: @@ -95,5 +99,6 @@ func isZero(i interface{}) bool { case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: return v.IsNil() } + return false } From 0c2fca82064928947f51c9c9d882df99572938aa Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 15 Apr 2021 18:04:52 +0200 Subject: [PATCH 4/9] Add decode_xml_wineventlog processor --- CHANGELOG.next.asciidoc | 2 +- libbeat/cmd/instance/imports_common.go | 1 + libbeat/docs/processors-list.asciidoc | 14 +- libbeat/processors/decode_xml/config.go | 1 - libbeat/processors/decode_xml/decode_xml.go | 30 ++- .../decode_xml/docs/decode_xml.asciidoc | 83 +------- libbeat/processors/decode_xml/schema.go | 87 -------- libbeat/processors/decode_xml/schema_test.go | 135 ------------ .../decode_xml/schema_wineventlog_fields.go | 49 ----- .../decode_xml_wineventlog/config.go | 37 ++++ .../decoder.go} | 24 ++- .../decoder_windows.go} | 41 ++-- .../docs/decode_xml_wineventlog.asciidoc | 114 ++++++++++ .../decode_xml_wineventlog/processor.go | 171 +++++++++++++++ .../decode_xml_wineventlog/processor_test.go | 197 ++++++++++++++++++ 15 files changed, 591 insertions(+), 395 deletions(-) delete mode 100644 libbeat/processors/decode_xml/schema.go delete mode 100644 libbeat/processors/decode_xml/schema_test.go delete mode 100644 libbeat/processors/decode_xml/schema_wineventlog_fields.go create mode 100644 libbeat/processors/decode_xml_wineventlog/config.go rename libbeat/processors/{decode_xml/schema_wineventlog.go => decode_xml_wineventlog/decoder.go} (71%) rename libbeat/processors/{decode_xml/schema_wineventlog_windows.go => decode_xml_wineventlog/decoder_windows.go} (74%) create mode 100644 libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc create mode 100644 libbeat/processors/decode_xml_wineventlog/processor.go create mode 100644 libbeat/processors/decode_xml_wineventlog/processor_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 44bfb61eb6e..0a728896087 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -612,11 +612,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added "add_network_direction" processor for determining perimeter-based network direction. {pull}23076[23076] - Added new `rate_limit` processor for enforcing rate limits on event throughput. {pull}22883[22883] - Allow node/namespace metadata to be disabled on kubernetes metagen and ensure add_kubernetes_metadata honors host {pull}23012[23012] -- Add `wineventlog` schema to `decode_xml` processor. {issue}23910[23910] {pull}24726[24726] - Add new ECS 1.9 field `cloud.service.name` to `add_cloud_metadata` processor. {pull}24993[24993] - Libbeat: report queue capacity, output batch size, and output client count to monitoring. {pull}24700[24700] - Add kubernetes.pod.ip field in kubernetes metadata. {pull}25037[25037] - Discover changes in Kubernetes namespace metadata as soon as they happen. {pull}25117[25117] +- Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25109[25109] *Auditbeat* diff --git a/libbeat/cmd/instance/imports_common.go b/libbeat/cmd/instance/imports_common.go index ac767a55964..724919364db 100644 --- a/libbeat/cmd/instance/imports_common.go +++ b/libbeat/cmd/instance/imports_common.go @@ -31,6 +31,7 @@ import ( _ "github.com/elastic/beats/v7/libbeat/processors/communityid" _ "github.com/elastic/beats/v7/libbeat/processors/convert" _ "github.com/elastic/beats/v7/libbeat/processors/decode_xml" + _ "github.com/elastic/beats/v7/libbeat/processors/decode_xml_wineventlog" _ "github.com/elastic/beats/v7/libbeat/processors/dissect" _ "github.com/elastic/beats/v7/libbeat/processors/dns" _ "github.com/elastic/beats/v7/libbeat/processors/extract_array" diff --git a/libbeat/docs/processors-list.asciidoc b/libbeat/docs/processors-list.asciidoc index 3900366708f..b063965ed38 100644 --- a/libbeat/docs/processors-list.asciidoc +++ b/libbeat/docs/processors-list.asciidoc @@ -63,7 +63,10 @@ ifndef::no_decode_json_fields_processor[] * <> endif::[] ifndef::no_decode_xml_processor[] -* <> +* <> +endif::[] +ifndef::no_decode_xml_wineventlog_processor[] +* <> endif::[] ifndef::no_decompress_gzip_field_processor[] * <> @@ -183,6 +186,12 @@ endif::[] ifndef::no_decode_json_fields_processor[] include::{libbeat-processors-dir}/actions/docs/decode_json_fields.asciidoc[] endif::[] +ifndef::no_decode_xml_processor[] +include::{libbeat-processors-dir}/decode_xml/docs/decode_xml.asciidoc[] +endif::[] +ifndef::no_decode_xml_wineventlog_processor[] +include::{libbeat-processors-dir}/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc[] +endif::[] ifndef::no_decompress_gzip_field_processor[] include::{libbeat-processors-dir}/actions/docs/decompress_gzip_field.asciidoc[] endif::[] @@ -234,8 +243,5 @@ endif::[] ifndef::no_urldecode_processor[] include::{libbeat-processors-dir}/urldecode/docs/urldecode.asciidoc[] endif::[] -ifndef::no_decode_xml_processor[] -include::{libbeat-processors-dir}/decode_xml/docs/decode_xml.asciidoc[] -endif::[] //# end::processors-include[] diff --git a/libbeat/processors/decode_xml/config.go b/libbeat/processors/decode_xml/config.go index 21bb426c5b2..289b2eaa0e9 100644 --- a/libbeat/processors/decode_xml/config.go +++ b/libbeat/processors/decode_xml/config.go @@ -25,7 +25,6 @@ type decodeXMLConfig struct { ToLower bool `config:"to_lower"` IgnoreMissing bool `config:"ignore_missing"` IgnoreFailure bool `config:"ignore_failure"` - Schema string `config:"schema"` } func defaultConfig() decodeXMLConfig { diff --git a/libbeat/processors/decode_xml/decode_xml.go b/libbeat/processors/decode_xml/decode_xml.go index 1bce1a5eb00..ced399c00fa 100644 --- a/libbeat/processors/decode_xml/decode_xml.go +++ b/libbeat/processors/decode_xml/decode_xml.go @@ -18,6 +18,7 @@ package decode_xml import ( + "bytes" "encoding/json" "errors" "fmt" @@ -25,6 +26,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/common/encoding/xml" "github.com/elastic/beats/v7/libbeat/common/jsontransform" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/processors" @@ -35,8 +37,7 @@ import ( type decodeXML struct { decodeXMLConfig - decode decoder - log *logp.Logger + log *logp.Logger } var ( @@ -56,10 +57,9 @@ func init() { "field", "target_field", "overwrite_keys", "document_id", "to_lower", "ignore_missing", - "ignore_failure", "schema", + "ignore_failure", ))) jsprocessor.RegisterPlugin(procName, New) - registerDecoders() } // New constructs a new decode_xml processor. @@ -83,7 +83,6 @@ func newDecodeXML(config decodeXMLConfig) (processors.Processor, error) { return &decodeXML{ decodeXMLConfig: config, - decode: newDecoder(config), log: logp.NewLogger(logName), }, nil } @@ -111,7 +110,7 @@ func (x *decodeXML) run(event *beat.Event) error { return errFieldIsNotString } - xmlOutput, ecsFields, err := x.decode([]byte(text)) + xmlOutput, err := x.decode([]byte(text)) if err != nil { return fmt.Errorf("error decoding XML field: %w", err) } @@ -132,16 +131,27 @@ func (x *decodeXML) run(event *beat.Event) error { jsontransform.WriteJSONKeys(event, xmlOutput, false, x.OverwriteKeys, !x.IgnoreFailure) } - if len(ecsFields) > 0 { - jsontransform.WriteJSONKeys(event, ecsFields, false, x.OverwriteKeys, !x.IgnoreFailure) - } - if id != "" { event.SetID(id) } + return nil } +func (x *decodeXML) decode(p []byte) (common.MapStr, error) { + dec := xml.NewDecoder(bytes.NewReader(p)) + if x.ToLower { + dec.LowercaseKeys() + } + + out, err := dec.Decode() + if err != nil { + return nil, err + } + + return common.MapStr(out), nil +} + func (x *decodeXML) String() string { json, _ := json.Marshal(x.decodeXMLConfig) return procName + "=" + string(json) diff --git a/libbeat/processors/decode_xml/docs/decode_xml.asciidoc b/libbeat/processors/decode_xml/docs/decode_xml.asciidoc index 8461f671321..67d1c49111f 100644 --- a/libbeat/processors/decode_xml/docs/decode_xml.asciidoc +++ b/libbeat/processors/decode_xml/docs/decode_xml.asciidoc @@ -1,4 +1,4 @@ -[[decode_xml]] +[[decode-xml]] === Decode XML ++++ @@ -100,9 +100,6 @@ default value is `true`. `to_lower`:: (Optional) Converts all keys to lowercase. Accepts either true or false. The default value is `true`. -`schema`:: (Optional) Specifies the schema of the message. Accepted schemas: `wineventlog`. -If no schema is specified it defaults to using the regular XML to JSON conversion. - `document_id`:: (Optional) XML key to use as the document ID. If configured, the field will be removed from the original XML document and stored in `@metadata._id`. @@ -114,81 +111,3 @@ when a specified field does not exist. Defaults to `false`. Defaults to `false`. See <> for a list of supported conditions. - - -==== Schemas - -When a schema is defined, the specific decoder will parse the configured field. -The ouput of the parsing will be specific to that schema. - -===== Wineventlog - -The `wineventlog` schema decodes Windows Events. - -The decoder will always output the fields formatted in the same way, the -`to_lower` option will be ignored when using this schema decoder. -The output fields will be the same as the -{winlogbeat-ref}/exported-fields-winlog.html#_winlog[winlogbeat winlog fields]. -Additionally, ECS mappings are going to be added when possible. - -Example: - -[source,yaml] -------------------------------------------------------------------------------- -processors: - - decode_xml: - field: event.original - target_field: winlog - to_lower: false -------------------------------------------------------------------------------- - -[source,json] -------------------------------------------------------------------------------- -{ - "event": { - "original": "4672001254800x802000000000000011303SecurityvagrantS-1-5-18SYSTEMNT AUTHORITY0x3e7SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeSpecial privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformationSpecial LogonInfoSecurityMicrosoft Windows security auditing.Audit Success" - } -} -------------------------------------------------------------------------------- - -Will produce the following output: - -[source,json] -------------------------------------------------------------------------------- -{ - "event": { - "original": "4672001254800x802000000000000011303SecurityvagrantS-1-5-18SYSTEMNT AUTHORITY0x3e7SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeSpecial privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformationSpecial LogonInfoSecurityMicrosoft Windows security auditing.Audit Success" - }, - "winlog": { - "channel": "Security", - "outcome": "success", - "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}", - "level": "information", - "event_id": 4672, - "provider_name": "Microsoft-Windows-Security-Auditing", - "record_id": 11303, - "computer_name": "vagrant", - "keywords_raw": 9232379236109516800, - "opcode": "Info", - "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", - "event_data": { - "SubjectUserSid": "S-1-5-18", - "SubjectUserName": "SYSTEM", - "SubjectDomainName": "NT AUTHORITY", - "SubjectLogonId": "0x3e7", - "PrivilegeList": "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege" - }, - "task": "Special Logon", - "keywords": [ - "Audit Success" - ], - "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", - "process": { - "pid": 652, - "thread": { - "id": 4660 - } - } - } -} -------------------------------------------------------------------------------- diff --git a/libbeat/processors/decode_xml/schema.go b/libbeat/processors/decode_xml/schema.go deleted file mode 100644 index b8c7ac1349b..00000000000 --- a/libbeat/processors/decode_xml/schema.go +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package decode_xml - -import ( - "bytes" - "errors" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/encoding/xml" - "github.com/elastic/beats/v7/libbeat/logp" -) - -const wineventlogSchema = "wineventlog" - -type newDecoderFunc func(cfg decodeXMLConfig) decoder - -// a decoder will decode the raw XML into a set of fields, -// optionally will return additional ECS mappings when possible -type decoder func(p []byte) (decodedXML, ecsFields common.MapStr, err error) - -var ( - registeredDecoders = map[string]newDecoderFunc{} - newDefaultDecoder newDecoderFunc = newSchemaLessDecoder -) - -func registerDecoder(schema string, dec newDecoderFunc) error { - if schema == "" { - return errors.New("schema can't be empty") - } - - if dec == nil { - return errors.New("decoder can't be nil") - } - - if _, found := registeredDecoders[schema]; found { - return errors.New("already registered") - } - - registeredDecoders[schema] = dec - - return nil -} - -func newDecoder(cfg decodeXMLConfig) decoder { - newDec, found := registeredDecoders[cfg.Schema] - if !found { - return newDefaultDecoder(cfg) - } - return newDec(cfg) -} - -func registerDecoders() { - log := logp.L().Named(logName) - log.Debug(registerDecoder(wineventlogSchema, newWineventlogDecoder)) -} - -func newSchemaLessDecoder(cfg decodeXMLConfig) decoder { - return func(p []byte) (common.MapStr, common.MapStr, error) { - dec := xml.NewDecoder(bytes.NewReader(p)) - if cfg.ToLower { - dec.LowercaseKeys() - } - - out, err := dec.Decode() - if err != nil { - return nil, nil, err - } - - return common.MapStr(out), nil, nil - } -} diff --git a/libbeat/processors/decode_xml/schema_test.go b/libbeat/processors/decode_xml/schema_test.go deleted file mode 100644 index aca09aebe16..00000000000 --- a/libbeat/processors/decode_xml/schema_test.go +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package decode_xml - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" -) - -func TestDecodeSchemas(t *testing.T) { - var testCases = []struct { - schema string - config decodeXMLConfig - Input common.MapStr - Output common.MapStr - error bool - errorMessage string - }{ - { - schema: wineventlogSchema, - config: decodeXMLConfig{ - Field: "message", - Target: &testXMLTargetField, - Schema: wineventlogSchema, - OverwriteKeys: true, - }, - Input: common.MapStr{ - "message": "" + - "4672001254800x8020000000000000" + - "11303Securityvagrant" + - "S-1-5-18SYSTEMNT AUTHORITY0x3e7" + - "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\t" + - "SeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege" + - "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\n" + - "Privileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\t" + - "SeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformation" + - "Special LogonInfoSecurityMicrosoft Windows security auditing.Audit Success", - }, - Output: common.MapStr{ - "event": common.MapStr{ - "action": "Special Logon", - "code": "4672", - "kind": "event", - "outcome": "success", - "provider": "Microsoft-Windows-Security-Auditing", - }, - "host": common.MapStr{ - "name": "vagrant", - }, - "log": common.MapStr{ - "level": "information", - }, - "xml": common.MapStr{ - "channel": "Security", - "outcome": "success", - "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}", - "level": "information", - "event_id": uint32(4672), - "provider_name": "Microsoft-Windows-Security-Auditing", - "record_id": uint64(11303), - "computer_name": "vagrant", - "time_created": func() time.Time { - t, _ := time.Parse(time.RFC3339Nano, "2021-03-23T09:56:13.137310000Z") - return t - }(), - "opcode": "Info", - "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", - "event_data": common.MapStr{ - "SubjectUserSid": "S-1-5-18", - "SubjectUserName": "SYSTEM", - "SubjectDomainName": "NT AUTHORITY", - "SubjectLogonId": "0x3e7", - "PrivilegeList": "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", - }, - "task": "Special Logon", - "keywords": []string{ - "Audit Success", - }, - "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", - "process": common.MapStr{ - "pid": uint32(652), - "thread": common.MapStr{ - "id": uint32(4660), - }, - }, - }, - "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", - }, - }, - } - - for _, test := range testCases { - test := test - t.Run(test.schema, func(t *testing.T) { - t.Parallel() - - f, err := newDecodeXML(test.config) - require.NoError(t, err) - - event := &beat.Event{ - Fields: test.Input, - } - newEvent, err := f.Run(event) - if !test.error { - assert.NoError(t, err) - } else { - if assert.Error(t, err) { - assert.Contains(t, err.Error(), test.errorMessage) - } - } - assert.Equal(t, test.Output, newEvent.Fields) - }) - } -} diff --git a/libbeat/processors/decode_xml/schema_wineventlog_fields.go b/libbeat/processors/decode_xml/schema_wineventlog_fields.go deleted file mode 100644 index c2ba9bea9d1..00000000000 --- a/libbeat/processors/decode_xml/schema_wineventlog_fields.go +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package decode_xml - -import ( - "fmt" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/winlogbeat/sys/winevent" -) - -func wineventlogFields(evt winevent.Event) (common.MapStr, common.MapStr) { - win := evt.Fields() - - ecs := common.MapStr{} - - ecs.Put("event.kind", "event") - ecs.Put("event.code", fmt.Sprint(evt.EventIdentifier.ID)) - ecs.Put("event.provider", evt.Provider.Name) - winevent.AddOptional(ecs, "event.action", evt.Task) - winevent.AddOptional(ecs, "host.name", evt.Computer) - winevent.AddOptional(ecs, "event.outcome", getValue(win, "outcome")) - winevent.AddOptional(ecs, "log.level", getValue(win, "level")) - winevent.AddOptional(ecs, "message", getValue(win, "message")) - winevent.AddOptional(ecs, "error.code", getValue(win, "error.code")) - winevent.AddOptional(ecs, "error.message", getValue(win, "error.message")) - - return win, ecs -} - -func getValue(m common.MapStr, key string) interface{} { - v, _ := m.GetValue(key) - return v -} diff --git a/libbeat/processors/decode_xml_wineventlog/config.go b/libbeat/processors/decode_xml_wineventlog/config.go new file mode 100644 index 00000000000..ec543c489f5 --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/config.go @@ -0,0 +1,37 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package decode_xml_wineventlog + +type config struct { + Field string `config:"field" validate:"required"` + Target string `config:"target_field"` + OverwriteKeys bool `config:"overwrite_keys"` + MapECSFields bool `config:"map_ecs_fields"` + DocumentID string `config:"document_id"` + IgnoreMissing bool `config:"ignore_missing"` + IgnoreFailure bool `config:"ignore_failure"` +} + +func defaultConfig() config { + return config{ + Field: "message", + OverwriteKeys: true, + MapECSFields: true, + Target: "winlog", + } +} diff --git a/libbeat/processors/decode_xml/schema_wineventlog.go b/libbeat/processors/decode_xml_wineventlog/decoder.go similarity index 71% rename from libbeat/processors/decode_xml/schema_wineventlog.go rename to libbeat/processors/decode_xml_wineventlog/decoder.go index 86530f165b4..c1b7843b18d 100644 --- a/libbeat/processors/decode_xml/schema_wineventlog.go +++ b/libbeat/processors/decode_xml_wineventlog/decoder.go @@ -17,21 +17,25 @@ // +build !windows -package decode_xml +package decode_xml_wineventlog import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/winlogbeat/sys/winevent" ) -func newWineventlogDecoder(decodeXMLConfig) decoder { - return func(p []byte) (common.MapStr, common.MapStr, error) { - evt, err := winevent.UnmarshalXML(p) - if err != nil { - return nil, nil, err - } - winevent.EnrichRawValuesWithNames(nil, &evt) - fields, ecs := wineventlogFields(evt) - return fields, ecs, nil +type nonWinDecoder struct{} + +func newDecoder() decoder { + return nonWinDecoder{} +} + +func (nonWinDecoder) decode(data []byte) (common.MapStr, common.MapStr, error) { + evt, err := winevent.UnmarshalXML(data) + if err != nil { + return nil, nil, err } + winevent.EnrichRawValuesWithNames(nil, &evt) + win, ecs := fields(evt) + return win, ecs, nil } diff --git a/libbeat/processors/decode_xml/schema_wineventlog_windows.go b/libbeat/processors/decode_xml_wineventlog/decoder_windows.go similarity index 74% rename from libbeat/processors/decode_xml/schema_wineventlog_windows.go rename to libbeat/processors/decode_xml_wineventlog/decoder_windows.go index 37f5a89daa2..3b3f7bf3599 100644 --- a/libbeat/processors/decode_xml/schema_wineventlog_windows.go +++ b/libbeat/processors/decode_xml_wineventlog/decoder_windows.go @@ -17,7 +17,7 @@ // +build windows -package decode_xml +package decode_xml_wineventlog import ( "sync" @@ -28,25 +28,35 @@ import ( "github.com/elastic/beats/v7/winlogbeat/sys/wineventlog" ) -func newWineventlogDecoder(decodeXMLConfig) decoder { - cache := &metadataCache{ - store: map[string]*winevent.WinMeta{}, +type winDecoder struct { + cache *metadataCache +} + +func newDecoder() decoder { + return &winDecoder{ + cache: &metadataCache{ + store: map[string]*winevent.WinMeta{}, + log: logp.NewLogger(logName), + }, } - return func(p []byte) (common.MapStr, common.MapStr, error) { - evt, err := winevent.UnmarshalXML(p) - if err != nil { - return nil, nil, err - } - md := cache.getPublisherMetadata(evt.Provider.Name) - winevent.EnrichRawValuesWithNames(md, &evt) - fields, ecs := wineventlogFields(evt) - return fields, ecs, nil +} + +func (dec *winDecoder) decode(data []byte) (common.MapStr, common.MapStr, error) { + evt, err := winevent.UnmarshalXML(data) + if err != nil { + return nil, nil, err } + md := dec.cache.getPublisherMetadata(evt.Provider.Name) + winevent.EnrichRawValuesWithNames(md, &evt) + win, ecs := fields(evt) + return win, ecs, nil } type metadataCache struct { store map[string]*winevent.WinMeta mutex sync.RWMutex + + log *logp.Logger } func (c *metadataCache) getPublisherMetadata(publisher string) *winevent.WinMeta { @@ -54,7 +64,6 @@ func (c *metadataCache) getPublisherMetadata(publisher string) *winevent.WinMeta // when a cache value needs initialized. c.mutex.RLock() - log := logp.L().Named(logName) // Lookup cached value. md, found := c.store[publisher] if !found { @@ -70,11 +79,11 @@ func (c *metadataCache) getPublisherMetadata(publisher string) *winevent.WinMeta } // Load metadata from the publisher. - md, err := wineventlog.NewPublisherMetadataStore(wineventlog.NilHandle, publisher, log) + md, err := wineventlog.NewPublisherMetadataStore(wineventlog.NilHandle, publisher, c.log) if err != nil { // Return an empty store on error (can happen in cases where the // log was forwarded and the provider doesn't exist on collector). - md = wineventlog.NewEmptyPublisherMetadataStore(publisher, log) + md = wineventlog.NewEmptyPublisherMetadataStore(publisher, c.log) } c.store[publisher] = &md.WinMeta } else { diff --git a/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc b/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc new file mode 100644 index 00000000000..1a91e5b4cb4 --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc @@ -0,0 +1,114 @@ +[[decode-xml-wineventlog]] +=== Decode XML Wineventlog + +++++ +decode_xml_wineventlog +++++ + +experimental[] + +The `decode_xml_wineventlog` processor decodes Windows Event Log data in XML format that is stored under the `field` +key. It outputs the result into the `target_field`. + +The output fields will be the same as the +{winlogbeat-ref}/exported-fields-winlog.html#_winlog[winlogbeat winlog fields]. + +The supported configuration options are: + +`field`:: (Required) Source field containing the XML. Defaults to `message`. + +`target_field`:: (Required) The field under which the decoded XML will be +written. To merge the decoded XML fields into the root of the event specify +`target_field` with an empty string (`target_field: ""`). The default value is +`winlog`. + +`overwrite_keys`:: (Optional) A boolean that specifies whether keys that already +exist in the event are overwritten by keys from the decoded XML object. The +default value is `true`. + +`map_ecs_fields`:: (Optional) A boolean that specifies whether to map additional ECS +fields when possible. The default value is `true`. + +`document_id`:: (Optional) XML key to use as the document ID. If configured, the +field will be removed from the original XML document and stored in +`@metadata._id`. + +`ignore_missing`:: (Optional) If `true` the processor will not return an error +when a specified field does not exist. Defaults to `false`. + +`ignore_failure`:: (Optional) Ignore all errors produced by the processor. +Defaults to `false`. + +Example: + +[source,yaml] +------------------------------------------------------------------------------- +processors: + - decode_xml_wineventlog: + field: event.original + target_field: winlog +------------------------------------------------------------------------------- + +[source,json] +------------------------------------------------------------------------------- +{ + "event": { + "original": "4672001254800x802000000000000011303SecurityvagrantS-1-5-18SYSTEMNT AUTHORITY0x3e7SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeSpecial privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformationSpecial LogonInfoSecurityMicrosoft Windows security auditing.Audit Success" + } +} +------------------------------------------------------------------------------- + +Will produce the following output: + +[source,json] +------------------------------------------------------------------------------- +{ + "event": { + "original": "4672001254800x802000000000000011303SecurityvagrantS-1-5-18SYSTEMNT AUTHORITY0x3e7SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeSpecial privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformationSpecial LogonInfoSecurityMicrosoft Windows security auditing.Audit Success", + "action": "Special Logon", + "code": "4672", + "kind": "event", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + }, + "host": { + "name": "vagrant", + }, + "log": { + "level": "information", + }, + "winlog": { + "channel": "Security", + "outcome": "success", + "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}", + "level": "information", + "event_id": 4672, + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 11303, + "computer_name": "vagrant", + "keywords_raw": 9232379236109516800, + "opcode": "Info", + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "event_data": { + "SubjectUserSid": "S-1-5-18", + "SubjectUserName": "SYSTEM", + "SubjectDomainName": "NT AUTHORITY", + "SubjectLogonId": "0x3e7", + "PrivilegeList": "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege" + }, + "task": "Special Logon", + "keywords": [ + "Audit Success" + ], + "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", + "process": { + "pid": 652, + "thread": { + "id": 4660 + } + } + } +} +------------------------------------------------------------------------------- + +See <> for a list of supported conditions. diff --git a/libbeat/processors/decode_xml_wineventlog/processor.go b/libbeat/processors/decode_xml_wineventlog/processor.go new file mode 100644 index 00000000000..9b9b698d7f3 --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/processor.go @@ -0,0 +1,171 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package decode_xml_wineventlog + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/common/jsontransform" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/beats/v7/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" + "github.com/elastic/beats/v7/winlogbeat/sys/winevent" +) + +var ( + errFieldIsNotString = errors.New("field value is not a string") +) + +const ( + procName = "decode_xml_wineventlog" + logName = "processor." + procName +) + +func init() { + processors.RegisterPlugin(procName, + checks.ConfigChecked(New, + checks.RequireFields("field"), + checks.AllowedFields( + "field", "target_field", + "overwrite_keys", "document_id", + "ignore_missing", "ignore_failure", + ))) + jsprocessor.RegisterPlugin(procName, New) +} + +type processor struct { + config + + decoder decoder + log *logp.Logger +} + +type decoder interface { + decode(data []byte) (win, ecs common.MapStr, err error) +} + +// New constructs a new decode_xml processor. +func New(c *common.Config) (processors.Processor, error) { + config := defaultConfig() + + if err := c.Unpack(&config); err != nil { + return nil, fmt.Errorf("fail to unpack the "+procName+" processor configuration: %s", err) + } + + return newProcessor(config) +} + +func newProcessor(config config) (processors.Processor, error) { + cfgwarn.Experimental("The " + procName + " processor is experimental.") + + return &processor{ + config: config, + decoder: newDecoder(), + log: logp.NewLogger(logName), + }, nil +} + +func (p *processor) Run(event *beat.Event) (*beat.Event, error) { + if err := p.run(event); err != nil && !p.IgnoreFailure { + err = fmt.Errorf("failed in decode_xml on the %q field: %w", p.Field, err) + event.PutValue("error.message", err.Error()) + return event, err + } + return event, nil +} + +func (p *processor) run(event *beat.Event) error { + data, err := event.GetValue(p.Field) + if err != nil { + if p.IgnoreMissing && err == common.ErrKeyNotFound { + return nil + } + return err + } + + text, ok := data.(string) + if !ok { + return errFieldIsNotString + } + + win, ecs, err := p.decoder.decode([]byte(text)) + if err != nil { + return fmt.Errorf("error decoding XML field: %w", err) + } + + var id string + if tmp, err := common.MapStr(win).GetValue(p.DocumentID); err == nil { + if v, ok := tmp.(string); ok { + id = v + common.MapStr(win).Delete(p.DocumentID) + } + } + + if p.Target != "" { + if _, err = event.PutValue(p.Target, win); err != nil { + return fmt.Errorf("failed to put value %v into field %q: %w", win, p.Target, err) + } + } else { + jsontransform.WriteJSONKeys(event, win, false, p.OverwriteKeys, !p.IgnoreFailure) + } + + if p.MapECSFields { + jsontransform.WriteJSONKeys(event, ecs, false, p.OverwriteKeys, !p.IgnoreFailure) + } + + if id != "" { + event.SetID(id) + } + + return nil +} + +func (p *processor) String() string { + json, _ := json.Marshal(p.config) + return procName + "=" + string(json) +} + +func fields(evt winevent.Event) (common.MapStr, common.MapStr) { + win := evt.Fields() + + ecs := common.MapStr{} + + ecs.Put("event.kind", "event") + ecs.Put("event.code", fmt.Sprint(evt.EventIdentifier.ID)) + ecs.Put("event.provider", evt.Provider.Name) + winevent.AddOptional(ecs, "event.action", evt.Task) + winevent.AddOptional(ecs, "host.name", evt.Computer) + winevent.AddOptional(ecs, "event.outcome", getValue(win, "outcome")) + winevent.AddOptional(ecs, "log.level", getValue(win, "level")) + winevent.AddOptional(ecs, "message", getValue(win, "message")) + winevent.AddOptional(ecs, "error.code", getValue(win, "error.code")) + winevent.AddOptional(ecs, "error.message", getValue(win, "error.message")) + + return win, ecs +} + +func getValue(m common.MapStr, key string) interface{} { + v, _ := m.GetValue(key) + return v +} diff --git a/libbeat/processors/decode_xml_wineventlog/processor_test.go b/libbeat/processors/decode_xml_wineventlog/processor_test.go new file mode 100644 index 00000000000..97ece6061c8 --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/processor_test.go @@ -0,0 +1,197 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package decode_xml_wineventlog + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestProcessor(t *testing.T) { + var testCases = []struct { + description string + config config + Input common.MapStr + Output common.MapStr + error bool + errorMessage string + }{ + { + description: "Decodes properly with default config", + config: defaultConfig(), + Input: common.MapStr{ + "message": "" + + "4672001254800x8020000000000000" + + "11303Securityvagrant" + + "S-1-5-18SYSTEMNT AUTHORITY0x3e7" + + "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\t" + + "SeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege" + + "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\n" + + "Privileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\t" + + "SeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformation" + + "Special LogonInfoSecurityMicrosoft Windows security auditing.Audit Success", + }, + Output: common.MapStr{ + "event": common.MapStr{ + "action": "Special Logon", + "code": "4672", + "kind": "event", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + }, + "host": common.MapStr{ + "name": "vagrant", + }, + "log": common.MapStr{ + "level": "information", + }, + "winlog": common.MapStr{ + "channel": "Security", + "outcome": "success", + "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}", + "level": "information", + "event_id": uint32(4672), + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": uint64(11303), + "computer_name": "vagrant", + "time_created": func() time.Time { + t, _ := time.Parse(time.RFC3339Nano, "2021-03-23T09:56:13.137310000Z") + return t + }(), + "opcode": "Info", + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "event_data": common.MapStr{ + "SubjectUserSid": "S-1-5-18", + "SubjectUserName": "SYSTEM", + "SubjectDomainName": "NT AUTHORITY", + "SubjectLogonId": "0x3e7", + "PrivilegeList": "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", + }, + "task": "Special Logon", + "keywords": []string{ + "Audit Success", + }, + "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", + "process": common.MapStr{ + "pid": uint32(652), + "thread": common.MapStr{ + "id": uint32(4660), + }, + }, + }, + "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", + }, + }, + { + description: "Decodes without ECS", + config: config{ + Field: "message", + OverwriteKeys: true, + MapECSFields: false, + Target: "winlog", + }, + Input: common.MapStr{ + "message": "" + + "4672001254800x8020000000000000" + + "11303Securityvagrant" + + "S-1-5-18SYSTEMNT AUTHORITY0x3e7" + + "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\t" + + "SeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege" + + "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\n" + + "Privileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\t" + + "SeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformation" + + "Special LogonInfoSecurityMicrosoft Windows security auditing.Audit Success", + }, + Output: common.MapStr{ + "winlog": common.MapStr{ + "channel": "Security", + "outcome": "success", + "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}", + "level": "information", + "event_id": uint32(4672), + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": uint64(11303), + "computer_name": "vagrant", + "time_created": func() time.Time { + t, _ := time.Parse(time.RFC3339Nano, "2021-03-23T09:56:13.137310000Z") + return t + }(), + "opcode": "Info", + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "event_data": common.MapStr{ + "SubjectUserSid": "S-1-5-18", + "SubjectUserName": "SYSTEM", + "SubjectDomainName": "NT AUTHORITY", + "SubjectLogonId": "0x3e7", + "PrivilegeList": "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", + }, + "task": "Special Logon", + "keywords": []string{ + "Audit Success", + }, + "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", + "process": common.MapStr{ + "pid": uint32(652), + "thread": common.MapStr{ + "id": uint32(4660), + }, + }, + }, + "message": "" + + "4672001254800x8020000000000000" + + "11303Securityvagrant" + + "S-1-5-18SYSTEMNT AUTHORITY0x3e7" + + "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\t" + + "SeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege" + + "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\n" + + "Privileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\t" + + "SeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilegeInformation" + + "Special LogonInfoSecurityMicrosoft Windows security auditing.Audit Success", + }, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.description, func(t *testing.T) { + t.Parallel() + + f, err := newProcessor(test.config) + require.NoError(t, err) + + event := &beat.Event{ + Fields: test.Input, + } + newEvent, err := f.Run(event) + if !test.error { + assert.NoError(t, err) + } else { + if assert.Error(t, err) { + assert.Contains(t, err.Error(), test.errorMessage) + } + } + assert.Equal(t, test.Output, newEvent.Fields) + }) + } +} From 61aac68c288c17cd81f641677f18473a7374ea6f Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 16 Apr 2021 10:26:23 +0200 Subject: [PATCH 5/9] Add missing fields to config checks --- libbeat/processors/decode_xml_wineventlog/processor.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libbeat/processors/decode_xml_wineventlog/processor.go b/libbeat/processors/decode_xml_wineventlog/processor.go index 9b9b698d7f3..c996d04229b 100644 --- a/libbeat/processors/decode_xml_wineventlog/processor.go +++ b/libbeat/processors/decode_xml_wineventlog/processor.go @@ -45,11 +45,12 @@ const ( func init() { processors.RegisterPlugin(procName, checks.ConfigChecked(New, - checks.RequireFields("field"), + checks.RequireFields("field", "target_field"), checks.AllowedFields( "field", "target_field", "overwrite_keys", "document_id", "ignore_missing", "ignore_failure", + "map_ecs_fields", ))) jsprocessor.RegisterPlugin(procName, New) } From 38c6a3248650d5eac42dea677529988cbd2a5fad Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 19 Apr 2021 17:55:49 +0200 Subject: [PATCH 6/9] Change event.code type --- libbeat/processors/decode_xml_wineventlog/processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/processors/decode_xml_wineventlog/processor.go b/libbeat/processors/decode_xml_wineventlog/processor.go index c996d04229b..362a727093f 100644 --- a/libbeat/processors/decode_xml_wineventlog/processor.go +++ b/libbeat/processors/decode_xml_wineventlog/processor.go @@ -153,7 +153,7 @@ func fields(evt winevent.Event) (common.MapStr, common.MapStr) { ecs := common.MapStr{} ecs.Put("event.kind", "event") - ecs.Put("event.code", fmt.Sprint(evt.EventIdentifier.ID)) + ecs.Put("event.code", evt.EventIdentifier.ID) ecs.Put("event.provider", evt.Provider.Name) winevent.AddOptional(ecs, "event.action", evt.Task) winevent.AddOptional(ecs, "host.name", evt.Computer) From d6ec2fdb886b8835e5cd101a46659f006497ce2b Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 19 Apr 2021 18:22:28 +0200 Subject: [PATCH 7/9] Fix PR number in changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0a728896087..1b10e0cae36 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -617,6 +617,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add kubernetes.pod.ip field in kubernetes metadata. {pull}25037[25037] - Discover changes in Kubernetes namespace metadata as soon as they happen. {pull}25117[25117] - Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25109[25109] +- Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25115[25115] *Auditbeat* From 50209e10de4830910e3f9b9bf4de844f05838e4b Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 19 Apr 2021 19:07:58 +0200 Subject: [PATCH 8/9] Fix test --- libbeat/processors/decode_xml_wineventlog/processor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/processors/decode_xml_wineventlog/processor_test.go b/libbeat/processors/decode_xml_wineventlog/processor_test.go index 97ece6061c8..4e31a013861 100644 --- a/libbeat/processors/decode_xml_wineventlog/processor_test.go +++ b/libbeat/processors/decode_xml_wineventlog/processor_test.go @@ -55,7 +55,7 @@ func TestProcessor(t *testing.T) { Output: common.MapStr{ "event": common.MapStr{ "action": "Special Logon", - "code": "4672", + "code": uint32(4672), "kind": "event", "outcome": "success", "provider": "Microsoft-Windows-Security-Auditing", From cfcb4f11ed8d7d791eaf418402927f812c342a78 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 20 Apr 2021 08:19:02 +0200 Subject: [PATCH 9/9] Remove document_id and make docs more clear --- CHANGELOG.next.asciidoc | 1 - .../processors/decode_xml_wineventlog/config.go | 1 - .../docs/decode_xml_wineventlog.asciidoc | 6 +----- .../decode_xml_wineventlog/processor.go | 17 ++--------------- 4 files changed, 3 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1b10e0cae36..262e4f6eea9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -616,7 +616,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Libbeat: report queue capacity, output batch size, and output client count to monitoring. {pull}24700[24700] - Add kubernetes.pod.ip field in kubernetes metadata. {pull}25037[25037] - Discover changes in Kubernetes namespace metadata as soon as they happen. {pull}25117[25117] -- Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25109[25109] - Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25115[25115] *Auditbeat* diff --git a/libbeat/processors/decode_xml_wineventlog/config.go b/libbeat/processors/decode_xml_wineventlog/config.go index ec543c489f5..061a4d0a52f 100644 --- a/libbeat/processors/decode_xml_wineventlog/config.go +++ b/libbeat/processors/decode_xml_wineventlog/config.go @@ -22,7 +22,6 @@ type config struct { Target string `config:"target_field"` OverwriteKeys bool `config:"overwrite_keys"` MapECSFields bool `config:"map_ecs_fields"` - DocumentID string `config:"document_id"` IgnoreMissing bool `config:"ignore_missing"` IgnoreFailure bool `config:"ignore_failure"` } diff --git a/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc b/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc index 1a91e5b4cb4..1df47d3f955 100644 --- a/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc +++ b/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc @@ -27,11 +27,7 @@ exist in the event are overwritten by keys from the decoded XML object. The default value is `true`. `map_ecs_fields`:: (Optional) A boolean that specifies whether to map additional ECS -fields when possible. The default value is `true`. - -`document_id`:: (Optional) XML key to use as the document ID. If configured, the -field will be removed from the original XML document and stored in -`@metadata._id`. +fields when possible. Note that ECS field keys are placed outside of `target_field`. The default value is `true`. `ignore_missing`:: (Optional) If `true` the processor will not return an error when a specified field does not exist. Defaults to `false`. diff --git a/libbeat/processors/decode_xml_wineventlog/processor.go b/libbeat/processors/decode_xml_wineventlog/processor.go index 362a727093f..b5487764a2b 100644 --- a/libbeat/processors/decode_xml_wineventlog/processor.go +++ b/libbeat/processors/decode_xml_wineventlog/processor.go @@ -48,9 +48,8 @@ func init() { checks.RequireFields("field", "target_field"), checks.AllowedFields( "field", "target_field", - "overwrite_keys", "document_id", + "overwrite_keys", "map_ecs_fields", "ignore_missing", "ignore_failure", - "map_ecs_fields", ))) jsprocessor.RegisterPlugin(procName, New) } @@ -89,7 +88,7 @@ func newProcessor(config config) (processors.Processor, error) { func (p *processor) Run(event *beat.Event) (*beat.Event, error) { if err := p.run(event); err != nil && !p.IgnoreFailure { - err = fmt.Errorf("failed in decode_xml on the %q field: %w", p.Field, err) + err = fmt.Errorf("failed in decode_xml_wineventlog on the %q field: %w", p.Field, err) event.PutValue("error.message", err.Error()) return event, err } @@ -115,14 +114,6 @@ func (p *processor) run(event *beat.Event) error { return fmt.Errorf("error decoding XML field: %w", err) } - var id string - if tmp, err := common.MapStr(win).GetValue(p.DocumentID); err == nil { - if v, ok := tmp.(string); ok { - id = v - common.MapStr(win).Delete(p.DocumentID) - } - } - if p.Target != "" { if _, err = event.PutValue(p.Target, win); err != nil { return fmt.Errorf("failed to put value %v into field %q: %w", win, p.Target, err) @@ -135,10 +126,6 @@ func (p *processor) run(event *beat.Event) error { jsontransform.WriteJSONKeys(event, ecs, false, p.OverwriteKeys, !p.IgnoreFailure) } - if id != "" { - event.SetID(id) - } - return nil }