From b0123c96f59d3922ab92925d9308dd0f8ec1b2a1 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 9 Jul 2021 11:20:52 -0400 Subject: [PATCH 1/2] Fixes for wineventlog experimental api Some valid message strings have a "message ID" of 0. So set the EvtFormatMessageId flag when we don't have a valid event handle when calling EvtFormatMessage. Change the delimiters used in message templates. There were a few cases were the template failed to parse because the raw message used the same delims in some way. EventMetadata is an exported type, but its EventData field was using an unexported type called `eventData`. To make external usage easier make that type exported as well. Add winevetlog.Publishers() to list all registered event publishers on the system. --- CHANGELOG.next.asciidoc | 1 + winlogbeat/sys/wineventlog/format_message.go | 2 +- winlogbeat/sys/wineventlog/metadata_store.go | 22 ++++++---- winlogbeat/sys/wineventlog/stringinserts.go | 8 +++- winlogbeat/sys/wineventlog/syscall_windows.go | 2 + winlogbeat/sys/wineventlog/template.go | 9 ++-- .../sys/wineventlog/wineventlog_windows.go | 41 +++++++++++++++++++ .../wineventlog/wineventlog_windows_test.go | 12 ++++++ .../sys/wineventlog/zsyscall_windows.go | 21 +++++++++- 9 files changed, 102 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 603d026be3e..4a4127bbc73 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -521,6 +521,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Protect against accessing undefined variables in Sysmon module. {issue}22219[22219] {pull}22236[22236] - Protect against accessing an undefined variable in Security module. {pull}22937[22937] - Fix related.ip field in renameCommonAuthFields {pull}24892[24892] +- Fix an issue with message template caching in the `wineventlog-experimental` API implementation. {pull}26826[26826] *Functionbeat* diff --git a/winlogbeat/sys/wineventlog/format_message.go b/winlogbeat/sys/wineventlog/format_message.go index 29182d6f6a0..285d130fc40 100644 --- a/winlogbeat/sys/wineventlog/format_message.go +++ b/winlogbeat/sys/wineventlog/format_message.go @@ -45,7 +45,7 @@ func getMessageString(metadata *PublisherMetadata, eventHandle EvtHandle, messag var flags EvtFormatMessageFlag if eventHandle > 0 { flags = EvtFormatMessageEvent - } else if messageID > 0 { + } else { flags = EvtFormatMessageId } diff --git a/winlogbeat/sys/wineventlog/metadata_store.go b/winlogbeat/sys/wineventlog/metadata_store.go index a02da985b4f..c568c24d320 100644 --- a/winlogbeat/sys/wineventlog/metadata_store.go +++ b/winlogbeat/sys/wineventlog/metadata_store.go @@ -283,7 +283,7 @@ type EventMetadata struct { Version uint8 // Event format version. MsgStatic string // Used when the message has no parameters. MsgTemplate *template.Template `json:"-"` // Template that expects an array of values as its data. - EventData []eventData // Names of parameters from XML template. + EventData []EventData // Names of parameters from XML template. } // newEventMetadataFromEventHandle collects metadata about an event type using @@ -307,11 +307,11 @@ func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle E } if len(event.EventData.Pairs) > 0 { for _, pair := range event.EventData.Pairs { - em.EventData = append(em.EventData, eventData{Name: pair.Key}) + em.EventData = append(em.EventData, EventData{Name: pair.Key}) } } else { for _, pair := range event.UserData.Pairs { - em.EventData = append(em.EventData, eventData{Name: pair.Key}) + em.EventData = append(em.EventData, EventData{Name: pair.Key}) } } @@ -396,10 +396,15 @@ func (em *EventMetadata) initEventMessage(itr *EventMetadataIterator, publisher if err != nil { return err } + // If the event definition does not specify a message, the value is –1. + if int32(messageID) == -1 { + return nil + } msg, err := getMessageString(publisher, NilHandle, messageID, templateInserts.Slice()) if err != nil { - return err + return errors.Wrapf(err, "failed to get message string using message "+ + "ID %v for for event ID %v", messageID, em.EventID) } return em.setMessage(msg) @@ -409,9 +414,12 @@ func (em *EventMetadata) setMessage(msg string) error { msg = sys.RemoveWindowsLineEndings(msg) tmplID := strconv.Itoa(int(em.EventID)) - tmpl, err := template.New(tmplID).Funcs(eventMessageTemplateFuncs).Parse(msg) + tmpl, err := template.New(tmplID). + Delims(leftTemplateDelim, rightTemplateDelim). + Funcs(eventMessageTemplateFuncs).Parse(msg) if err != nil { - return err + return errors.Wrapf(err, "failed to parse message template for "+ + "event ID %v (template='%v')", em.EventID, msg) } // One node means there were no parameters so this will optimize that case @@ -432,7 +440,7 @@ func (em *EventMetadata) equal(other *EventMetadata) bool { return false } - eventDataNamesEqual := func(a, b []eventData) bool { + eventDataNamesEqual := func(a, b []EventData) bool { if len(a) != len(b) { return false } diff --git a/winlogbeat/sys/wineventlog/stringinserts.go b/winlogbeat/sys/wineventlog/stringinserts.go index bf63a1c258e..f53039732d0 100644 --- a/winlogbeat/sys/wineventlog/stringinserts.go +++ b/winlogbeat/sys/wineventlog/stringinserts.go @@ -30,6 +30,9 @@ const ( // maxInsertStrings is the maximum number of parameters supported in a // Windows event message. maxInsertStrings = 99 + + leftTemplateDelim = "[{{" + rightTemplateDelim = "}}]" ) // templateInserts contains EvtVariant values that can be used to substitute @@ -45,7 +48,7 @@ type stringInserts struct { evtVariants [maxInsertStrings]EvtVariant } -// Pointer returns a pointer the EvtVariant array. +// Slice returns a slice of the full EvtVariant array. func (si *stringInserts) Slice() []EvtVariant { return si.evtVariants[:] } @@ -66,7 +69,8 @@ func newTemplateStringInserts() *stringInserts { for i := 0; i < len(si.evtVariants); i++ { // Use i+1 to keep our inserts numbered the same as Window's inserts. - strSlice, err := windows.UTF16FromString(`{{eventParam $ ` + strconv.Itoa(i+1) + `}}`) + templateParam := leftTemplateDelim + `eventParam $ ` + strconv.Itoa(i+1) + rightTemplateDelim + strSlice, err := windows.UTF16FromString(templateParam) if err != nil { // This will never happen. panic(err) diff --git a/winlogbeat/sys/wineventlog/syscall_windows.go b/winlogbeat/sys/wineventlog/syscall_windows.go index d64c1a804cf..6b60b8a6ba0 100644 --- a/winlogbeat/sys/wineventlog/syscall_windows.go +++ b/winlogbeat/sys/wineventlog/syscall_windows.go @@ -661,3 +661,5 @@ func EvtClearLog(session EvtHandle, channelPath string, targetFilePath string) e //sys _EvtNextEventMetadata(enumerator EvtHandle, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtNextEventMetadata //sys _EvtGetObjectArrayProperty(objectArray EvtObjectArrayPropertyHandle, propertyID EvtPublisherMetadataPropertyID, arrayIndex uint32, flags uint32, bufferSize uint32, evtVariant *EvtVariant, bufferUsed *uint32) (err error) = wevtapi.EvtGetObjectArrayProperty //sys _EvtGetObjectArraySize(objectArray EvtObjectArrayPropertyHandle, arraySize *uint32) (err error) = wevtapi.EvtGetObjectArraySize +//sys _EvtOpenPublisherEnum(session EvtHandle, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtOpenPublisherEnum +//sys _EvtNextPublisherId(enumerator EvtHandle, bufferSize uint32, buffer *byte, bufferUsed *uint32) (err error) = wevtapi.EvtNextPublisherId diff --git a/winlogbeat/sys/wineventlog/template.go b/winlogbeat/sys/wineventlog/template.go index e5b5c9b99ae..ac19322ef0a 100644 --- a/winlogbeat/sys/wineventlog/template.go +++ b/winlogbeat/sys/wineventlog/template.go @@ -22,12 +22,13 @@ import ( ) type eventTemplate struct { - Data []eventData `xml:"data"` + Data []EventData `xml:"data"` } -type eventData struct { - Name string `xml:"name,attr"` - Type string `xml:"outType,attr"` +type EventData struct { + Name string `xml:"name,attr"` + InType string `xml:"inType,attr"` + OutType string `xml:"outType,attr"` } func (t *eventTemplate) Unmarshal(xmlData []byte) error { diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows.go b/winlogbeat/sys/wineventlog/wineventlog_windows.go index e0f36680320..833ef8c8dfa 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows.go @@ -25,6 +25,7 @@ import ( "io" "reflect" "runtime" + "sort" "syscall" "github.com/elastic/beats/v7/libbeat/common" @@ -419,6 +420,46 @@ func FormatEventString( return common.UTF16ToUTF8Bytes(buffer[:bufferUsed], out) } +// Publishers returns a sort list of event publishers on the local computer. +func Publishers() ([]string, error) { + publisherEnumerator, err := _EvtOpenPublisherEnum(NilHandle, 0) + if err != nil { + return nil, fmt.Errorf("failed in EvtOpenPublisherEnum: %w", err) + } + defer Close(publisherEnumerator) + + var ( + publishers []string + bufferUsed uint32 + buffer = make([]byte, 1024) + ) + +loop: + for { + if err = _EvtNextPublisherId(publisherEnumerator, uint32(len(buffer)), &buffer[0], &bufferUsed); err != nil { + switch err { + case ERROR_NO_MORE_ITEMS: + break loop + case ERROR_INSUFFICIENT_BUFFER: + buffer = make([]byte, 2*bufferUsed) + continue loop + default: + return nil, fmt.Errorf("failed in EvtNextPublisherId: %w", err) + } + } + + provider, err := sys.UTF16BytesToString(buffer) + if err != nil { + return nil, err + } + + publishers = append(publishers, provider) + } + + sort.Strings(publishers) + return publishers, nil +} + // offset reads a pointer value from the reader then calculates an offset from // the start of the buffer to the pointer location. If the pointer value is // NULL or is outside of the bounds of the buffer then an error is returned. diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows_test.go b/winlogbeat/sys/wineventlog/wineventlog_windows_test.go index 9701cfd3679..ff4fe566e9c 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows_test.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows_test.go @@ -108,3 +108,15 @@ func TestChannels(t *testing.T) { } } } + +func TestPublishers(t *testing.T) { + publishers, err := Publishers() + if err != nil { + t.Fatal(err) + } + + assert.NotEmpty(t, publishers) + for _, p := range publishers { + t.Log(p) + } +} diff --git a/winlogbeat/sys/wineventlog/zsyscall_windows.go b/winlogbeat/sys/wineventlog/zsyscall_windows.go index e8e22513c16..1af4dfff888 100644 --- a/winlogbeat/sys/wineventlog/zsyscall_windows.go +++ b/winlogbeat/sys/wineventlog/zsyscall_windows.go @@ -69,9 +69,11 @@ var ( procEvtNext = modwevtapi.NewProc("EvtNext") procEvtNextChannelPath = modwevtapi.NewProc("EvtNextChannelPath") procEvtNextEventMetadata = modwevtapi.NewProc("EvtNextEventMetadata") + procEvtNextPublisherId = modwevtapi.NewProc("EvtNextPublisherId") procEvtOpenChannelEnum = modwevtapi.NewProc("EvtOpenChannelEnum") procEvtOpenEventMetadataEnum = modwevtapi.NewProc("EvtOpenEventMetadataEnum") procEvtOpenLog = modwevtapi.NewProc("EvtOpenLog") + procEvtOpenPublisherEnum = modwevtapi.NewProc("EvtOpenPublisherEnum") procEvtOpenPublisherMetadata = modwevtapi.NewProc("EvtOpenPublisherMetadata") procEvtQuery = modwevtapi.NewProc("EvtQuery") procEvtRender = modwevtapi.NewProc("EvtRender") @@ -179,6 +181,14 @@ func _EvtNextEventMetadata(enumerator EvtHandle, flags uint32) (handle EvtHandle return } +func _EvtNextPublisherId(enumerator EvtHandle, bufferSize uint32, buffer *byte, bufferUsed *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procEvtNextPublisherId.Addr(), 4, uintptr(enumerator), uintptr(bufferSize), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(bufferUsed)), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + func _EvtOpenChannelEnum(session EvtHandle, flags uint32) (handle EvtHandle, err error) { r0, _, e1 := syscall.Syscall(procEvtOpenChannelEnum.Addr(), 2, uintptr(session), uintptr(flags), 0) handle = EvtHandle(r0) @@ -206,6 +216,15 @@ func _EvtOpenLog(session EvtHandle, path *uint16, flags uint32) (handle EvtHandl return } +func _EvtOpenPublisherEnum(session EvtHandle, flags uint32) (handle EvtHandle, err error) { + r0, _, e1 := syscall.Syscall(procEvtOpenPublisherEnum.Addr(), 2, uintptr(session), uintptr(flags), 0) + handle = EvtHandle(r0) + if handle == 0 { + err = errnoErr(e1) + } + return +} + func _EvtOpenPublisherMetadata(session EvtHandle, publisherIdentity *uint16, logFilePath *uint16, locale uint32, flags uint32) (handle EvtHandle, err error) { r0, _, e1 := syscall.Syscall6(procEvtOpenPublisherMetadata.Addr(), 5, uintptr(session), uintptr(unsafe.Pointer(publisherIdentity)), uintptr(unsafe.Pointer(logFilePath)), uintptr(locale), uintptr(flags), 0) handle = EvtHandle(r0) @@ -233,12 +252,10 @@ func _EvtRender(context EvtHandle, fragment EvtHandle, flags EvtRenderFlag, buff } func _EvtSeek(resultSet EvtHandle, position int64, bookmark EvtHandle, timeout uint32, flags uint32) (success bool, err error) { - var ( r0 uintptr e1 syscall.Errno ) - if unsafe.Sizeof(uintptr(0)) == unsafe.Sizeof(uint64(0)) { r0, _, e1 = syscall.Syscall6(procEvtSeek.Addr(), 5, uintptr(resultSet), uintptr(position), uintptr(bookmark), uintptr(timeout), uintptr(flags), 0) } else { From 70ff550a80b9663481ee485f57ccc82c6a4cd3b5 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 12 Jul 2021 13:57:21 -0400 Subject: [PATCH 2/2] Use uint16 to represent TCHARs --- winlogbeat/sys/wineventlog/syscall_windows.go | 2 +- winlogbeat/sys/wineventlog/wineventlog_windows.go | 10 +++------- winlogbeat/sys/wineventlog/zsyscall_windows.go | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/winlogbeat/sys/wineventlog/syscall_windows.go b/winlogbeat/sys/wineventlog/syscall_windows.go index 6b60b8a6ba0..836741f53f0 100644 --- a/winlogbeat/sys/wineventlog/syscall_windows.go +++ b/winlogbeat/sys/wineventlog/syscall_windows.go @@ -662,4 +662,4 @@ func EvtClearLog(session EvtHandle, channelPath string, targetFilePath string) e //sys _EvtGetObjectArrayProperty(objectArray EvtObjectArrayPropertyHandle, propertyID EvtPublisherMetadataPropertyID, arrayIndex uint32, flags uint32, bufferSize uint32, evtVariant *EvtVariant, bufferUsed *uint32) (err error) = wevtapi.EvtGetObjectArrayProperty //sys _EvtGetObjectArraySize(objectArray EvtObjectArrayPropertyHandle, arraySize *uint32) (err error) = wevtapi.EvtGetObjectArraySize //sys _EvtOpenPublisherEnum(session EvtHandle, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtOpenPublisherEnum -//sys _EvtNextPublisherId(enumerator EvtHandle, bufferSize uint32, buffer *byte, bufferUsed *uint32) (err error) = wevtapi.EvtNextPublisherId +//sys _EvtNextPublisherId(enumerator EvtHandle, bufferSize uint32, buffer *uint16, bufferUsed *uint32) (err error) = wevtapi.EvtNextPublisherId diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows.go b/winlogbeat/sys/wineventlog/wineventlog_windows.go index 833ef8c8dfa..20898f5fbe6 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows.go @@ -431,7 +431,7 @@ func Publishers() ([]string, error) { var ( publishers []string bufferUsed uint32 - buffer = make([]byte, 1024) + buffer = make([]uint16, 1024) ) loop: @@ -441,18 +441,14 @@ loop: case ERROR_NO_MORE_ITEMS: break loop case ERROR_INSUFFICIENT_BUFFER: - buffer = make([]byte, 2*bufferUsed) + buffer = make([]uint16, bufferUsed) continue loop default: return nil, fmt.Errorf("failed in EvtNextPublisherId: %w", err) } } - provider, err := sys.UTF16BytesToString(buffer) - if err != nil { - return nil, err - } - + provider := windows.UTF16ToString(buffer) publishers = append(publishers, provider) } diff --git a/winlogbeat/sys/wineventlog/zsyscall_windows.go b/winlogbeat/sys/wineventlog/zsyscall_windows.go index 1af4dfff888..0388835e30f 100644 --- a/winlogbeat/sys/wineventlog/zsyscall_windows.go +++ b/winlogbeat/sys/wineventlog/zsyscall_windows.go @@ -181,7 +181,7 @@ func _EvtNextEventMetadata(enumerator EvtHandle, flags uint32) (handle EvtHandle return } -func _EvtNextPublisherId(enumerator EvtHandle, bufferSize uint32, buffer *byte, bufferUsed *uint32) (err error) { +func _EvtNextPublisherId(enumerator EvtHandle, bufferSize uint32, buffer *uint16, bufferUsed *uint32) (err error) { r1, _, e1 := syscall.Syscall6(procEvtNextPublisherId.Addr(), 4, uintptr(enumerator), uintptr(bufferSize), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(bufferUsed)), 0, 0) if r1 == 0 { err = errnoErr(e1)