Fixes for wineventlog experimental api

Some valid message strings have a "message ID" of 0. So set the EvtFormatMessageId flag when we don't have a valid event handle when calling EvtFormatMessage.

Change the delimiters used in message templates. There were a few cases were the template failed to parse because the raw message used the same delims in some way.

EventMetadata is an exported type, but its EventData field was using an unexported type called `eventData`. To make external usage easier make that type exported as well.

Add winevetlog.Publishers() to list all registered event publishers on the system.

CHANGELOG.next.asciidoc

@@ -521,6 +521,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Protect against accessing undefined variables in Sysmon module. {issue}22219[22219] {pull}22236[22236]
 - Protect against accessing an undefined variable in Security module. {pull}22937[22937]
 - Fix related.ip field in renameCommonAuthFields {pull}24892[24892]
+- Fix an issue with message template caching in the `wineventlog-experimental` API implementation. {pull}26826[26826]
 
 *Functionbeat*
 

winlogbeat/sys/wineventlog/format_message.go

@@ -45,7 +45,7 @@ func getMessageString(metadata *PublisherMetadata, eventHandle EvtHandle, messag
 	var flags EvtFormatMessageFlag
 	if eventHandle > 0 {
 		flags = EvtFormatMessageEvent
-	} else if messageID > 0 {
+	} else {
 		flags = EvtFormatMessageId
 	}
 

winlogbeat/sys/wineventlog/metadata_store.go

@@ -283,7 +283,7 @@ type EventMetadata struct {
 	Version     uint8              // Event format version.
 	MsgStatic   string             // Used when the message has no parameters.
 	MsgTemplate *template.Template `json:"-"` // Template that expects an array of values as its data.
-	EventData   []eventData        // Names of parameters from XML template.
+	EventData   []EventData        // Names of parameters from XML template.
 }
 
 // newEventMetadataFromEventHandle collects metadata about an event type using
@@ -307,11 +307,11 @@ func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle E
 	}
 	if len(event.EventData.Pairs) > 0 {
 		for _, pair := range event.EventData.Pairs {
-			em.EventData = append(em.EventData, eventData{Name: pair.Key})
+			em.EventData = append(em.EventData, EventData{Name: pair.Key})
 		}
 	} else {
 		for _, pair := range event.UserData.Pairs {
-			em.EventData = append(em.EventData, eventData{Name: pair.Key})
+			em.EventData = append(em.EventData, EventData{Name: pair.Key})
 		}
 	}
 
@@ -396,10 +396,15 @@ func (em *EventMetadata) initEventMessage(itr *EventMetadataIterator, publisher
 	if err != nil {
 		return err
 	}
+	// If the event definition does not specify a message, the value is –1.
+	if int32(messageID) == -1 {
+		return nil
+	}
 
 	msg, err := getMessageString(publisher, NilHandle, messageID, templateInserts.Slice())
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "failed to get message string using message "+
+			"ID %v for for event ID %v", messageID, em.EventID)
 	}
 
 	return em.setMessage(msg)
@@ -409,9 +414,12 @@ func (em *EventMetadata) setMessage(msg string) error {
 	msg = sys.RemoveWindowsLineEndings(msg)
 	tmplID := strconv.Itoa(int(em.EventID))
 
-	tmpl, err := template.New(tmplID).Funcs(eventMessageTemplateFuncs).Parse(msg)
+	tmpl, err := template.New(tmplID).
+		Delims(leftTemplateDelim, rightTemplateDelim).
+		Funcs(eventMessageTemplateFuncs).Parse(msg)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "failed to parse message template for "+
+			"event ID %v (template='%v')", em.EventID, msg)
 	}
 
 	// One node means there were no parameters so this will optimize that case
@@ -432,7 +440,7 @@ func (em *EventMetadata) equal(other *EventMetadata) bool {
 		return false
 	}
 
-	eventDataNamesEqual := func(a, b []eventData) bool {
+	eventDataNamesEqual := func(a, b []EventData) bool {
 		if len(a) != len(b) {
 			return false
 		}

winlogbeat/sys/wineventlog/stringinserts.go

@@ -30,6 +30,9 @@ const (
 	// maxInsertStrings is the maximum number of parameters supported in a
 	// Windows event message.
 	maxInsertStrings = 99
+
+	leftTemplateDelim  = "[{{"
+	rightTemplateDelim = "}}]"
 )
 
 // templateInserts contains EvtVariant values that can be used to substitute
@@ -45,7 +48,7 @@ type stringInserts struct {
 	evtVariants   [maxInsertStrings]EvtVariant
 }
 
-// Pointer returns a pointer the EvtVariant array.
+// Slice returns a slice of the full EvtVariant array.
 func (si *stringInserts) Slice() []EvtVariant {
 	return si.evtVariants[:]
 }
@@ -66,7 +69,8 @@ func newTemplateStringInserts() *stringInserts {
 
 	for i := 0; i < len(si.evtVariants); i++ {
 		// Use i+1 to keep our inserts numbered the same as Window's inserts.
-		strSlice, err := windows.UTF16FromString(`{{eventParam $ ` + strconv.Itoa(i+1) + `}}`)
+		templateParam := leftTemplateDelim + `eventParam $ ` + strconv.Itoa(i+1) + rightTemplateDelim
+		strSlice, err := windows.UTF16FromString(templateParam)
 		if err != nil {
 			// This will never happen.
 			panic(err)

winlogbeat/sys/wineventlog/syscall_windows.go

@@ -661,3 +661,5 @@ func EvtClearLog(session EvtHandle, channelPath string, targetFilePath string) e
 //sys   _EvtNextEventMetadata(enumerator EvtHandle, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtNextEventMetadata
 //sys   _EvtGetObjectArrayProperty(objectArray EvtObjectArrayPropertyHandle, propertyID EvtPublisherMetadataPropertyID, arrayIndex uint32, flags uint32, bufferSize uint32, evtVariant *EvtVariant, bufferUsed *uint32) (err error) = wevtapi.EvtGetObjectArrayProperty
 //sys   _EvtGetObjectArraySize(objectArray EvtObjectArrayPropertyHandle, arraySize *uint32) (err error) = wevtapi.EvtGetObjectArraySize
+//sys   _EvtOpenPublisherEnum(session EvtHandle, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtOpenPublisherEnum
+//sys   _EvtNextPublisherId(enumerator EvtHandle, bufferSize uint32, buffer *byte, bufferUsed *uint32) (err error) = wevtapi.EvtNextPublisherId

winlogbeat/sys/wineventlog/template.go

@@ -22,12 +22,13 @@ import (
 )
 
 type eventTemplate struct {
-	Data []eventData `xml:"data"`
+	Data []EventData `xml:"data"`
 }
 
-type eventData struct {
-	Name string `xml:"name,attr"`
-	Type string `xml:"outType,attr"`
+type EventData struct {
+	Name    string `xml:"name,attr"`
+	InType  string `xml:"inType,attr"`
+	OutType string `xml:"outType,attr"`
 }
 
 func (t *eventTemplate) Unmarshal(xmlData []byte) error {

winlogbeat/sys/wineventlog/wineventlog_windows.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"reflect"
 	"runtime"
+	"sort"
 	"syscall"
 
 	"github.com/elastic/beats/v7/libbeat/common"
@@ -419,6 +420,46 @@ func FormatEventString(
 	return common.UTF16ToUTF8Bytes(buffer[:bufferUsed], out)
 }
 
+// Publishers returns a sort list of event publishers on the local computer.
+func Publishers() ([]string, error) {
+	publisherEnumerator, err := _EvtOpenPublisherEnum(NilHandle, 0)
+	if err != nil {
+		return nil, fmt.Errorf("failed in EvtOpenPublisherEnum: %w", err)
+	}
+	defer Close(publisherEnumerator)
+
+	var (
+		publishers []string
+		bufferUsed uint32
+		buffer     = make([]byte, 1024)
+	)
+
+loop:
+	for {
+		if err = _EvtNextPublisherId(publisherEnumerator, uint32(len(buffer)), &buffer[0], &bufferUsed); err != nil {
+			switch err {
+			case ERROR_NO_MORE_ITEMS:
+				break loop
+			case ERROR_INSUFFICIENT_BUFFER:
+				buffer = make([]byte, 2*bufferUsed)
+				continue loop
+			default:
+				return nil, fmt.Errorf("failed in EvtNextPublisherId: %w", err)
+			}
+		}
+
+		provider, err := sys.UTF16BytesToString(buffer)
+		if err != nil {
+			return nil, err
+		}
+
+		publishers = append(publishers, provider)
+	}
+
+	sort.Strings(publishers)
+	return publishers, nil
+}
+
 // offset reads a pointer value from the reader then calculates an offset from
 // the start of the buffer to the pointer location. If the pointer value is
 // NULL or is outside of the bounds of the buffer then an error is returned.

winlogbeat/sys/wineventlog/wineventlog_windows_test.go

@@ -108,3 +108,15 @@ func TestChannels(t *testing.T) {
 		}
 	}
 }
+
+func TestPublishers(t *testing.T) {
+	publishers, err := Publishers()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.NotEmpty(t, publishers)
+	for _, p := range publishers {
+		t.Log(p)
+	}
+}