Skip to content

Commit

Permalink
[pkg/stanza][receiver/windowseventlog] Improve EventDataType support …
Browse files Browse the repository at this point in the history
…in EventXML (Windows) (open-telemetry#28587)

**Description:**
The [XML schema for Windows events supports `Data` elements without the
`Name`
attribute](https://learn.microsoft.com/en-us/windows/win32/wes/eventschema-datafieldtype-complextype),
however, the current implementation doesn't capture `Data` elements
without the `Name` attribute.

Capturing such elements is specially important for events for which the
publisher metadata is invalid. These elements contain the data that will
give a user a much better chance of actually understanding the event,
see
[here](open-telemetry#21491 (comment))
for an example.

I'm adding also the optional `Binary` element. Although this element
typically requires knowledge of the actual data type it is representing
sometimes it can be useful together with the data elements.

I consider this to be a breaking change because it modifies the layout
of the event generated by the package. It isn't an addition, the old
representation is changed, please refer to the changes in tests to see
the difference.

**Link to tracking Issue:**
This is the last pending item to fix open-telemetry#24493, open-telemetry#21491 ([item
5](open-telemetry#21491)).

**Testing:**
- Local run of the affected receiver and package
- "Run Windows" on my fork

**Documentation:**
N/A

---------

Co-authored-by: Daniel Jaglowski <jaglows3@gmail.com>
  • Loading branch information
2 people authored and jmsnll committed Nov 12, 2023
1 parent 8d9f25d commit 992b1ea
Show file tree
Hide file tree
Showing 5 changed files with 201 additions and 55 deletions.
27 changes: 27 additions & 0 deletions .chloggen/improve-event-data-support-for-windows-event-xml.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Use this changelog template to create an entry for release notes.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: breaking

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: pkg/stanza

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Improve parsing of Windows Event XML by handling anonymous `Data` elements.

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [21491]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext: This improves the contents of Windows log events for which the publisher manifest is unavailable. Previously, anonymous `Data` elements were ignored. This is a breaking change for users who were relying on the previous data format.

# If your change doesn't affect end users or the exported elements of any package,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
# Optional: The change log or logs in which this entry should be included.
# e.g. '[user]' or '[user, api]'
# Include 'user' if the change is relevant to end users.
# Include 'api' if there is a change to a library API.
# Default: '[user]'
change_logs: [user]
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<Event xmlns="http://schemas.microsoft.com/win/2004/08/events/event">
<System>
<Provider Name="VSS" />
<EventID Qualifiers="0">8194</EventID>
<Version>0</Version>
<Level>2</Level>
<Task>0</Task>
<Opcode>0</Opcode>
<Keywords>0x80000000000000</Keywords>
<TimeCreated SystemTime="2023-10-19T21:57:58.0685414Z" />
<EventRecordID>383972</EventRecordID>
<Correlation />
<Execution ProcessID="0" ThreadID="0" />
<Channel>Application</Channel>
<Computer>computer</Computer>
<Security />
</System>
<EventData>
<Data>1st_value</Data>
<Data>2nd_value</Data>
<Binary>2D20</Binary>
</EventData>
</Event>
74 changes: 47 additions & 27 deletions pkg/stanza/operator/input/windows/xml.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,24 @@ import (

// EventXML is the rendered xml of an event.
type EventXML struct {
EventID EventID `xml:"System>EventID"`
Provider Provider `xml:"System>Provider"`
Computer string `xml:"System>Computer"`
Channel string `xml:"System>Channel"`
RecordID uint64 `xml:"System>EventRecordID"`
TimeCreated TimeCreated `xml:"System>TimeCreated"`
Message string `xml:"RenderingInfo>Message"`
RenderedLevel string `xml:"RenderingInfo>Level"`
Level string `xml:"System>Level"`
RenderedTask string `xml:"RenderingInfo>Task"`
Task string `xml:"System>Task"`
RenderedOpcode string `xml:"RenderingInfo>Opcode"`
Opcode string `xml:"System>Opcode"`
RenderedKeywords []string `xml:"RenderingInfo>Keywords>Keyword"`
Keywords []string `xml:"System>Keywords"`
Security *Security `xml:"System>Security"`
Execution *Execution `xml:"System>Execution"`
EventData []EventDataEntry `xml:"EventData>Data"`
EventID EventID `xml:"System>EventID"`
Provider Provider `xml:"System>Provider"`
Computer string `xml:"System>Computer"`
Channel string `xml:"System>Channel"`
RecordID uint64 `xml:"System>EventRecordID"`
TimeCreated TimeCreated `xml:"System>TimeCreated"`
Message string `xml:"RenderingInfo>Message"`
RenderedLevel string `xml:"RenderingInfo>Level"`
Level string `xml:"System>Level"`
RenderedTask string `xml:"RenderingInfo>Task"`
Task string `xml:"System>Task"`
RenderedOpcode string `xml:"RenderingInfo>Opcode"`
Opcode string `xml:"System>Opcode"`
RenderedKeywords []string `xml:"RenderingInfo>Keywords>Keyword"`
Keywords []string `xml:"System>Keywords"`
Security *Security `xml:"System>Security"`
Execution *Execution `xml:"System>Execution"`
EventData EventData `xml:"EventData"`
}

// parseTimestamp will parse the timestamp of the event.
Expand Down Expand Up @@ -148,19 +148,30 @@ func (e *EventXML) parseMessage() (string, map[string]interface{}) {
}
}

// parse event data entries into a map[string]interface
// where the key is the Name attribute, and value is the element value
// entries without Name are ignored
// parse event data into a map[string]interface
// see: https://learn.microsoft.com/en-us/windows/win32/wes/eventschema-datafieldtype-complextype
func parseEventData(entries []EventDataEntry) map[string]interface{} {
outputMap := make(map[string]interface{}, len(entries))
func parseEventData(eventData EventData) map[string]interface{} {
outputMap := make(map[string]interface{}, 3)
if eventData.Name != "" {
outputMap["name"] = eventData.Name
}
if eventData.Binary != "" {
outputMap["binary"] = eventData.Binary
}

if len(eventData.Data) == 0 {
return outputMap
}

for _, entry := range entries {
if entry.Name != "" {
outputMap[entry.Name] = entry.Value
dataMaps := make([]interface{}, len(eventData.Data))
for i, data := range eventData.Data {
dataMaps[i] = map[string]interface{}{
data.Name: data.Value,
}
}

outputMap["data"] = dataMaps

return outputMap
}

Expand Down Expand Up @@ -191,7 +202,16 @@ type Provider struct {
EventSourceName string `xml:"EventSourceName,attr"`
}

type EventDataEntry struct {
type EventData struct {
// https://learn.microsoft.com/en-us/windows/win32/wes/eventschema-eventdatatype-complextype
// ComplexData is not supported.
Name string `xml:"Name,attr"`
Data []Data `xml:"Data"`
Binary string `xml:"Binary"`
}

type Data struct {
// https://learn.microsoft.com/en-us/windows/win32/wes/eventschema-datafieldtype-complextype
Name string `xml:"Name,attr"`
Value string `xml:",chardata"`
}
Expand Down
128 changes: 101 additions & 27 deletions pkg/stanza/operator/input/windows/xml_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,8 @@ func TestParseBody(t *testing.T) {
Task: "task",
Opcode: "opcode",
Keywords: []string{"keyword"},
EventData: []EventDataEntry{
{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"},
EventData: EventData{
Data: []Data{{Name: "1st_name", Value: "value"}, {Name: "2nd_name", Value: "another_value"}},
},
RenderedLevel: "rendered_level",
RenderedTask: "rendered_task",
Expand All @@ -110,7 +110,12 @@ func TestParseBody(t *testing.T) {
"task": "rendered_task",
"opcode": "rendered_opcode",
"keywords": []string{"RenderedKeywords"},
"event_data": map[string]interface{}{"name": "value", "another_name": "another_value"},
"event_data": map[string]interface{}{
"data": []interface{}{
map[string]interface{}{"1st_name": "value"},
map[string]interface{}{"2nd_name": "another_value"},
},
},
}

require.Equal(t, expected, xml.parseBody())
Expand Down Expand Up @@ -138,8 +143,8 @@ func TestParseBodySecurityExecution(t *testing.T) {
Task: "task",
Opcode: "opcode",
Keywords: []string{"keyword"},
EventData: []EventDataEntry{
{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"},
EventData: EventData{
Data: []Data{{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"}},
},
Execution: &Execution{
ProcessID: 13,
Expand Down Expand Up @@ -180,7 +185,12 @@ func TestParseBodySecurityExecution(t *testing.T) {
"security": map[string]any{
"user_id": "my-user-id",
},
"event_data": map[string]interface{}{"name": "value", "another_name": "another_value"},
"event_data": map[string]interface{}{
"data": []interface{}{
map[string]interface{}{"name": "value"},
map[string]interface{}{"another_name": "another_value"},
},
},
}

require.Equal(t, expected, xml.parseBody())
Expand Down Expand Up @@ -214,8 +224,8 @@ func TestParseBodyFullExecution(t *testing.T) {
Task: "task",
Opcode: "opcode",
Keywords: []string{"keyword"},
EventData: []EventDataEntry{
{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"},
EventData: EventData{
Data: []Data{{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"}},
},
Execution: &Execution{
ProcessID: 13,
Expand Down Expand Up @@ -266,7 +276,12 @@ func TestParseBodyFullExecution(t *testing.T) {
"security": map[string]any{
"user_id": "my-user-id",
},
"event_data": map[string]interface{}{"name": "value", "another_name": "another_value"},
"event_data": map[string]interface{}{
"data": []interface{}{
map[string]interface{}{"name": "value"},
map[string]interface{}{"another_name": "another_value"},
},
},
}

require.Equal(t, expected, xml.parseBody())
Expand Down Expand Up @@ -294,8 +309,8 @@ func TestParseNoRendered(t *testing.T) {
Task: "task",
Opcode: "opcode",
Keywords: []string{"keyword"},
EventData: []EventDataEntry{
{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"},
EventData: EventData{
Data: []Data{{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"}},
},
}

Expand All @@ -318,7 +333,12 @@ func TestParseNoRendered(t *testing.T) {
"task": "task",
"opcode": "opcode",
"keywords": []string{"keyword"},
"event_data": map[string]interface{}{"name": "value", "another_name": "another_value"},
"event_data": map[string]interface{}{
"data": []interface{}{
map[string]interface{}{"name": "value"},
map[string]interface{}{"another_name": "another_value"},
},
},
}

require.Equal(t, expected, xml.parseBody())
Expand Down Expand Up @@ -346,8 +366,8 @@ func TestParseBodySecurity(t *testing.T) {
Task: "task",
Opcode: "opcode",
Keywords: []string{"keyword"},
EventData: []EventDataEntry{
{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"},
EventData: EventData{
Data: []Data{{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"}},
},
RenderedLevel: "rendered_level",
RenderedTask: "rendered_task",
Expand All @@ -374,32 +394,48 @@ func TestParseBodySecurity(t *testing.T) {
"task": "rendered_task",
"opcode": "rendered_opcode",
"keywords": []string{"RenderedKeywords"},
"event_data": map[string]interface{}{"name": "value", "another_name": "another_value"},
"event_data": map[string]interface{}{
"data": []interface{}{
map[string]interface{}{"name": "value"},
map[string]interface{}{"another_name": "another_value"},
},
},
}

require.Equal(t, expected, xml.parseBody())
}

func TestParseEventData(t *testing.T) {
xmlMap := EventXML{
EventData: []EventDataEntry{
{Name: "name", Value: "value"},
EventData: EventData{
Name: "EVENT_DATA",
Data: []Data{{Name: "name", Value: "value"}},
Binary: "2D20",
},
}

parsed := xmlMap.parseBody()
expectedMap := map[string]interface{}{"name": "value"}
expectedMap := map[string]interface{}{
"name": "EVENT_DATA",
"data": []interface{}{
map[string]interface{}{"name": "value"},
},
"binary": "2D20",
}
require.Equal(t, expectedMap, parsed["event_data"])

xmlMixed := EventXML{
EventData: []EventDataEntry{
{Name: "name", Value: "value"},
{Value: "noname"},
EventData: EventData{
Data: []Data{{Name: "name", Value: "value"}, {Value: "no_name"}},
},
}

parsed = xmlMixed.parseBody()
expectedSlice := map[string]interface{}{"name": "value"}
expectedSlice := map[string]interface{}{
"data": []interface{}{
map[string]interface{}{"name": "value"},
map[string]interface{}{"": "no_name"}},
}
require.Equal(t, expectedSlice, parsed["event_data"])
}

Expand Down Expand Up @@ -428,16 +464,54 @@ func TestUnmarshalWithEventData(t *testing.T) {
TimeCreated: TimeCreated{
SystemTime: "2022-04-22T10:20:52.3778625Z",
},
Computer: "computer",
Channel: "Application",
RecordID: 23401,
Level: "4",
Message: "",
Task: "0",
Opcode: "0",
Execution: &Execution{},
Security: &Security{},
EventData: EventData{
Data: []Data{
{Name: "Time", Value: "2022-04-28T19:48:52Z"},
{Name: "Source", Value: "RulesEngine"}},
},
Keywords: []string{"0x80000000000000"},
}

require.Equal(t, xml, event)
}

func TestUnmarshalWithAnonymousEventDataEntries(t *testing.T) {
data, err := os.ReadFile(filepath.Join("testdata", "xmlWithAnonymousEventDataEntries.xml"))
require.NoError(t, err)

event, err := unmarshalEventXML(data)
require.NoError(t, err)

xml := EventXML{
EventID: EventID{
ID: 8194,
Qualifiers: 0,
},
Provider: Provider{
Name: "VSS",
},
TimeCreated: TimeCreated{
SystemTime: "2023-10-19T21:57:58.0685414Z",
},
Computer: "computer",
Channel: "Application",
RecordID: 23401,
Level: "4",
RecordID: 383972,
Level: "2",
Message: "",
Task: "0",
Opcode: "0",
EventData: []EventDataEntry{
{Name: "Time", Value: "2022-04-28T19:48:52Z"},
{Name: "Source", Value: "RulesEngine"},
EventData: EventData{
Data: []Data{{Name: "", Value: "1st_value"}, {Name: "", Value: "2nd_value"}},
Binary: "2D20",
},
Keywords: []string{"0x80000000000000"},
Security: &Security{},
Expand Down
4 changes: 3 additions & 1 deletion receiver/windowseventlogreceiver/receiver_windows_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ func TestReadWindowsEventLogger(t *testing.T) {
eventData := body["event_data"]
eventDataMap, ok := eventData.(map[string]interface{})
require.True(t, ok)
require.Equal(t, map[string]interface{}{}, eventDataMap)
require.Equal(t, map[string]interface{}{
"data": []interface{}{map[string]interface{}{"": "Test log"}},
}, eventDataMap)

eventID := body["event_id"]
require.NotNil(t, eventID)
Expand Down

0 comments on commit 992b1ea

Please sign in to comment.