diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 2ee8cf55a5e..00aeb800234 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -31,6 +31,8 @@ https://github.com/elastic/beats/compare/v1.1.0...master[Check the HEAD diff] - Scalar values in used in the `fields` configuration setting are no longer automatically converted to strings. {pull}1092[1092] *Winlogbeat* +- The `message_inserts` field was replaced with the `event_data` field {issue}1053[1053] +- The `category` field was renamed to `task` to better align with the Windows Event Log API naming {issue}1053[1053] ==== Bugfixes @@ -96,6 +98,9 @@ https://github.com/elastic/beats/compare/v1.1.0...master[Check the HEAD diff] - Add caching of event metadata handles and the system render context for the wineventlog API {pull}888[888] - Improve config validation by checking for unknown top-level YAML keys. {pull}1100[1100] - Add the ability to set tags, fields, and fields_under_root as options for each event log {pull}1092[1092] +- Add additional data to the events published by Winlogbeat. The new fields are `activity_id`, +`event_data`, `keywords`, `opcode`, `process_id`, `provider_guid`, `related_activity_id`, +`task`, `thread_id`, `user_data`. and `version`. {issue}1053[1053] ==== Deprecated diff --git a/winlogbeat/beater/winlogbeat.go b/winlogbeat/beater/winlogbeat.go index a5bd509420b..b6fe8a77eef 100644 --- a/winlogbeat/beater/winlogbeat.go +++ b/winlogbeat/beater/winlogbeat.go @@ -250,9 +250,9 @@ loop: // of event if it is going to be filtered. // TODO: Add a severity filter. // TODO: Check the global IgnoreOlder filter. - if ignoreOlder != 0 && time.Since(lr.TimeGenerated) > ignoreOlder { - detailf("EventLog[%s] ignore_older filter dropping event: %s", - api.Name(), lr.String()) + if ignoreOlder != 0 && time.Since(lr.TimeCreated.SystemTime) > ignoreOlder { + detailf("EventLog[%s] ignore_older filter dropping event: %+v", + api.Name(), lr) ignoredEvents.Add("total", 1) ignoredEvents.Add(api.Name(), 1) continue @@ -276,8 +276,8 @@ loop: } eb.checkpoint.Persist(api.Name(), - records[len(records)-1].RecordNumber, - records[len(records)-1].TimeGenerated.UTC()) + records[len(records)-1].RecordID, + records[len(records)-1].TimeCreated.SystemTime.UTC()) } } diff --git a/winlogbeat/config/config.go b/winlogbeat/config/config.go index 6259cf48750..46110ff0018 100644 --- a/winlogbeat/config/config.go +++ b/winlogbeat/config/config.go @@ -41,7 +41,7 @@ func (s Settings) Validate() error { // Check for invalid top-level keys. var errs multierror.Errors - for k, _ := range s.All { + for k := range s.All { k = strings.ToLower(k) i := sort.SearchStrings(validKeys, k) if i >= len(validKeys) || validKeys[i] != k { diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc index 7a5f90d584c..6f68c3eea9f 100644 --- a/winlogbeat/docs/fields.asciidoc +++ b/winlogbeat/docs/fields.asciidoc @@ -66,6 +66,15 @@ Contains data from a Windows event log record. +==== activity_id + +type: string + +required: False + +A globally unique identifier that identifies the current activity. The events that are published with this identifier are part of the same activity. + + ==== computer_name type: string @@ -75,13 +84,13 @@ required: True The name of the computer that generated the record. When using Windows event forwarding, this name can differ from the `beat.hostname`. -==== category +==== event_data -type: string +type: dictionary required: False -The category for this event. The meaning of this value depends on the source of the event. +The event specific data. This field is mutually exclusive with `user_data`. ==== event_id @@ -93,6 +102,15 @@ required: True The event identifier. The value is specific to the source of the event. +==== keywords + +type: string[] + +required: False + +The keywords are used to classify an event. + + ==== log_name type: string @@ -106,7 +124,7 @@ The name of the event log from which this record was read. This value is one of type: string -required: True +required: False The level of the event. There are five levels of events that can be logged: Success, Information, Warning, Error, Audit Success, and Audit Failure. @@ -126,25 +144,52 @@ type: string required: False -The error that occurred while reading and formatting the message from the log. This field is mutually exclusive with `message`. +The error that occurred while reading and formatting the message from the log. -==== message_inserts +==== record_number -type: list +type: string + +required: True + +The record number of the event log record. The first record written to an event log is record number 1, and other records are numbered sequentially. If the record number reaches the maximum value (2^32^ for the Event Logging API and 2^64^ for the Windows Event Log API), the next record number will be 0. + + +==== related_activity_id + +type: string required: False -The raw message data logged by an application. Normally this data is inserted into a parameterized string to create `message`, but in case of an error, Winlogbeat attempts to provide this raw data. This field is mutually exclusive with `message`. +A globally unique identifier that identifies the activity to which control was transferred to. The related events would then have this identifier as their `activity_id` identifier. -==== record_number +==== opcode type: string -required: True +required: False -The record number of the event log record. The first record written to an event log is record number 1, and other records are numbered sequentially. If the record number reaches the maximum value (2^32^ for the Event Logging API and 2^64^ for the Windows Event Log API), the next record number will be 0. +The opcode defined in the event. Task and opcode are typically used to identify the location in the application from where the event was logged. + + +==== provider_guid + +type: string + +required: False + +A globally unique identifier that identifies the provider that logged the event. + + +==== process_id + +type: long + +required: False + +The process_id identifies the process that generated the event. ==== source_name @@ -156,6 +201,33 @@ required: True The source of the event log record (the application or service that logged the record). +==== task + +type: string + +required: False + +The task defined in the event. Task and opcode are typically used to identify the location in the application from where the event was logged. The category used by the Event Logging API (on pre Windows Vista operating systems) is written to this field. + + +==== thread_id + +type: long + +required: False + +The thread_id identifies the thread that generated the event. + + +==== user_data + +type: dictionary + +required: False + +The event specific data. This field is mutually exclusive with `event_data`. + + ==== user.identifier type: string @@ -196,3 +268,11 @@ required: False The type of account associated with this event. +==== version + +type: int + +required: False + +The version number of the event's definition. + diff --git a/winlogbeat/etc/fields.yml b/winlogbeat/etc/fields.yml index 5afdbac79dc..2ff4d2cb6dd 100644 --- a/winlogbeat/etc/fields.yml +++ b/winlogbeat/etc/fields.yml @@ -56,6 +56,14 @@ eventlog: description: > Contains data from a Windows event log record. fields: + - name: activity_id + type: string + required: false + description: > + A globally unique identifier that identifies the current activity. The + events that are published with this identifier are part of the same + activity. + - name: computer_name type: string required: true @@ -63,12 +71,12 @@ eventlog: The name of the computer that generated the record. When using Windows event forwarding, this name can differ from the `beat.hostname`. - - name: category - type: string + - name: event_data + type: dictionary required: false description: > - The category for this event. The meaning of this value depends on the - source of the event. + The event specific data. This field is mutually exclusive with + `user_data`. - name: event_id type: long @@ -76,19 +84,26 @@ eventlog: description: > The event identifier. The value is specific to the source of the event. + - name: keywords + type: string[] + required: false + description: > + The keywords are used to classify an event. + - name: log_name type: string required: true description: > - The name of the event log from which this record was read. This value is one - of the names from the `event_logs` collection in the configuration. + The name of the event log from which this record was read. This value is + one of the names from the `event_logs` collection in the configuration. - name: level type: string - required: true + required: false description: > - The level of the event. There are five levels of events that can be logged: - Success, Information, Warning, Error, Audit Success, and Audit Failure. + The level of the event. There are five levels of events that can be + logged: Success, Information, Warning, Error, Audit Success, and Audit + Failure. - name: message type: string @@ -104,16 +119,7 @@ eventlog: required: false description: > The error that occurred while reading and formatting the message from - the log. This field is mutually exclusive with `message`. - - - name: message_inserts - type: list - required: false - description: > - The raw message data logged by an application. Normally this data is - inserted into a parameterized string to create `message`, but in case of - an error, Winlogbeat attempts to provide this raw data. This field is - mutually exclusive with `message`. + the log. - name: record_number type: string @@ -125,6 +131,35 @@ eventlog: for the Event Logging API and 2^64^ for the Windows Event Log API), the next record number will be 0. + - name: related_activity_id + type: string + required: false + description: > + A globally unique identifier that identifies the activity to which + control was transferred to. The related events would then have this + identifier as their `activity_id` identifier. + + - name: opcode + type: string + required: false + description: > + The opcode defined in the event. Task and opcode are typically used to + identify the location in the application from where the event was + logged. + + - name: provider_guid + type: string + required: false + description: > + A globally unique identifier that identifies the provider that logged + the event. + + - name: process_id + type: long + required: false + description: > + The process_id identifies the process that generated the event. + - name: source_name type: string required: true @@ -132,6 +167,28 @@ eventlog: The source of the event log record (the application or service that logged the record). + - name: task + type: string + required: false + description: > + The task defined in the event. Task and opcode are typically used to + identify the location in the application from where the event was + logged. The category used by the Event Logging API (on pre Windows Vista + operating systems) is written to this field. + + - name: thread_id + type: long + required: false + description: > + The thread_id identifies the thread that generated the event. + + - name: user_data + type: dictionary + required: false + description: > + The event specific data. This field is mutually exclusive with + `event_data`. + - name: user.identifier type: string required: false @@ -164,6 +221,11 @@ eventlog: description: > The type of account associated with this event. + - name: version + type: int + required: false + description: The version number of the event's definition. + sections: - ["common", "Common Beat"] - ["eventlog", "Event Log Record"] diff --git a/winlogbeat/etc/winlogbeat.template.json b/winlogbeat/etc/winlogbeat.template.json index bcddd61ea21..b3e419feae5 100644 --- a/winlogbeat/etc/winlogbeat.template.json +++ b/winlogbeat/etc/winlogbeat.template.json @@ -34,6 +34,14 @@ "enabled": false }, "type": "string" + }, + "process_id": { + "doc_values": "true", + "type": "long" + }, + "thread_id": { + "doc_values": "true", + "type": "long" } } } diff --git a/winlogbeat/eventlog/cache.go b/winlogbeat/eventlog/cache.go index 3607f580f08..6236f9bc009 100644 --- a/winlogbeat/eventlog/cache.go +++ b/winlogbeat/eventlog/cache.go @@ -9,7 +9,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/winlogbeat/sys/eventlogging" + "github.com/elastic/beats/winlogbeat/sys" ) // Stats for the message file caches. @@ -26,7 +26,7 @@ const ( // Function type for loading event message files associated with the given // event log and source name. -type messageFileLoaderFunc func(eventLogName, sourceName string) eventlogging.MessageFiles +type messageFileLoaderFunc func(eventLogName, sourceName string) sys.MessageFiles // Function type for freeing Handles. type freeHandleFunc func(handle uintptr) error @@ -75,7 +75,7 @@ func newMessageFilesCache(eventLogName string, loader messageFileLoaderFunc, // If no item is cached, then one is loaded, stored, and returned. // Callers should check the MessageFiles.Err value to see if an error occurred // while loading the message files. -func (hc *messageFilesCache) get(sourceName string) eventlogging.MessageFiles { +func (hc *messageFilesCache) get(sourceName string) sys.MessageFiles { v := hc.cache.Get(sourceName) if v == nil { hc.miss() @@ -89,11 +89,11 @@ func (hc *messageFilesCache) get(sourceName string) eventlogging.MessageFiles { existing := hc.cache.PutIfAbsent(sourceName, v) if existing != nil { // A value was already loaded, so free the handles we just created. - messageFiles, _ := v.(eventlogging.MessageFiles) + messageFiles, _ := v.(sys.MessageFiles) hc.freeHandles(messageFiles) // Return the existing cached value. - messageFiles, _ = existing.(eventlogging.MessageFiles) + messageFiles, _ = existing.(sys.MessageFiles) return messageFiles } hc.size() @@ -101,7 +101,7 @@ func (hc *messageFilesCache) get(sourceName string) eventlogging.MessageFiles { hc.hit() } - messageFiles, _ := v.(eventlogging.MessageFiles) + messageFiles, _ := v.(sys.MessageFiles) return messageFiles } @@ -111,7 +111,7 @@ func (hc *messageFilesCache) evictionHandler(k common.Key, v common.Value) { // Update the size on a different goroutine after the callback completes. defer func() { go hc.size() }() - messageFiles, ok := v.(eventlogging.MessageFiles) + messageFiles, ok := v.(sys.MessageFiles) if !ok { return } @@ -123,7 +123,7 @@ func (hc *messageFilesCache) evictionHandler(k common.Key, v common.Value) { // freeHandles free the event message file Handles so that the modules can // be unloaded. The Handles are no longer valid after being freed. -func (hc *messageFilesCache) freeHandles(mf eventlogging.MessageFiles) { +func (hc *messageFilesCache) freeHandles(mf sys.MessageFiles) { for _, fh := range mf.Handles { err := hc.freer(fh.Handle) if err != nil { diff --git a/winlogbeat/eventlog/eventlog.go b/winlogbeat/eventlog/eventlog.go index ca9d80fd117..83efaeb45c0 100644 --- a/winlogbeat/eventlog/eventlog.go +++ b/winlogbeat/eventlog/eventlog.go @@ -2,18 +2,24 @@ package eventlog import ( "fmt" + "reflect" "strconv" - "strings" - "time" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/winlogbeat/sys" +) + +// Debug selectors used in this package. +const ( + debugSelector = "eventlog" + detailSelector = "eventlog_detail" ) // Debug logging functions for this package. var ( - debugf = logp.MakeDebug("eventlog") - detailf = logp.MakeDebug("eventlog_detail") + debugf = logp.MakeDebug(debugSelector) + detailf = logp.MakeDebug(detailSelector) ) // EventLog is an interface to a Windows Event Log. @@ -35,100 +41,133 @@ type EventLog interface { // Record represents a single event from the log. type Record struct { - API string // The event log API type used to read the record. - - EventLogName string // The name of the event log from which this record was read. - SourceName string // The source of the event log record (the application or service that logged the record). - ComputerName string // The name of the computer that generated the record. - RecordNumber uint64 // The record number of the event log record. - EventID uint32 // The event identifier. The value is specific to the source of the event. - Level string // The level or severity of the event. - Category string // The category for this event. The meaning of this value depends on the event source. - TimeGenerated time.Time // The timestamp when the record was generated. - User *User // The user that logged the record. - - Message string // The message from the event log. - MessageInserts []string // The raw message data logged by an application. - MessageErr error // The error that occurred while reading and formatting the message from the event log. - - common.EventMetadata // Fields and tags to add to the event. -} - -// String returns a string representation of Record. -func (r Record) String() string { - return fmt.Sprintf("Record API[%s] EventLogName[%s] SourceName[%s] "+ - "ComputerName[%s] RecordNumber[%d] EventID[%d] Level[%s] "+ - "Category[%s] TimeGenerated[%s] User[%s] "+ - "Message[%s] MessageInserts[%s] MessageErr[%s]", r.API, - r.EventLogName, r.SourceName, r.ComputerName, r.RecordNumber, - r.EventID, r.Level, r.Category, r.TimeGenerated, r.User, - r.Message, strings.Join(r.MessageInserts, ", "), r.MessageErr) + API string // The event log API type used to read the record. + common.EventMetadata // Fields and tags to add to the event. + sys.Event } // ToMapStr returns a new MapStr containing the data from this Record. -func (r Record) ToMapStr() common.MapStr { +func (e Record) ToMapStr() common.MapStr { m := common.MapStr{ - common.EventMetadataKey: r.EventMetadata, - "@timestamp": common.Time(r.TimeGenerated), - "log_name": r.EventLogName, - "source_name": r.SourceName, - "computer_name": r.ComputerName, - // Use a string to represent this uint64 data because its value can - // be outside the range represented by a Java long. - "record_number": strconv.FormatUint(r.RecordNumber, 10), - "event_id": r.EventID, - "level": r.Level, - "type": r.API, - "count": 1, + "type": e.API, + common.EventMetadataKey: e.EventMetadata, + "count": 1, + "@timestamp": common.Time(e.TimeCreated.SystemTime), + "log_name": e.Channel, + "source_name": e.Provider.Name, + "computer_name": e.Computer, + "record_number": strconv.FormatUint(e.RecordID, 10), + "event_id": e.EventIdentifier.ID, } - if r.Message != "" { - m["message"] = r.Message - } else { - if len(r.MessageInserts) > 0 { - m["message_inserts"] = r.MessageInserts + addOptional(m, "provider_guid", e.Provider.GUID) + addOptional(m, "version", e.Version) + addOptional(m, "level", e.Level) + addOptional(m, "task", e.Task) + addOptional(m, "opcode", e.Opcode) + addOptional(m, "keywords", e.Keywords) + addOptional(m, "message", sys.RemoveWindowsLineEndings(e.Message)) + addOptional(m, "message_error", e.RenderErr) + + // Correlation + addOptional(m, "activity_id", e.Correlation.ActivityID) + addOptional(m, "related_activity_id", e.Correlation.RelatedActivityID) + + // Execution + addOptional(m, "process_id", e.Execution.ProcessID) + addOptional(m, "thread_id", e.Execution.ThreadID) + addOptional(m, "processor_id", e.Execution.ProcessorID) + addOptional(m, "session_id", e.Execution.SessionID) + addOptional(m, "kernel_time", e.Execution.KernelTime) + addOptional(m, "user_time", e.Execution.UserTime) + addOptional(m, "processor_time", e.Execution.ProcessorTime) + + if e.User.Identifier != "" { + user := common.MapStr{ + "identifier": e.User.Identifier, } + m["user"] = user - if r.MessageErr != nil { - m["message_error"] = r.MessageErr.Error() - } + addOptional(user, "name", e.User.Name) + addOptional(user, "domain", e.User.Domain) + addOptional(user, "type", e.User.Type.String()) + } + + addPairs(m, "event_data", e.EventData.Pairs) + userData := addPairs(m, "user_data", e.UserData.Pairs) + addOptional(userData, "xml_name", e.UserData.Name.Local) + + return m +} + +// addOptional adds a key and value to the given MapStr if the value is not the +// zero value for the type of v. It is safe to call the function with a nil +// MapStr. +func addOptional(m common.MapStr, key string, v interface{}) { + if m != nil && !isZero(v) { + m[key] = v } +} - if r.Category != "" { - m["category"] = r.Category +// addPairs adds a new dictionary to the given MapStr. The key/value pairs are +// added to the new dictionary. If any keys are duplicates, the first key/value +// pair is added and the remaining duplicates are dropped. +// +// The new dictionary is added to the given MapStr and it is also returned for +// convenience purposes. +func addPairs(m common.MapStr, key string, pairs []sys.KeyValue) common.MapStr { + if len(pairs) == 0 { + return nil } - if r.User != nil { - user := common.MapStr{ - "identifier": r.User.Identifier, + h := make(common.MapStr, len(pairs)) + for i, kv := range pairs { + // Ignore empty values. + if kv.Value == "" { + continue } - m["user"] = user - // Optional fields. - if r.User.Name != "" { - user["name"] = r.User.Name - } - if r.User.Domain != "" { - user["domain"] = r.User.Domain + // If the key name is empty or if it the default of "Data" then + // assign a generic name of paramN. + k := kv.Key + if k == "" || k == "Data" { + k = fmt.Sprintf("param%d", i+1) } - if r.User.Type != "" { - user["type"] = r.User.Type + + // Do not overwrite. + _, exists := h[k] + if !exists { + h[k] = sys.RemoveWindowsLineEndings(kv.Value) + } else { + debugf("Droping key/value (k=%s, v=%s) pair because key already "+ + "exists. event=%+v", k, kv.Value, m) } } - return m -} + if len(h) == 0 { + return nil + } -// User contains information about a Windows account. -type User struct { - Identifier string // Unique identifier used by Windows to ID the account. - Name string // User name - Domain string // Domain that the user is a member of - Type string // Type of account (e.g. User, Computer, Service) + m[key] = h + return h } -// String returns a string representation of Record. -func (u User) String() string { - return fmt.Sprintf("User Name[%s] Domain[%s] Type[%s]", - u.Name, u.Domain, u.Type) +// isZero return true if the given value is the zero value for its type. +func isZero(i interface{}) bool { + v := reflect.ValueOf(i) + switch v.Kind() { + case reflect.Array, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return v.IsNil() + } + return false } diff --git a/winlogbeat/eventlog/eventlogging.go b/winlogbeat/eventlog/eventlogging.go index 8e2c4ce3925..b4a698ccb0f 100644 --- a/winlogbeat/eventlog/eventlogging.go +++ b/winlogbeat/eventlog/eventlogging.go @@ -8,7 +8,8 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" - sys "github.com/elastic/beats/winlogbeat/sys/eventlogging" + "github.com/elastic/beats/winlogbeat/sys" + win "github.com/elastic/beats/winlogbeat/sys/eventlogging" ) const ( @@ -25,7 +26,7 @@ var _ EventLog = &eventLogging{} type eventLogging struct { uncServerPath string // UNC name of remote server. name string // Name of the log that is opened. - handle sys.Handle // Handle to the event log. + handle win.Handle // Handle to the event log. readBuf []byte // Buffer for reading in events. formatBuf []byte // Buffer for formatting messages. handles *messageFilesCache // Cached mapping of source name to event message file handles. @@ -45,12 +46,12 @@ func (l eventLogging) Name() string { func (l *eventLogging) Open(recordNumber uint64) error { detailf("%s Open(recordNumber=%d) calling OpenEventLog(uncServerPath=%s, "+ "providerName=%s)", l.logPrefix, recordNumber, l.uncServerPath, l.name) - handle, err := sys.OpenEventLog(l.uncServerPath, l.name) + handle, err := win.OpenEventLog(l.uncServerPath, l.name) if err != nil { return err } - numRecords, err := sys.GetNumberOfEventLogRecords(handle) + numRecords, err := win.GetNumberOfEventLogRecords(handle) if err != nil { return err } @@ -61,7 +62,7 @@ func (l *eventLogging) Open(recordNumber uint64) error { l.seek = true l.ignoreFirst = true - oldestRecord, err = sys.GetOldestEventLogRecord(handle) + oldestRecord, err = win.GetOldestEventLogRecord(handle) if err != nil { return err } @@ -86,9 +87,9 @@ func (l *eventLogging) Open(recordNumber uint64) error { } func (l *eventLogging) Read() ([]Record, error) { - flags := sys.EVENTLOG_SEQUENTIAL_READ | sys.EVENTLOG_FORWARDS_READ + flags := win.EVENTLOG_SEQUENTIAL_READ | win.EVENTLOG_FORWARDS_READ if l.seek { - flags = sys.EVENTLOG_SEEK_READ | sys.EVENTLOG_FORWARDS_READ + flags = win.EVENTLOG_SEEK_READ | win.EVENTLOG_FORWARDS_READ l.seek = false } @@ -98,7 +99,7 @@ func (l *eventLogging) Read() ([]Record, error) { l.readBuf = l.readBuf[0:cap(l.readBuf)] // TODO: Use number of bytes to grow the buffer size as needed. var err error - numBytesRead, err = sys.ReadEventLog( + numBytesRead, err = win.ReadEventLog( l.handle, flags, l.recordNumber, @@ -113,7 +114,7 @@ func (l *eventLogging) Read() ([]Record, error) { detailf("%s ReadEventLog read %d bytes", l.logPrefix, numBytesRead) l.readBuf = l.readBuf[0:numBytesRead] - events, _, err := sys.RenderEvents( + events, _, err := win.RenderEvents( l.readBuf[:numBytesRead], 0, l.formatBuf, l.handles.get) if err != nil { return nil, err @@ -122,42 +123,26 @@ func (l *eventLogging) Read() ([]Record, error) { records := make([]Record, 0, len(events)) for _, e := range events { - r := Record{ - API: eventLoggingAPIName, - EventLogName: l.name, - RecordNumber: uint64(e.RecordID), - EventID: e.EventID, // TODO: Subtract out high order bytes (upper 2 bytes) - Level: e.Level, - SourceName: e.SourceName, - ComputerName: e.Computer, - Category: e.Category, - Message: e.Message, - MessageInserts: e.MessageInserts, - MessageErr: e.MessageErr, - EventMetadata: l.eventMetadata, - } - - if e.TimeGenerated != nil { - r.TimeGenerated = *e.TimeGenerated - } else if e.TimeWritten != nil { - r.TimeGenerated = *e.TimeWritten - } + // The events do not contain the name of the event log so we must add + // the name of the log from which we are reading. + e.Channel = l.name - if e.UserSID != nil { - r.User = &User{ - Identifier: e.UserSID.Identifier, - Name: e.UserSID.Name, - Domain: e.UserSID.Domain, - Type: e.UserSID.Type.String(), - } + err = sys.PopulateAccount(&e.User) + if err != nil { + debugf("%s SID %s account lookup failed. %v", l.logPrefix, + e.User.Identifier, err) } - records = append(records, r) + records = append(records, Record{ + API: eventLoggingAPIName, + EventMetadata: l.eventMetadata, + Event: e, + }) } if l.ignoreFirst && len(records) > 0 { - debugf("%s Ignoring first event with record number %d", l.logPrefix, - records[0].RecordNumber) + debugf("%s Ignoring first event with record ID %d", l.logPrefix, + records[0].RecordID) records = records[1:] l.ignoreFirst = false } @@ -168,7 +153,7 @@ func (l *eventLogging) Read() ([]Record, error) { func (l *eventLogging) Close() error { debugf("%s Closing handle", l.logPrefix) - return sys.CloseEventLog(l.handle) + return win.CloseEventLog(l.handle) } // readRetryErrorHandler handles errors returned from the readEventLog function @@ -179,10 +164,10 @@ func (l *eventLogging) readRetryErrorHandler(err error) error { var reopen bool switch errno { - case sys.ERROR_EVENTLOG_FILE_CHANGED: + case win.ERROR_EVENTLOG_FILE_CHANGED: debugf("Re-opening event log because event log file was changed") reopen = true - case sys.ERROR_EVENTLOG_FILE_CORRUPT: + case win.ERROR_EVENTLOG_FILE_CORRUPT: debugf("Re-opening event log because event log file is corrupt") reopen = true } @@ -199,8 +184,8 @@ func (l *eventLogging) readRetryErrorHandler(err error) error { func readErrorHandler(err error) ([]Record, error) { switch err { case syscall.ERROR_HANDLE_EOF, - sys.ERROR_EVENTLOG_FILE_CHANGED, - sys.ERROR_EVENTLOG_FILE_CORRUPT: + win.ERROR_EVENTLOG_FILE_CHANGED, + win.ERROR_EVENTLOG_FILE_CORRUPT: return []Record{}, nil } return nil, err @@ -212,18 +197,18 @@ func newEventLogging(c Config) (EventLog, error) { return &eventLogging{ uncServerPath: c.RemoteAddress, name: c.Name, - handles: newMessageFilesCache(c.Name, sys.QueryEventMessageFiles, - sys.FreeLibrary), + handles: newMessageFilesCache(c.Name, win.QueryEventMessageFiles, + win.FreeLibrary), logPrefix: fmt.Sprintf("EventLogging[%s]", c.Name), - readBuf: make([]byte, 0, sys.MaxEventBufferSize), - formatBuf: make([]byte, sys.MaxFormatMessageBufferSize), + readBuf: make([]byte, 0, win.MaxEventBufferSize), + formatBuf: make([]byte, win.MaxFormatMessageBufferSize), eventMetadata: c.EventMetadata, }, nil } func init() { // Register eventlogging API if it is available. - available, _ := sys.IsAvailable() + available, _ := win.IsAvailable() if available { Register(eventLoggingAPIName, 1, newEventLogging, nil) } diff --git a/winlogbeat/eventlog/eventlogging_test.go b/winlogbeat/eventlog/eventlogging_test.go index 25e913932ab..ac1aa41286f 100644 --- a/winlogbeat/eventlog/eventlogging_test.go +++ b/winlogbeat/eventlog/eventlogging_test.go @@ -190,9 +190,9 @@ func TestRead(t *testing.T) { assert.Len(t, records, len(messages)) for _, record := range records { t.Log(record) - m, exists := messages[record.EventID] + m, exists := messages[record.EventIdentifier.ID] if !exists { - t.Errorf("Unknown EventId %d Read() from event log. %v", record.EventID, record) + t.Errorf("Unknown EventId %d Read() from event log. %v", record.EventIdentifier.ID, record) continue } assert.Equal(t, eventlogging.EventType(m.eventType).String(), record.Level) @@ -253,9 +253,9 @@ func TestReadUnknownEventId(t *testing.T) { if len(records) != 1 { t.FailNow() } - assert.Equal(t, eventID, records[0].EventID) - assert.Equal(t, msg, records[0].MessageInserts[0]) - assert.NotNil(t, records[0].MessageErr) + assert.Equal(t, eventID, records[0].EventIdentifier.ID) + assert.Equal(t, msg, records[0].EventData.Pairs[0].Value) + assert.NotNil(t, records[0].RenderErr) assert.Equal(t, "", records[0].Message) } @@ -310,7 +310,7 @@ func TestReadTriesMultipleEventMsgFiles(t *testing.T) { if len(records) != 1 { t.FailNow() } - assert.Equal(t, eventID, records[0].EventID) + assert.Equal(t, eventID, records[0].EventIdentifier.ID) assert.Equal(t, msg, strings.TrimRight(records[0].Message, "\r\n")) } @@ -366,7 +366,7 @@ func TestReadMultiParameterMsg(t *testing.T) { if len(records) != 1 { t.FailNow() } - assert.Equal(t, eventID, records[0].EventID) + assert.Equal(t, eventID, records[0].EventIdentifier.ID) assert.Equal(t, fmt.Sprintf(template, msgs[0], msgs[1]), strings.TrimRight(records[0].Message, "\r\n")) } @@ -435,7 +435,7 @@ func TestReadNoParameterMsg(t *testing.T) { if len(records) != 1 { t.FailNow() } - assert.Equal(t, eventID, records[0].EventID) + assert.Equal(t, eventID, records[0].EventIdentifier.ID) assert.Equal(t, template, strings.TrimRight(records[0].Message, "\r\n")) } @@ -487,7 +487,7 @@ func TestReadWhileCleared(t *testing.T) { assert.NoError(t, err, "Expected 1 message but received error") assert.Len(t, lr, 1, "Expected 1 message") if len(lr) > 0 { - assert.Equal(t, uint32(3), lr[0].EventID) + assert.Equal(t, uint32(3), lr[0].EventIdentifier.ID) } } diff --git a/winlogbeat/eventlog/wineventlog.go b/winlogbeat/eventlog/wineventlog.go index 6b9021353b0..d02a97aca44 100644 --- a/winlogbeat/eventlog/wineventlog.go +++ b/winlogbeat/eventlog/wineventlog.go @@ -7,8 +7,8 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/winlogbeat/sys/eventlogging" - sys "github.com/elastic/beats/winlogbeat/sys/wineventlog" + "github.com/elastic/beats/winlogbeat/sys" + win "github.com/elastic/beats/winlogbeat/sys/wineventlog" "golang.org/x/sys/windows" ) @@ -32,11 +32,10 @@ var _ EventLog = &winEventLog{} type winEventLog struct { remoteServer string // Name of the remote server from which to read. channelName string // Name of the channel from which to read. - subscription sys.EvtHandle // Handle to the subscription. + subscription win.EvtHandle // Handle to the subscription. maxRead int // Maximum number returned in one Read. renderBuf []byte // Buffer used for rendering event. - systemCtx sys.EvtHandle // System render context. cache *messageFilesCache // Cached mapping of source name to event message file handles. logPrefix string // String to prefix on log messages. @@ -49,11 +48,11 @@ func (l *winEventLog) Name() string { } func (l *winEventLog) Open(recordNumber uint64) error { - bookmark, err := sys.CreateBookmark(l.channelName, recordNumber) + bookmark, err := win.CreateBookmark(l.channelName, recordNumber) if err != nil { return err } - defer sys.Close(bookmark) + defer win.Close(bookmark) // Using a pull subscription to receive events. See: // https://msdn.microsoft.com/en-us/library/windows/desktop/aa385771(v=vs.85).aspx#pull @@ -62,13 +61,13 @@ func (l *winEventLog) Open(recordNumber uint64) error { return nil } - subscriptionHandle, err := sys.Subscribe( + subscriptionHandle, err := win.Subscribe( 0, // null session (used for connecting to remote event logs) signalEvent, l.channelName, "", // Query - nil means all events bookmark, // Bookmark - for resuming from a specific event - sys.EvtSubscribeStartAfterBookmark) + win.EvtSubscribeStartAfterBookmark) if err != nil { return err } @@ -78,8 +77,8 @@ func (l *winEventLog) Open(recordNumber uint64) error { } func (l *winEventLog) Read() ([]Record, error) { - handles, err := sys.EventHandles(l.subscription, l.maxRead) - if err == sys.ERROR_NO_MORE_ITEMS { + handles, err := win.EventHandles(l.subscription, l.maxRead) + if err == win.ERROR_NO_MORE_ITEMS { detailf("%s No more events", l.logPrefix) return nil, nil } @@ -89,47 +88,49 @@ func (l *winEventLog) Read() ([]Record, error) { } defer func() { for _, h := range handles { - sys.Close(h) + win.Close(h) } }() detailf("%s EventHandles returned %d handles", l.logPrefix, len(handles)) var records []Record for _, h := range handles { - e, err := sys.RenderEvent(h, l.systemCtx, 0, l.renderBuf, l.cache.get) + x, err := win.RenderEvent(h, 0, l.renderBuf, l.cache.get) + if bufErr, ok := err.(sys.InsufficientBufferError); ok { + detailf("%s Increasing render buffer size to %d", l.logPrefix, + bufErr.RequiredSize) + l.renderBuf = make([]byte, bufErr.RequiredSize) + x, err = win.RenderEvent(h, 0, l.renderBuf, l.cache.get) + } if err != nil { logp.Err("%s Dropping event with rendering error. %v", l.logPrefix, err) continue } - r := Record{ - API: winEventLogAPIName, - EventLogName: e.Channel, - SourceName: e.ProviderName, - ComputerName: e.Computer, - RecordNumber: e.RecordID, - EventID: uint32(e.EventID), - Level: e.Level, - Category: e.Task, - Message: e.Message, - MessageErr: e.MessageErr, - EventMetadata: l.eventMetadata, + e, err := sys.UnmarshalEventXML([]byte(x)) + if err != nil { + logp.Err("%s Dropping event. Failed to unmarshal XML='%s'. %v", + l.logPrefix, x, err) + continue } - if e.TimeCreated != nil { - r.TimeGenerated = *e.TimeCreated + err = sys.PopulateAccount(&e.User) + if err != nil { + debugf("%s SID %s account lookup failed. %v", l.logPrefix, + e.User.Identifier, err) } - if e.UserSID != nil { - r.User = &User{ - Identifier: e.UserSID.Identifier, - Name: e.UserSID.Name, - Domain: e.UserSID.Domain, - Type: e.UserSID.Type.String(), - } + // TODO: Enrich the event with RenderErr when there is a RenderErrorCode. + + if logp.IsDebug(detailSelector) { + detailf("%s XML=%s Event=%+v", l.logPrefix, x, e) } - records = append(records, r) + records = append(records, Record{ + API: winEventLogAPIName, + EventMetadata: l.eventMetadata, + Event: e, + }) } debugf("%s Read() is returning %d records", l.logPrefix, len(records)) @@ -138,31 +139,26 @@ func (l *winEventLog) Read() ([]Record, error) { func (l *winEventLog) Close() error { debugf("%s Closing handle", l.logPrefix) - return sys.Close(l.subscription) + return win.Close(l.subscription) } // newWinEventLog creates and returns a new EventLog for reading event logs // using the Windows Event Log. func newWinEventLog(c Config) (EventLog, error) { - eventMetadataHandle := func(providerName, sourceName string) eventlogging.MessageFiles { - mf := eventlogging.MessageFiles{SourceName: sourceName} - h, err := sys.OpenPublisherMetadata(0, sourceName, 0) + eventMetadataHandle := func(providerName, sourceName string) sys.MessageFiles { + mf := sys.MessageFiles{SourceName: sourceName} + h, err := win.OpenPublisherMetadata(0, sourceName, 0) if err != nil { mf.Err = err return mf } - mf.Handles = []eventlogging.FileHandle{eventlogging.FileHandle{Handle: uintptr(h)}} + mf.Handles = []sys.FileHandle{sys.FileHandle{Handle: uintptr(h)}} return mf } freeHandle := func(handle uintptr) error { - return sys.Close(sys.EvtHandle(handle)) - } - - ctx, err := sys.CreateRenderContext(nil, sys.EvtRenderContextSystem) - if err != nil { - return nil, err + return win.Close(win.EvtHandle(handle)) } return &winEventLog{ @@ -170,7 +166,6 @@ func newWinEventLog(c Config) (EventLog, error) { remoteServer: c.RemoteAddress, maxRead: defaultMaxNumRead, renderBuf: make([]byte, renderBufferSize), - systemCtx: ctx, cache: newMessageFilesCache(c.Name, eventMetadataHandle, freeHandle), logPrefix: fmt.Sprintf("WinEventLog[%s]", c.Name), eventMetadata: c.EventMetadata, @@ -179,8 +174,8 @@ func newWinEventLog(c Config) (EventLog, error) { func init() { // Register eventlogging API if it is available. - available, _ := sys.IsAvailable() + available, _ := win.IsAvailable() if available { - Register(winEventLogAPIName, 0, newWinEventLog, sys.Channels) + Register(winEventLogAPIName, 0, newWinEventLog, win.Channels) } } diff --git a/winlogbeat/sys/errors.go b/winlogbeat/sys/errors.go new file mode 100644 index 00000000000..2e97d7b7828 --- /dev/null +++ b/winlogbeat/sys/errors.go @@ -0,0 +1,13 @@ +package sys + +// InsufficientBufferError indicates the buffer passed to a system call is too +// small. +type InsufficientBufferError struct { + Cause error + RequiredSize int // Size of the buffer that is required. +} + +// Error returns the cause of the insufficient buffer error. +func (e InsufficientBufferError) Error() string { + return e.Cause.Error() +} diff --git a/winlogbeat/sys/event.go b/winlogbeat/sys/event.go new file mode 100644 index 00000000000..59cdbf4635f --- /dev/null +++ b/winlogbeat/sys/event.go @@ -0,0 +1,187 @@ +package sys + +import ( + "encoding/xml" + "fmt" + "time" +) + +// UnmarshalEventXML unmarshals the given XML into a new Event. +func UnmarshalEventXML(rawXML []byte) (Event, error) { + var event Event + err := xml.Unmarshal(rawXML, &event) + return event, err +} + +// Event holds the data from a log record. +type Event struct { + // System + Provider Provider `xml:"System>Provider"` + EventIdentifier EventIdentifier `xml:"System>EventID"` + Version uint8 `xml:"System>Version"` + LevelRaw uint8 `xml:"System>Level"` + TaskRaw uint16 `xml:"System>Task"` + OpcodeRaw uint8 `xml:"System>Opcode"` + TimeCreated TimeCreated `xml:"System>TimeCreated"` + RecordID uint64 `xml:"System>EventRecordID"` + Correlation Correlation `xml:"System>Correlation"` + Execution Execution `xml:"System>Execution"` + Channel string `xml:"System>Channel"` + Computer string `xml:"System>Computer"` + User SID `xml:"System>Security"` + + EventData EventData `xml:"EventData"` + UserData UserData `xml:"UserData"` + + // RenderingInfo + Message string `xml:"RenderingInfo>Message"` + Level string `xml:"RenderingInfo>Level"` + Task string `xml:"RenderingInfo>Task"` + Opcode string `xml:"RenderingInfo>Opcode"` + Keywords []string `xml:"RenderingInfo>Keywords>Keyword"` + + // ProcessingErrorData + RenderErrorCode uint32 `xml:"ProcessingErrorData>ErrorCode"` + RenderErrorDataItemName string `xml:"ProcessingErrorData>DataItemName"` + RenderErr string +} + +// Provider identifies the provider that logged the event. The Name and GUID +// attributes are included if the provider used an instrumentation manifest to +// define its events; otherwise, the EventSourceName attribute is included if a +// legacy event provider (using the Event Logging API) logged the event. +type Provider struct { + Name string `xml:"Name,attr"` + GUID string `xml:"Guid,attr"` + EventSourceName string `xml:"EventSourceName,attr"` +} + +// Correlation contains activity identifiers that consumers can use to group +// related events together. +type Correlation struct { + ActivityID string `xml:"ActivityID,attr"` + RelatedActivityID string `xml:"RelatedActivityID,attr"` +} + +// Execution contains information about the process and thread that logged the +// event. +type Execution struct { + ProcessID uint32 `xml:"ProcessID,attr"` + ThreadID uint32 `xml:"ThreadID,attr"` + + // Only available for events logged to an event tracing log file (.etl file). + ProcessorID uint32 `xml:"ProcessorID,attr"` + SessionID uint32 `xml:"SessionID,attr"` + KernelTime uint32 `xml:"KernelTime,attr"` + UserTime uint32 `xml:"UserTime,attr"` + ProcessorTime uint32 `xml:"ProcessorTime,attr"` +} + +// EventIdentifier is the identifer that the provider uses to identify a +// specific event type. +type EventIdentifier struct { + Qualifiers uint16 `xml:"Qualifiers,attr"` + ID uint32 `xml:",chardata"` +} + +// TimeCreated contains the system time of when the event was logged. +type TimeCreated struct { + SystemTime time.Time +} + +// UnmarshalXML unmarshals an XML dataTime string. +func (t *TimeCreated) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + attrs := struct { + SystemTime string `xml:"SystemTime,attr"` + RawTime uint64 `xml:"RawTime,attr"` + }{} + + err := d.DecodeElement(&attrs, &start) + if err != nil { + return err + } + + if attrs.SystemTime != "" { + // This works but XML dateTime is really ISO8601. + t.SystemTime, err = time.Parse(time.RFC3339Nano, attrs.SystemTime) + } else if attrs.RawTime != 0 { + // The units for RawTime are not specified in the documentation. I think + // it is only used in event tracing so this shouldn't be a problem. + err = fmt.Errorf("failed to unmarshal TimeCreated RawTime='%d'", attrs.RawTime) + } + + return err +} + +// EventData contains the event data. The EventData section is used if the +// message provider template does not contain a UserData section. +type EventData struct { + Pairs []KeyValue `xml:",any"` +} + +// UserData contains the event data. +type UserData struct { + Name xml.Name + Pairs []KeyValue +} + +// UnmarshalXML unmarshals UserData XML. +func (u *UserData) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + // Assume that UserData has the same general key-value structure as + // EventData does. + in := struct { + Pairs []KeyValue `xml:",any"` + }{} + + // Read tokens until we find the first StartElement then unmarshal it. + for { + t, err := d.Token() + if err != nil { + return err + } + + if se, ok := t.(xml.StartElement); ok { + err = d.DecodeElement(&in, &se) + if err != nil { + return err + } + + u.Name = se.Name + u.Pairs = in.Pairs + d.Skip() + break + } + } + + return nil +} + +// KeyValue is a key value pair of strings. +type KeyValue struct { + Key string + Value string +} + +// UnmarshalXML unmarshals an arbitrary XML element into a KeyValue. The key +// becomes the name of the element or value of the Name attribute if it exists. +// The value is the character data contained within the element. +func (kv *KeyValue) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + elem := struct { + XMLName xml.Name + Name string `xml:"Name,attr"` + Value string `xml:",chardata"` + }{} + + err := d.DecodeElement(&elem, &start) + if err != nil { + return err + } + + kv.Key = elem.XMLName.Local + if elem.Name != "" { + kv.Key = elem.Name + } + kv.Value = elem.Value + + return nil +} diff --git a/winlogbeat/sys/event_test.go b/winlogbeat/sys/event_test.go new file mode 100644 index 00000000000..1d52e65772d --- /dev/null +++ b/winlogbeat/sys/event_test.go @@ -0,0 +1,161 @@ +package sys + +import ( + "encoding/json" + "encoding/xml" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +const allXML = ` + + + + 91 + 0 + 4 + 9 + 0 + 0x4000000000000004 + + 100 + + + Microsoft-Windows-WinRM/Operational + vagrant-2012-r2 + + + + winlogbeat + running + 770069006E006C006F00670062006500610074002F0034000000 + + + + \\VAGRANT-2012-R2 + vagrant + + + + 15005 + shellId + 68007400740070003A002F002F0073006300680065006D00610073002E006D006900630072006F0073006F00660074002E0063006F006D002F007700620065006D002F00770073006D0061006E002F0031002F00770069006E0064006F00770073002F007300680065006C006C002F0063006D0064000000 + + + Creating WSMan shell on server with ResourceUri: %1 + Information + Request handling + Info + Microsoft-Windows-WinRM/Operational + Microsoft-Windows-Windows Remote Management + + Server + + + +` + +func TestXML(t *testing.T) { + allXMLTimeCreated, _ := time.Parse(time.RFC3339Nano, "2016-01-28T20:33:27.990735300Z") + + var tests = []struct { + xml string + event Event + }{ + { + xml: allXML, + event: Event{ + Provider: Provider{ + Name: "Microsoft-Windows-WinRM", + GUID: "{a7975c8f-ac13-49f1-87da-5a984a4ab417}", + EventSourceName: "Service Control Manager", + }, + EventIdentifier: EventIdentifier{ID: 91}, + LevelRaw: 4, + TaskRaw: 9, + TimeCreated: TimeCreated{allXMLTimeCreated}, + RecordID: 100, + Correlation: Correlation{"{A066CCF1-8AB3-459B-B62F-F79F957A5036}", "{85FC0930-9C49-42DA-804B-A7368104BD1B}"}, + Execution: Execution{ProcessID: 920, ThreadID: 1152}, + Channel: "Microsoft-Windows-WinRM/Operational", + Computer: "vagrant-2012-r2", + User: SID{Identifier: "S-1-5-21-3541430928-2051711210-1391384369-1001"}, + EventData: EventData{ + Pairs: []KeyValue{ + KeyValue{"param1", "winlogbeat"}, + KeyValue{"param2", "running"}, + KeyValue{"Binary", "770069006E006C006F00670062006500610074002F0034000000"}, + }, + }, + UserData: UserData{ + Name: xml.Name{ + Local: "EventXML", + Space: "Event_NS", + }, + Pairs: []KeyValue{ + KeyValue{"ServerName", `\\VAGRANT-2012-R2`}, + KeyValue{"UserName", "vagrant"}, + }, + }, + Message: "Creating WSMan shell on server with ResourceUri: %1", + Level: "Information", + Task: "Request handling", + Opcode: "Info", + Keywords: []string{"Server"}, + RenderErrorCode: 15005, + RenderErrorDataItemName: "shellId", + }, + }, + { + xml: ` + + + + {00000000-0000-0000-0000-000000000000} + + + + `, + event: Event{ + UserData: UserData{ + Name: xml.Name{ + Local: "Operation_ClientFailure", + Space: "http://manifests.microsoft.com/win/2006/windows/WMI", + }, + Pairs: []KeyValue{ + KeyValue{"Id", "{00000000-0000-0000-0000-000000000000}"}, + }, + }, + }, + }, + } + + for _, test := range tests { + event, err := UnmarshalEventXML([]byte(test.xml)) + if err != nil { + t.Error(err) + continue + } + assert.Equal(t, test.event, event) + + if testing.Verbose() { + json, err := json.MarshalIndent(event, "", " ") + if err != nil { + t.Error(err) + } + fmt.Println(string(json)) + } + } +} + +func BenchmarkXMLUnmarshal(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := UnmarshalEventXML([]byte(allXML)) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/winlogbeat/sys/eventlogging/eventlogging.go b/winlogbeat/sys/eventlogging/eventlogging.go deleted file mode 100644 index dedd2ac0c7f..00000000000 --- a/winlogbeat/sys/eventlogging/eventlogging.go +++ /dev/null @@ -1,28 +0,0 @@ -package eventlogging - -import ( - "time" -) - -// Event represents a Windows Event within the Windows Event Log after it has -// been converted from bytes to a structure. -type Event struct { - RecordID uint32 `json:",omitempty"` - TimeGenerated *time.Time `json:",omitempty"` - TimeWritten *time.Time `json:",omitempty"` - EventID uint32 `json:",omitempty"` - Level string `json:",omitempty"` - SourceName string `json:",omitempty"` - Computer string `json:",omitempty"` - - UserSID *SID `json:",omitempty"` - UserSIDErr error - - // Strings that must be resolved by DLL lookups. - Message string `json:",omitempty"` - Category string `json:",omitempty"` - - MessageInserts []string // Strings inserted into a message template to - // create Message. - MessageErr error // Possible error that occurred while formatting Message. -} diff --git a/winlogbeat/sys/eventlogging/eventlogging_windows.go b/winlogbeat/sys/eventlogging/eventlogging_windows.go index 45654af263c..9e15e15e467 100644 --- a/winlogbeat/sys/eventlogging/eventlogging_windows.go +++ b/winlogbeat/sys/eventlogging/eventlogging_windows.go @@ -5,17 +5,25 @@ import ( "encoding/binary" "fmt" "reflect" - "strconv" "strings" "syscall" "time" "unsafe" "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/winlogbeat/sys" "golang.org/x/sys/windows" "golang.org/x/sys/windows/registry" ) +// The value of EventID element contains the low-order 16 bits of the event +// identifier and the Qualifier attribute contains the high-order 16 bits of the +// event identifier. +const ( + eventIDLowerMask uint32 = 0xFFFF + eventIDUpperMask uint32 = 0xFFFF0000 +) + // IsAvailable returns true if the Event Logging API is supported by this // operating system. If not supported then false is returned with the // accompanying error. @@ -71,7 +79,7 @@ func ReadEventLog( &buffer[0], uint32(len(buffer)), &numBytesRead, &minBytesRequiredToRead) if err == syscall.ERROR_INSUFFICIENT_BUFFER { - return 0, InsufficientBufferError{err, int(minBytesRequiredToRead)} + return 0, sys.InsufficientBufferError{err, int(minBytesRequiredToRead)} } if err != nil { return 0, err @@ -85,15 +93,15 @@ func ReadEventLog( return int(numBytesRead), nil } -// RenderEvents takes raw events, formats them into a structured event, and adds -// each event to a slice. The slice of formatted events is then returned. +// RenderEvents reads raw events from the provided buffer, formats them into +// structured events, and adds each on to a slice that is returned. func RenderEvents( eventsRaw []byte, lang uint32, buffer []byte, - pubHandleProvider func(string) MessageFiles, -) ([]Event, int, error) { - var events []Event + pubHandleProvider func(string) sys.MessageFiles, +) ([]sys.Event, int, error) { + var events []sys.Event var offset int for { if offset >= len(eventsRaw) { @@ -105,14 +113,18 @@ func RenderEvents( if err != nil { return nil, 0, err } - event := Event{ - RecordID: record.recordNumber, - TimeGenerated: unixTime(record.timeGenerated), - TimeWritten: unixTime(record.timeWritten), - EventID: record.eventID, - Level: EventType(record.eventType).String(), - SourceName: record.sourceName, - Computer: record.computerName, + + var qualifier uint16 = uint16((record.eventID & eventIDUpperMask) >> 16) + var eventID uint32 = record.eventID & eventIDLowerMask + event := sys.Event{ + Provider: sys.Provider{Name: record.sourceName}, + EventIdentifier: sys.EventIdentifier{ID: eventID, Qualifiers: qualifier}, + LevelRaw: uint8(record.eventType), // Possible overflow + TaskRaw: record.eventCategory, + TimeCreated: sys.TimeCreated{unixTime(record.timeGenerated)}, + RecordID: uint64(record.recordNumber), + Computer: record.computerName, + Level: EventType(record.eventType).String(), } // Create a slice from the larger buffer only data from the one record. @@ -120,25 +132,33 @@ func RenderEvents( recordBuf := eventsRaw[offset : offset+int(record.length)] offset += int(record.length) + // Parse and format the user that logged the event. + sid, _ := parseSID(record, recordBuf) // TODO: do something with this error + if sid != nil { + event.User = *sid + } + // Parse the UTF-16 message insert strings. stringInserts, stringInsertPtrs, err := parseInsertStrings(record, recordBuf) if err != nil { - event.MessageErr = err + event.RenderErr = err.Error() events = append(events, event) continue } - event.MessageInserts = stringInserts - // Format the parameterized message using the insert strings. + for _, s := range stringInserts { + event.EventData.Pairs = append(event.EventData.Pairs, sys.KeyValue{Value: s}) + } + + // Format the parametrized message using the insert strings. event.Message, _, err = formatMessage(record.sourceName, record.eventID, lang, stringInsertPtrs, buffer, pubHandleProvider) - event.MessageErr = err - - // Parse and format the user that logged the event. - event.UserSID, event.UserSIDErr = parseSID(record, recordBuf) - - // TODO: Parse the message category string. - event.Category = strconv.FormatUint(uint64(record.eventCategory), 10) + if err != nil { + event.RenderErr = err.Error() + if errno, ok := err.(syscall.Errno); ok { + event.RenderErrorCode = uint32(errno) + } + } events = append(events, event) } @@ -148,9 +168,9 @@ func RenderEvents( // unixTime takes a time which is an unsigned 32-bit integer, and converts it // into a Golang time.Time pointer formatted as a unix time. -func unixTime(sec uint32) *time.Time { +func unixTime(sec uint32) time.Time { t := time.Unix(int64(sec), 0) - return &t + return t } // formatmessage takes event data and formats the event message into a @@ -161,7 +181,7 @@ func formatMessage( lang uint32, stringInserts []uintptr, buffer []byte, - pubHandleProvider func(string) MessageFiles, + pubHandleProvider func(string) sys.MessageFiles, ) (string, int, error) { var addr uintptr if len(stringInserts) > 0 { @@ -171,7 +191,7 @@ func formatMessage( messageFiles := pubHandleProvider(sourceName) var lastErr error - var fh FileHandle + var fh sys.FileHandle var message string for _, fh = range messageFiles.Handles { if fh.Err != nil { @@ -196,12 +216,10 @@ func formatMessage( continue } - message, _, err = UTF16BytesToString(buffer[:numChars*2]) + message, _, err = sys.UTF16BytesToString(buffer[:numChars*2]) if err != nil { return "", 0, err } - - message = RemoveWindowsLineEndings(message) } if message == "" { @@ -340,7 +358,7 @@ func parseEventLogRecord(buffer []byte) (eventLogRecord, error) { // SourceName (null-terminated UTF-16 string) begin, _ := reader.Seek(0, 1) - sourceName, length, err := UTF16BytesToString(buffer[begin:]) + sourceName, length, err := sys.UTF16BytesToString(buffer[begin:]) if err != nil { return record, err } @@ -351,7 +369,7 @@ func parseEventLogRecord(buffer []byte) (eventLogRecord, error) { } // ComputerName (null-terminated UTF-16 string) - computerName, length, err := UTF16BytesToString(buffer[begin:]) + computerName, length, err := sys.UTF16BytesToString(buffer[begin:]) if err != nil { return record, err } @@ -384,7 +402,7 @@ func parseInsertStrings(record eventLogRecord, buffer []byte) ([]string, []uintp "offset=%d, len(buffer)=%d, record=%+v", i+1, offset, len(buffer), record) } - insertStr, length, err := UTF16BytesToString(buffer[offset:]) + insertStr, length, err := sys.UTF16BytesToString(buffer[offset:]) if err != nil { return nil, nil, err } @@ -396,7 +414,7 @@ func parseInsertStrings(record eventLogRecord, buffer []byte) ([]string, []uintp return inserts, insertPtrs, nil } -func parseSID(record eventLogRecord, buffer []byte) (*SID, error) { +func parseSID(record eventLogRecord, buffer []byte) (*sys.SID, error) { if record.userSidLength == 0 { return nil, nil } @@ -407,18 +425,7 @@ func parseSID(record eventLogRecord, buffer []byte) (*SID, error) { return nil, err } - account, domain, accountType, err := sid.LookupAccount("") - if err != nil { - // Ignore the error and return a partially populated SID. - return &SID{Identifier: identifier}, nil - } - - return &SID{ - Identifier: identifier, - Name: account, - Domain: domain, - Type: SIDType(accountType), - }, nil + return &sys.SID{Identifier: identifier}, nil } // ClearEventLog takes an event log file handle and empties the log. If a backup @@ -480,8 +487,8 @@ func CloseEventLog(handle Handle) error { // event log messages. If found, it loads the libraries as a datafiles and // returns a slice of Handles to the libraries. Those handles must be closed // by the caller. -func QueryEventMessageFiles(providerName, sourceName string) MessageFiles { - mf := MessageFiles{SourceName: sourceName} +func QueryEventMessageFiles(providerName, sourceName string) sys.MessageFiles { + mf := sys.MessageFiles{SourceName: sourceName} // Open key in registry: registryKeyName := fmt.Sprintf( @@ -523,7 +530,7 @@ func QueryEventMessageFiles(providerName, sourceName string) MessageFiles { strings.Join(eventMessageFiles, ",")) // Load the libraries: - var files []FileHandle + var files []sys.FileHandle for _, eventMessageFile := range eventMessageFiles { sPtr, err := syscall.UTF16PtrFromString(eventMessageFile) if err != nil { @@ -538,7 +545,7 @@ func QueryEventMessageFiles(providerName, sourceName string) MessageFiles { "Skipping. %v", eventMessageFile, err) } - f := FileHandle{File: eventMessageFile, Handle: uintptr(handle), Err: err} + f := sys.FileHandle{File: eventMessageFile, Handle: uintptr(handle), Err: err} files = append(files, f) } diff --git a/winlogbeat/sys/msgfile.go b/winlogbeat/sys/msgfile.go new file mode 100644 index 00000000000..c5a9f16b1c7 --- /dev/null +++ b/winlogbeat/sys/msgfile.go @@ -0,0 +1,16 @@ +package sys + +// MessageFiles contains handles to event message files associated with an +// event log source. +type MessageFiles struct { + SourceName string + Err error + Handles []FileHandle +} + +// FileHandle contains the handle to a single Windows message file. +type FileHandle struct { + File string // Fully-qualified path to the event message file. + Handle uintptr // Handle to the loaded event message file. + Err error // Error that occurred while loading Handle. +} diff --git a/winlogbeat/sys/eventlogging/common.go b/winlogbeat/sys/sid.go similarity index 59% rename from winlogbeat/sys/eventlogging/common.go rename to winlogbeat/sys/sid.go index 3a8f87c29ea..8688d7dbed8 100644 --- a/winlogbeat/sys/eventlogging/common.go +++ b/winlogbeat/sys/sid.go @@ -1,4 +1,4 @@ -package eventlogging +package sys import ( "fmt" @@ -6,7 +6,7 @@ import ( // SID represents the Windows Security Identifier for an account. type SID struct { - Identifier string + Identifier string `xml:"UserID,attr"` Name string Domain string Type SIDType @@ -54,30 +54,3 @@ var sidTypeToString = map[SIDType]string{ func (st SIDType) String() string { return sidTypeToString[st] } - -// MessageFiles contains handles to event message files associated with an -// event log source. -type MessageFiles struct { - SourceName string - Err error - Handles []FileHandle -} - -// FileHandle contains the handle to a single Windows message file. -type FileHandle struct { - File string // Fully-qualified path to the event message file. - Handle uintptr // Handle to the loaded event message file. - Err error // Error that occurred while loading Handle. -} - -// InsufficientBufferError indicates the buffer passed to a system call is too -// small. -type InsufficientBufferError struct { - Cause error - RequiredSize int // Size of the buffer that is required. -} - -// Error returns the cause of the insufficient buffer error. -func (e InsufficientBufferError) Error() string { - return e.Cause.Error() -} diff --git a/winlogbeat/sys/eventlogging/common_test.go b/winlogbeat/sys/sid_test.go similarity index 97% rename from winlogbeat/sys/eventlogging/common_test.go rename to winlogbeat/sys/sid_test.go index dc4c4c551b5..1a553ffca24 100644 --- a/winlogbeat/sys/eventlogging/common_test.go +++ b/winlogbeat/sys/sid_test.go @@ -1,6 +1,6 @@ // +build !integration -package eventlogging +package sys import ( "testing" diff --git a/winlogbeat/sys/sid_windows.go b/winlogbeat/sys/sid_windows.go new file mode 100644 index 00000000000..f7b4bfe4c5d --- /dev/null +++ b/winlogbeat/sys/sid_windows.go @@ -0,0 +1,26 @@ +package sys + +import "golang.org/x/sys/windows" + +// PopulateAccount lookups the account name and type associated with a SID. +// The account name, domain, and type are added to the given sid. +func PopulateAccount(sid *SID) error { + if sid == nil || sid.Identifier == "" { + return nil + } + + s, err := windows.StringToSid(sid.Identifier) + if err != nil { + return err + } + + account, domain, accType, err := s.LookupAccount("") + if err != nil { + return err + } + + sid.Name = account + sid.Domain = domain + sid.Type = SIDType(accType) + return nil +} diff --git a/winlogbeat/sys/eventlogging/strings.go b/winlogbeat/sys/strings.go similarity index 97% rename from winlogbeat/sys/eventlogging/strings.go rename to winlogbeat/sys/strings.go index cab48c2d11a..25f27a9d3a5 100644 --- a/winlogbeat/sys/eventlogging/strings.go +++ b/winlogbeat/sys/strings.go @@ -1,4 +1,4 @@ -package eventlogging +package sys import ( "fmt" diff --git a/winlogbeat/sys/wineventlog/syscall_windows.go b/winlogbeat/sys/wineventlog/syscall_windows.go index 5acbe4743f1..3ee3f736dd4 100644 --- a/winlogbeat/sys/wineventlog/syscall_windows.go +++ b/winlogbeat/sys/wineventlog/syscall_windows.go @@ -201,7 +201,7 @@ func (e EvtSystemPropertyID) String() string { //sys _EvtSubscribe(session EvtHandle, signalEvent uintptr, channelPath *uint16, query *uint16, bookmark EvtHandle, context uintptr, callback syscall.Handle, flags EvtSubscribeFlag) (handle EvtHandle, err error) = wevtapi.EvtSubscribe //sys _EvtCreateBookmark(bookmarkXML *uint16) (handle EvtHandle, err error) = wevtapi.EvtCreateBookmark //sys _EvtUpdateBookmark(bookmark EvtHandle, event EvtHandle) (err error) = wevtapi.EvtUpdateBookmark -//sys _EvtCreateRenderContext(ValuePathsCount uint32, valuePaths *uint16, flags EvtRenderContextFlag) (handle EvtHandle, err error) = wevtapi.EvtCreateRenderContext +//sys _EvtCreateRenderContext(ValuePathsCount uint32, valuePaths uintptr, flags EvtRenderContextFlag) (handle EvtHandle, err error) = wevtapi.EvtCreateRenderContext //sys _EvtRender(context EvtHandle, fragment EvtHandle, flags EvtRenderFlag, bufferSize uint32, buffer *byte, bufferUsed *uint32, propertyCount *uint32) (err error) = wevtapi.EvtRender //sys _EvtClose(object EvtHandle) (err error) = wevtapi.EvtClose //sys _EvtNext(resultSet EvtHandle, eventArraySize uint32, eventArray *EvtHandle, timeout uint32, flags uint32, numReturned *uint32) (err error) = wevtapi.EvtNext diff --git a/winlogbeat/sys/wineventlog/wineventlog.go b/winlogbeat/sys/wineventlog/wineventlog.go deleted file mode 100644 index 1f3224c0ee4..00000000000 --- a/winlogbeat/sys/wineventlog/wineventlog.go +++ /dev/null @@ -1,43 +0,0 @@ -package wineventlog - -import ( - "time" - - "github.com/elastic/beats/winlogbeat/sys/eventlogging" -) - -// Event holds the data from the a log record. -type Event struct { - // System context properties. - ProviderName string `json:",omitempty"` - ProviderGUID string `json:",omitempty"` - EventID uint16 `json:",omitempty"` - Qualifiers uint16 `json:",omitempty"` - TimeCreated *time.Time `json:",omitempty"` - RecordID uint64 `json:",omitempty"` - ActivityID string `json:",omitempty"` - RelatedActivityID string `json:",omitempty"` - ProcessID uint32 `json:",omitempty"` - ThreadID uint32 `json:",omitempty"` - Channel string `json:",omitempty"` - Computer string `json:",omitempty"` - UserSID *eventlogging.SID `json:",omitempty"` - Version uint8 `json:",omitempty"` - - // String properties that require a call to FormatMessage. - - Message string `json:",omitempty"` - MessageErr error - - Level string `json:",omitempty"` - LevelErr error - - Task string `json:",omitempty"` - TaskErr error - - Opcode string `json:",omitempty"` - OpcodeErr error - - Keywords []string `json:",omitempty"` - KeywordsError error -} diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows.go b/winlogbeat/sys/wineventlog/wineventlog_windows.go index ed66ab17eb5..20028e01b47 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows.go @@ -9,10 +9,8 @@ import ( "reflect" "runtime" "syscall" - "time" - "unsafe" - "github.com/elastic/beats/winlogbeat/sys/eventlogging" + "github.com/elastic/beats/winlogbeat/sys" "golang.org/x/sys/windows" ) @@ -29,6 +27,14 @@ var ( const bookmarkTemplate = `` +var providerNameContext EvtHandle + +func init() { + if avail, _ := IsAvailable(); avail { + providerNameContext, _ = CreateRenderContext([]string{"Event/System/Provider/@Name"}, EvtRenderContextValues) + } +} + // IsAvailable returns true if the Windows Event Log API is supported by this // operating system. If not supported then false is returned with the // accompanying error. @@ -138,56 +144,21 @@ func EventHandles(subscription EvtHandle, maxHandles int) ([]EvtHandle, error) { } // RenderEvent reads the event data associated with the EvtHandle and renders -// the data so that it can used. +// the data as XML. func RenderEvent( eventHandle EvtHandle, - systemContext EvtHandle, lang uint32, renderBuf []byte, - pubHandleProvider func(string) eventlogging.MessageFiles, -) (Event, error) { - var err error - - // Create a render context for local machine. - if systemContext == 0 { - systemContext, err = _EvtCreateRenderContext(0, nil, EvtRenderContextSystem) - if err != nil { - return Event{}, err - } - defer _EvtClose(systemContext) - } - - var bufferUsed, propertyCount uint32 - err = _EvtRender(systemContext, eventHandle, EvtRenderEventValues, - uint32(len(renderBuf)), &renderBuf[0], &bufferUsed, - &propertyCount) - if err == ERROR_INSUFFICIENT_BUFFER { - return Event{}, eventlogging.InsufficientBufferError{err, int(bufferUsed)} - } - if err != nil { - return Event{}, err - } - - // Validate bufferUsed set by Windows. - if int(bufferUsed) > len(renderBuf) { - return Event{}, fmt.Errorf("Bytes used (%d) is greater than the "+ - "buffer size (%d)", bufferUsed, len(renderBuf)) - } - - // Ignore any additional unknown properties that might exist. - if propertyCount > uint32(EvtSystemPropertyIdEND) { - propertyCount = uint32(EvtSystemPropertyIdEND) - } - - var e Event - err = parseRenderEventBuffer(renderBuf[:bufferUsed], &e) + pubHandleProvider func(string) sys.MessageFiles, +) (string, error) { + providerName, err := evtRenderProviderName(renderBuf, eventHandle) if err != nil { - return Event{}, err + return "", err } var publisherHandle uintptr if pubHandleProvider != nil { - messageFiles := pubHandleProvider(e.ProviderName) + messageFiles := pubHandleProvider(providerName) if messageFiles.Err == nil { // There is only ever a single handle when using the Windows Event // Log API. @@ -195,108 +166,10 @@ func RenderEvent( } } - // Populate strings that must be looked up. - populateStrings(eventHandle, EvtHandle(publisherHandle), lang, renderBuf, &e) - - return e, nil -} - -// parseRenderEventBuffer parses the system context data from buffer. This -// function can be used on the data written by the EvtRender system call. -func parseRenderEventBuffer(buffer []byte, evt *Event) error { - reader := bytes.NewReader(buffer) - - for i := 0; i < int(EvtSystemPropertyIdEND); i++ { - // Each EVT_VARIANT is 16 bytes. - _, err := reader.Seek(int64(16*i), 0) - if err != nil { - return fmt.Errorf("Error seeking to read %s: %v", - EvtSystemPropertyID(i), err) - } - - switch EvtSystemPropertyID(i) { - case EvtSystemKeywords, EvtSystemLevel, EvtSystemOpcode, EvtSystemTask: - // These are rendered as strings so ignore them here. - continue - case EvtSystemProviderName: - evt.ProviderName, err = readString(buffer, reader) - case EvtSystemComputer: - evt.Computer, err = readString(buffer, reader) - case EvtSystemChannel: - evt.Channel, err = readString(buffer, reader) - case EvtSystemVersion: - err = binary.Read(reader, binary.LittleEndian, &evt.Version) - case EvtSystemEventID: - err = binary.Read(reader, binary.LittleEndian, &evt.EventID) - case EvtSystemQualifiers: - err = binary.Read(reader, binary.LittleEndian, &evt.Qualifiers) - case EvtSystemThreadID: - err = binary.Read(reader, binary.LittleEndian, &evt.ThreadID) - case EvtSystemProcessID: - err = binary.Read(reader, binary.LittleEndian, &evt.ProcessID) - case EvtSystemEventRecordId: - err = binary.Read(reader, binary.LittleEndian, &evt.RecordID) - case EvtSystemTimeCreated: - evt.TimeCreated, err = readFiletime(reader) - case EvtSystemActivityID: - evt.ActivityID, err = readGUID(buffer, reader) - case EvtSystemRelatedActivityID: - evt.RelatedActivityID, err = readGUID(buffer, reader) - case EvtSystemProviderGuid: - evt.ProviderGUID, err = readGUID(buffer, reader) - case EvtSystemUserID: - evt.UserSID, err = readSID(buffer, reader) - } - - if err != nil { - return fmt.Errorf("Error reading %s: %v", EvtSystemPropertyID(i), err) - } - } - - return nil -} - -// populateStrings populates the string fields of the Event that require -// formatting (Message, Level, Task, Opcode, and Keywords). It attempts to -// populate each field even if an error occurs. Any errors that occur are -// written to the Event (see MessageErr, LevelErr, TaskErr, OpcodeErr, and -// KeywordsErr). -func populateStrings( - eventHandle EvtHandle, - providerHandle EvtHandle, - lang uint32, - buffer []byte, - event *Event, -) { - var strs []string - strs, event.MessageErr = FormatEventString(EvtFormatMessageEvent, - eventHandle, event.ProviderName, providerHandle, lang, buffer) - if len(strs) > 0 { - event.Message = strs[0] - } - // TODO: Populate the MessageInserts when there is a MessageErr. - - strs, event.LevelErr = FormatEventString(EvtFormatMessageLevel, - eventHandle, event.ProviderName, providerHandle, lang, buffer) - if len(strs) > 0 { - event.Level = strs[0] - } - - strs, event.TaskErr = FormatEventString(EvtFormatMessageTask, - eventHandle, event.ProviderName, providerHandle, lang, buffer) - if len(strs) > 0 { - event.Task = strs[0] - } - - strs, event.OpcodeErr = FormatEventString(EvtFormatMessageOpcode, - eventHandle, event.ProviderName, providerHandle, lang, buffer) - if len(strs) > 0 { - event.Opcode = strs[0] - } - - event.Keywords, event.KeywordsError = FormatEventString( - EvtFormatMessageKeyword, eventHandle, event.ProviderName, - providerHandle, lang, buffer) + // Only a single string is returned when rendering XML. + xml, err := FormatEventString(EvtFormatMessageXml, + eventHandle, providerName, EvtHandle(publisherHandle), lang, renderBuf) + return xml, err } // CreateBookmark creates a new handle to a bookmark. Close must be called on @@ -319,7 +192,22 @@ func CreateBookmark(channel string, recordID uint64) (EvtHandle, error) { // CreateRenderContext creates a render context. Close must be called on // returned EvtHandle when finished with the handle. func CreateRenderContext(valuePaths []string, flag EvtRenderContextFlag) (EvtHandle, error) { - context, err := _EvtCreateRenderContext(0, nil, EvtRenderContextSystem) + var paths []uintptr + for _, path := range valuePaths { + utf16, err := syscall.UTF16FromString(path) + if err != nil { + return 0, err + } + + paths = append(paths, reflect.ValueOf(&utf16[0]).Pointer()) + } + + var pathsAddr uintptr + if len(paths) > 0 { + pathsAddr = reflect.ValueOf(&paths[0]).Pointer() + } + + context, err := _EvtCreateRenderContext(uint32(len(paths)), pathsAddr, flag) if err != nil { return 0, err } @@ -368,18 +256,13 @@ func FormatEventString( publisherHandle EvtHandle, lang uint32, buffer []byte, -) ([]string, error) { - p, err := syscall.UTF16PtrFromString(publisher) - if err != nil { - return nil, err - } - +) (string, error) { // Open a publisher handle if one was not provided. ph := publisherHandle if ph == 0 { - ph, err = _EvtOpenPublisherMetadata(0, p, nil, lang, 0) + ph, err := OpenPublisherMetadata(0, publisher, 0) if err != nil { - return nil, err + return "", err } defer _EvtClose(ph) } @@ -387,45 +270,31 @@ func FormatEventString( // Create a buffer if one was not provider. var bufferUsed uint32 if buffer == nil { - err = _EvtFormatMessage(ph, eventHandle, 0, 0, 0, messageFlag, + err := _EvtFormatMessage(ph, eventHandle, 0, 0, 0, messageFlag, 0, nil, &bufferUsed) - bufferUsed *= 2 // It returns the number of utf-16 chars. if err != nil && err != ERROR_INSUFFICIENT_BUFFER { - return nil, err + return "", err } + bufferUsed *= 2 buffer = make([]byte, bufferUsed) bufferUsed = 0 } - err = _EvtFormatMessage(ph, eventHandle, 0, 0, 0, messageFlag, + err := _EvtFormatMessage(ph, eventHandle, 0, 0, 0, messageFlag, uint32(len(buffer)/2), &buffer[0], &bufferUsed) - bufferUsed *= 2 // It returns the number of utf-16 chars. + bufferUsed *= 2 if err == ERROR_INSUFFICIENT_BUFFER { - return nil, eventlogging.InsufficientBufferError{err, int(bufferUsed)} + return "", sys.InsufficientBufferError{err, int(bufferUsed)} } if err != nil { - return nil, err - } - - var value string - var offset int - var size int - var values []string - for { - value, size, err = eventlogging.UTF16BytesToString(buffer[offset:bufferUsed]) - if err != nil { - return nil, err - } - offset += size - values = append(values, eventlogging.RemoveWindowsLineEndings(value)) - - if offset >= int(bufferUsed) { - break - } + return "", err } - return values, nil + // This assumes there is only a single string value to read. This will + // not work to read keys (when messageFlag == EvtFormatMessageKeyword). + value, _, err := sys.UTF16BytesToString(buffer[0:bufferUsed]) + return value, err } // offset reads a pointer value from the reader then calculates an offset from @@ -480,110 +349,23 @@ func readString(buffer []byte, reader io.Reader) (string, error) { } return "", err } - str, _, err := eventlogging.UTF16BytesToString(buffer[offset:]) + str, _, err := sys.UTF16BytesToString(buffer[offset:]) return str, err } -// readFiletime reads a Windows Filetime struct and converts it to a -// time.Time value with a UTC timezone. -func readFiletime(reader io.Reader) (*time.Time, error) { - var filetime syscall.Filetime - err := binary.Read(reader, binary.LittleEndian, &filetime.LowDateTime) - if err != nil { - return nil, err - } - err = binary.Read(reader, binary.LittleEndian, &filetime.HighDateTime) - if err != nil { - return nil, err - } - t := time.Unix(0, filetime.Nanoseconds()).UTC() - return &t, nil -} - -// readSID reads a pointer using the reader then parses the Windows SID -// data that the pointer addresses within the buffer. -func readSID(buffer []byte, reader io.Reader) (*eventlogging.SID, error) { - offset, err := offset(buffer, reader) - if err != nil { - // Ignore NULL values. - if err == ErrorEvtVarTypeNull { - return nil, nil - } - return nil, err - } - sid := (*windows.SID)(unsafe.Pointer(&buffer[offset])) - identifier, err := sid.String() - if err != nil { - return nil, err - } - - account, domain, accountType, err := sid.LookupAccount("") - if err != nil { - // Ignore the error and return a partially populated SID. - return &eventlogging.SID{Identifier: identifier}, nil - } - - return &eventlogging.SID{ - Identifier: identifier, - Name: account, - Domain: domain, - Type: eventlogging.SIDType(accountType), - }, nil -} - -// readGUID reads a pointer using the reader then parses the Windows GUID -// data that the pointer addresses within the buffer. -func readGUID(buffer []byte, reader io.ReadSeeker) (string, error) { - offset, err := offset(buffer, reader) - if err != nil { - // Ignore NULL values. - if err == ErrorEvtVarTypeNull { - return "", nil - } - return "", err - } - - guid := &syscall.GUID{} - _, err = reader.Seek(int64(offset), 0) - if err != nil { - return "", err - } - err = binary.Read(reader, binary.LittleEndian, &guid.Data1) - if err != nil { - return "", err - } - err = binary.Read(reader, binary.LittleEndian, &guid.Data2) - if err != nil { - return "", err - } - err = binary.Read(reader, binary.LittleEndian, &guid.Data3) - if err != nil { - return "", err - } - err = binary.Read(reader, binary.LittleEndian, &guid.Data4) - if err != nil { - return "", err - } - - guidStr, err := StringFromGUID(guid) - if err != nil { - return "", err - } - - return guidStr, nil -} - -// StringFromGUID returns a displayable GUID string from the GUID struct. -func StringFromGUID(guid *syscall.GUID) (string, error) { - if guid == nil { - return "", nil +// evtRenderProviderName renders the ProviderName of an event. +func evtRenderProviderName(renderBuf []byte, eventHandle EvtHandle) (string, error) { + var bufferUsed, propertyCount uint32 + err := _EvtRender(providerNameContext, eventHandle, EvtRenderEventValues, + uint32(len(renderBuf)), &renderBuf[0], &bufferUsed, + &propertyCount) + if err == ERROR_INSUFFICIENT_BUFFER { + return "", sys.InsufficientBufferError{err, int(bufferUsed)} } - - buf := make([]uint16, 40) - err := _StringFromGUID2(guid, &buf[0], uint32(len(buf))) if err != nil { - return "", err + return "", fmt.Errorf("evtRenderProviderName %v", err) } - return syscall.UTF16ToString(buf), nil + reader := bytes.NewReader(renderBuf) + return readString(renderBuf, reader) } diff --git a/winlogbeat/sys/wineventlog/zsyscall_windows.go b/winlogbeat/sys/wineventlog/zsyscall_windows.go index 540cbf8deef..c2c081e44dd 100644 --- a/winlogbeat/sys/wineventlog/zsyscall_windows.go +++ b/winlogbeat/sys/wineventlog/zsyscall_windows.go @@ -77,8 +77,8 @@ func _EvtUpdateBookmark(bookmark EvtHandle, event EvtHandle) (err error) { return } -func _EvtCreateRenderContext(ValuePathsCount uint32, valuePaths *uint16, flags EvtRenderContextFlag) (handle EvtHandle, err error) { - r0, _, e1 := syscall.Syscall(procEvtCreateRenderContext.Addr(), 3, uintptr(ValuePathsCount), uintptr(unsafe.Pointer(valuePaths)), uintptr(flags)) +func _EvtCreateRenderContext(ValuePathsCount uint32, valuePaths uintptr, flags EvtRenderContextFlag) (handle EvtHandle, err error) { + r0, _, e1 := syscall.Syscall(procEvtCreateRenderContext.Addr(), 3, uintptr(ValuePathsCount), uintptr(valuePaths), uintptr(flags)) handle = EvtHandle(r0) if handle == 0 { if e1 != 0 { diff --git a/winlogbeat/tests/system/test_eventlog.py b/winlogbeat/tests/system/test_eventlog.py index beb395c4d95..dcc3747431d 100644 --- a/winlogbeat/tests/system/test_eventlog.py +++ b/winlogbeat/tests/system/test_eventlog.py @@ -13,7 +13,6 @@ Contains tests for reading from the Windows Event Log (both APIs). """ - class Test(BaseTest): providerName = "WinlogbeatTestPython" applicationName = "SystemTest" @@ -58,11 +57,41 @@ def get_sid(self): def get_sid_string(self): if self.sidString == None: - self.sidString = win32security.ConvertSidToStringSid( - self.get_sid()) + self.sidString = win32security.ConvertSidToStringSid(self.get_sid()) return self.sidString + def assert_common_fields(self, evt, api, msg=None, eventID=10, sid=None, extra=None): + assert evt["computer_name"].lower() == win32api.GetComputerName().lower() + assert "record_number" in evt + self.assertDictContainsSubset({ + "count": 1, + "event_id": eventID, + "level": "Information", + "log_name": self.providerName, + "source_name": self.applicationName, + "type": api, + }, evt) + + if msg == None: + assert "message" not in evt + else: + self.assertEquals(evt["message"], msg) + self.assertDictContainsSubset({"event_data.param1": msg}, evt) + + if sid == None: + self.assertEquals(evt["user.identifier"], self.get_sid_string()) + self.assertEquals(evt["user.name"].lower(), win32api.GetUserName().lower()) + self.assertEquals(evt["user.type"], "User") + assert "user.domain" in evt + else: + self.assertEquals(evt["user.identifier"], sid) + assert "user.name" not in evt + assert "user.type" not in evt + + if extra != None: + self.assertDictContainsSubset(extra, evt) + @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") def test_eventlogging_read_one_event(self): """ @@ -75,12 +104,15 @@ def test_wineventlog_read_one_event(self): """ Win Event Log - Read one event """ - self.read_one_event("wineventlog") + evt = self.read_one_event("wineventlog") + self.assertDictContainsSubset({ + "keywords": ["Classic"], + "opcode": "Info", + }, evt) def read_one_event(self, api): msg = "Read One Event Testcase" - eventID = 11 - self.write_event_log(msg, eventID) + self.write_event_log(msg) # Run Winlogbeat self.render_config_template( @@ -96,19 +128,7 @@ def read_one_event(self, api): events = self.read_output() assert len(events) == 1 evt = events[0] - assert evt["type"] == api - assert evt["event_id"] == eventID - assert evt["level"] == "Information" - assert evt["log_name"] == self.providerName - assert evt["source_name"] == self.applicationName - assert evt["computer_name"].lower( - ) == win32api.GetComputerName().lower() - assert evt["user.identifier"] == self.get_sid_string() - assert evt["user.name"] == win32api.GetUserName() - assert "user.type" in evt - assert "user.domain" in evt - assert evt["message"] == msg - + self.assert_common_fields(evt, api, msg) return evt @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") @@ -118,7 +138,6 @@ def test_eventlogging_read_unknown_event_id(self): """ evt = self.read_unknown_event_id("eventlogging") - assert "message_inserts" in evt assert evt["message_error"].lower() == ("The system cannot find " "message text for message number 1111 in the message file for " "C:\\Windows\\system32\\EventCreate.exe.").lower() @@ -129,11 +148,14 @@ def test_wineventlog_read_unknown_event_id(self): Win Event Log - Read unknown event ID """ evt = self.read_unknown_event_id("wineventlog") + self.assertDictContainsSubset({ + "keywords": ["Classic"], + "opcode": "Info", + }, evt) - # TODO: messageInserts has not been implemented for wineventlog. - # assert "messageInserts" in evt - assert evt["message_error"] == ("the message resource is present but " - "the message is not found in the string/message table") + # No rendering error is being given. + #assert evt["message_error"] == ("the message resource is present but " + # "the message is not found in the string/message table") def read_unknown_event_id(self, api): msg = "Unknown Event ID Testcase" @@ -154,19 +176,7 @@ def read_unknown_event_id(self, api): events = self.read_output() assert len(events) == 1 evt = events[0] - assert evt["type"] == api - assert evt["event_id"] == eventID - assert evt["level"] == "Information" - assert evt["log_name"] == self.providerName - assert evt["source_name"] == self.applicationName - assert evt["computer_name"].lower( - ) == win32api.GetComputerName().lower() - assert evt["user.identifier"] == self.get_sid_string() - assert evt["user.name"] == win32api.GetUserName() - assert "user.type" in evt - assert "user.domain" in evt - assert "message" not in evt - + self.assert_common_fields(evt, api, None, eventID=1111, extra={"event_data.param1": msg}) return evt @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") @@ -181,7 +191,11 @@ def test_wineventlog_read_unknown_sid(self): """ Win Event Log - Read event with unknown SID """ - self.read_unknown_sid("wineventlog") + evt = self.read_unknown_sid("wineventlog") + self.assertDictContainsSubset({ + "keywords": ["Classic"], + "opcode": "Info", + }, evt) def read_unknown_sid(self, api): # Fake SID that was made up. @@ -189,8 +203,7 @@ def read_unknown_sid(self, api): sid = win32security.ConvertStringSidToSid(accountIdentifier) msg = "Unknown SID of " + accountIdentifier - eventID = 40 - self.write_event_log(msg, eventID, sid) + self.write_event_log(msg, sid=sid) # Run Winlogbeat self.render_config_template( @@ -206,19 +219,7 @@ def read_unknown_sid(self, api): events = self.read_output() assert len(events) == 1 evt = events[0] - assert evt["type"] == api - assert evt["event_id"] == eventID - assert evt["level"] == "Information" - assert evt["log_name"] == self.providerName - assert evt["source_name"] == self.applicationName - assert evt["computer_name"].lower( - ) == win32api.GetComputerName().lower() - assert evt["user.identifier"] == accountIdentifier - assert "user.name" not in evt - assert "user.type" not in evt - assert "user.domain" not in evt - assert evt["message"] == msg - + self.assert_common_fields(evt, api, msg, sid=accountIdentifier) return evt @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") @@ -267,6 +268,7 @@ def fields_under_root(self, api): "local": "field", "tags": ["global", "local"], }, evt) + return evt @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") def test_eventlogging_fields_not_under_root(self): @@ -311,3 +313,4 @@ def fields_not_under_root(self, api): "fields.local": "field", "fields.num": 1, }, evt) + return evt