diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d479054b1ee..6a758a2959d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -377,6 +377,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Simplify regex for organization custom prefix in AWS/CloudTrail fileset. {issue}23203[23203] {pull}23204[23204] - Fix syslog header parsing in infoblox module. {issue}23272[23272] {pull}23273[23273] - Fix concurrent modification exception in Suricata ingest node pipeline. {pull}23534[23534] +- Fix Zoom module parameters for basic auth and url path. {pull}23779[23779] +- Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777] *Heartbeat* @@ -619,6 +621,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add ECS categorization info for auditd module {pull}18596[18596] - Add several improvements for auditd module for improved ECS field mapping {pull}22647[22647] - Add ECS 1.7 `configuration` categorization in certain events in auditd module. {pull}23000[23000] +- Improve file_integrity monitoring when a file is created/deleted in quick succession. {issue}17347[17347] {pull}22170[22170] - system/host: Add new ECS 1.8 field `os.type` in `host.os.type`. {pull}23513[23513] *Filebeat* @@ -831,6 +834,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `encode_as` and `decode_as` options to httpjson along with pluggable encoders/decoders {pull}23478[23478] - Added `application/x-ndjson` as decode option for httpjson input {pull}23521[23521] - Added `application/x-www-form-urlencoded` as encode option for httpjson input {pull}23521[23521] +- Added RFC6587 framing option for tcp and unix inputs {issue}23663[23663] {pull}23724[23724] *Heartbeat* diff --git a/auditbeat/module/file_integrity/config.go b/auditbeat/module/file_integrity/config.go index c280fb52d0e..39882fe3e0f 100644 --- a/auditbeat/module/file_integrity/config.go +++ b/auditbeat/module/file_integrity/config.go @@ -18,6 +18,7 @@ package file_integrity import ( + "math" "path/filepath" "sort" "strings" @@ -29,6 +30,9 @@ import ( "github.com/elastic/beats/v7/libbeat/common/match" ) +// MaxValidFileSizeLimit is the largest possible value for `max_file_size`. +const MaxValidFileSizeLimit = math.MaxInt64 - 1 + // HashType identifies a cryptographic algorithm. type HashType string @@ -110,7 +114,7 @@ nextHash: } c.MaxFileSizeBytes, err = humanize.ParseBytes(c.MaxFileSize) - if err != nil { + if err != nil || c.MaxFileSizeBytes > MaxValidFileSizeLimit { errs = append(errs, errors.Wrap(err, "invalid max_file_size value")) } else if c.MaxFileSizeBytes <= 0 { errs = append(errs, errors.Errorf("max_file_size value (%v) must be positive", c.MaxFileSize)) diff --git a/auditbeat/module/file_integrity/event.go b/auditbeat/module/file_integrity/event.go index 1ee28b7ce35..f3d67d74ffc 100644 --- a/auditbeat/module/file_integrity/event.go +++ b/auditbeat/module/file_integrity/event.go @@ -27,6 +27,7 @@ import ( "fmt" "hash" "io" + "math" "os" "path/filepath" "runtime" @@ -119,8 +120,9 @@ type Event struct { Hashes map[HashType]Digest `json:"hash,omitempty"` // File hashes. // Metadata - rtt time.Duration // Time taken to collect the info. - errors []error // Errors that occurred while collecting the info. + rtt time.Duration // Time taken to collect the info. + errors []error // Errors that occurred while collecting the info. + hashFailed bool // Set when hashing the file failed. } // Metadata contains file metadata. @@ -183,11 +185,16 @@ func NewEventFromFileInfo( switch event.Info.Type { case FileType: if event.Info.Size <= maxFileSize { - hashes, err := hashFile(event.Path, hashTypes...) + hashes, nbytes, err := hashFile(event.Path, maxFileSize, hashTypes...) if err != nil { event.errors = append(event.errors, err) - } else { + event.hashFailed = true + } else if hashes != nil { + // hashFile returns nil hashes and no error when: + // - There's no hashes configured. + // - File size at the time of hashing is larger than configured limit. event.Hashes = hashes + event.Info.Size = nbytes } } case SymlinkType: @@ -319,6 +326,17 @@ func buildMetricbeatEvent(e *Event, existedBefore bool) mb.Event { out.MetricSetFields.Put("event.type", None.ECSTypes()) } + if n := len(e.errors); n > 0 { + errors := make([]string, n) + for idx, err := range e.errors { + errors[idx] = err.Error() + } + if n == 1 { + out.MetricSetFields.Put("error.message", errors[0]) + } else { + out.MetricSetFields.Put("error.message", errors) + } + } return out } @@ -327,7 +345,7 @@ func buildMetricbeatEvent(e *Event, existedBefore bool) mb.Event { // contains a superset of new's hashes then false is returned. func diffEvents(old, new *Event) (Action, bool) { if old == new { - return 0, false + return None, false } if old == nil && new != nil { @@ -389,9 +407,9 @@ func diffEvents(old, new *Event) (Action, bool) { return result, result != None } -func hashFile(name string, hashType ...HashType) (map[HashType]Digest, error) { +func hashFile(name string, maxSize uint64, hashType ...HashType) (nameToHash map[HashType]Digest, nbytes uint64, err error) { if len(hashType) == 0 { - return nil, nil + return nil, 0, nil } var hashes []hash.Hash @@ -433,27 +451,40 @@ func hashFile(name string, hashType ...HashType) (map[HashType]Digest, error) { case XXH64: hashes = append(hashes, xxhash.New()) default: - return nil, errors.Errorf("unknown hash type '%v'", name) + return nil, 0, errors.Errorf("unknown hash type '%v'", name) } } f, err := file.ReadOpen(name) if err != nil { - return nil, errors.Wrap(err, "failed to open file for hashing") + return nil, 0, errors.Wrap(err, "failed to open file for hashing") } defer f.Close() hashWriter := multiWriter(hashes) - if _, err := io.Copy(hashWriter, f); err != nil { - return nil, errors.Wrap(err, "failed to calculate file hashes") + // Make sure it hashes up to the limit in case the file is growing + // since its size was checked. + validSizeLimit := maxSize < math.MaxInt64-1 + var r io.Reader = f + if validSizeLimit { + r = io.LimitReader(r, int64(maxSize+1)) + } + written, err := io.Copy(hashWriter, r) + if err != nil { + return nil, 0, errors.Wrap(err, "failed to calculate file hashes") + } + + // The file grew larger than configured limit. + if validSizeLimit && written > int64(maxSize) { + return nil, 0, nil } - nameToHash := make(map[HashType]Digest, len(hashes)) + nameToHash = make(map[HashType]Digest, len(hashes)) for i, h := range hashes { nameToHash[hashType[i]] = h.Sum(nil) } - return nameToHash, nil + return nameToHash, uint64(written), nil } func multiWriter(hash []hash.Hash) io.Writer { diff --git a/auditbeat/module/file_integrity/event_test.go b/auditbeat/module/file_integrity/event_test.go index efaafd02041..dd57c590a41 100644 --- a/auditbeat/module/file_integrity/event_test.go +++ b/auditbeat/module/file_integrity/event_test.go @@ -22,6 +22,7 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math" "os" "runtime" "testing" @@ -172,6 +173,18 @@ func TestDiffEvents(t *testing.T) { } func TestHashFile(t *testing.T) { + f, err := ioutil.TempFile("", "input.txt") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + const data = "hello world!\n" + const dataLen = uint64(len(data)) + f.WriteString(data) + f.Sync() + f.Close() + t.Run("valid hashes", func(t *testing.T) { // Computed externally. expectedHashes := map[HashType]Digest{ @@ -193,21 +206,11 @@ func TestHashFile(t *testing.T) { XXH64: mustDecodeHex("d3e8573b7abf279a"), } - f, err := ioutil.TempFile("", "input.txt") + hashes, size, err := hashFile(f.Name(), dataLen, validHashes...) if err != nil { t.Fatal(err) } - defer os.Remove(f.Name()) - - f.WriteString("hello world!\n") - f.Sync() - f.Close() - - hashes, err := hashFile(f.Name(), validHashes...) - if err != nil { - t.Fatal(err) - } - + assert.Equal(t, dataLen, size) for _, hashType := range validHashes { if hash, found := hashes[hashType]; !found { t.Errorf("%v not found", hashType) @@ -228,21 +231,107 @@ func TestHashFile(t *testing.T) { }) t.Run("no hashes", func(t *testing.T) { - hashes, err := hashFile("anyfile.txt") + hashes, size, err := hashFile("anyfile.txt", 1234) assert.Nil(t, hashes) assert.NoError(t, err) + assert.Zero(t, size) }) t.Run("invalid hash", func(t *testing.T) { - hashes, err := hashFile("anyfile.txt", "md4") + hashes, size, err := hashFile("anyfile.txt", 1234, "md4") assert.Nil(t, hashes) assert.Error(t, err) + assert.Zero(t, size) }) t.Run("invalid file", func(t *testing.T) { - hashes, err := hashFile("anyfile.txt", "md5") + hashes, size, err := hashFile("anyfile.txt", 1234, "md5") assert.Nil(t, hashes) assert.Error(t, err) + assert.Zero(t, size) + }) + + t.Run("size over hash limit", func(t *testing.T) { + hashes, size, err := hashFile(f.Name(), dataLen-1, SHA1) + assert.Nil(t, hashes) + assert.Zero(t, size) + assert.NoError(t, err) + }) + t.Run("size at hash limit", func(t *testing.T) { + hashes, size, err := hashFile(f.Name(), dataLen, SHA1) + assert.NotNil(t, hashes) + assert.Equal(t, dataLen, size) + assert.NoError(t, err) + }) + t.Run("size below hash limit", func(t *testing.T) { + hashes, size, err := hashFile(f.Name(), dataLen+1, SHA1) + assert.NotNil(t, hashes) + assert.Equal(t, dataLen, size) + assert.NoError(t, err) + }) + t.Run("no size limit", func(t *testing.T) { + hashes, size, err := hashFile(f.Name(), math.MaxInt64, SHA1) + assert.NotNil(t, hashes) + assert.Equal(t, dataLen, size) + assert.NoError(t, err) + }) +} + +func TestNewEventFromFileInfoHash(t *testing.T) { + f, err := ioutil.TempFile("", "input.txt") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + const data = "hello world!\n" + const dataLen = uint64(len(data)) + f.WriteString(data) + f.Sync() + defer f.Close() + + info, err := os.Stat(f.Name()) + if err != nil { + t.Fatal(err) + } + + t.Run("file stays the same", func(t *testing.T) { + ev := NewEventFromFileInfo(f.Name(), info, nil, Updated, SourceFSNotify, MaxValidFileSizeLimit, []HashType{SHA1}) + if !assert.NotNil(t, ev) { + t.Fatal("nil event") + } + assert.Equal(t, dataLen, ev.Info.Size) + assert.NotNil(t, ev.Hashes) + digest := Digest(mustDecodeHex("f951b101989b2c3b7471710b4e78fc4dbdfa0ca6")) + assert.Equal(t, digest, ev.Hashes[SHA1]) + }) + t.Run("file grows before hashing", func(t *testing.T) { + f.WriteString(data) + f.Sync() + ev := NewEventFromFileInfo(f.Name(), info, nil, Updated, SourceFSNotify, MaxValidFileSizeLimit, []HashType{SHA1}) + if !assert.NotNil(t, ev) { + t.Fatal("nil event") + } + assert.Equal(t, dataLen*2, ev.Info.Size) + assert.NotNil(t, ev.Hashes) + digest := Digest(mustDecodeHex("62e8a0ef77ed7596347a065cae28a860f87e382f")) + assert.Equal(t, digest, ev.Hashes[SHA1]) + }) + t.Run("file shrinks before hashing", func(t *testing.T) { + err = f.Truncate(0) + if !assert.NoError(t, err) { + t.Fatal(err) + } + f.Sync() + assert.NoError(t, err) + ev := NewEventFromFileInfo(f.Name(), info, nil, Updated, SourceFSNotify, MaxValidFileSizeLimit, []HashType{SHA1}) + if !assert.NotNil(t, ev) { + t.Fatal("nil event") + } + assert.Zero(t, ev.Info.Size) + assert.NotNil(t, ev.Hashes) + digest := Digest(mustDecodeHex("da39a3ee5e6b4b0d3255bfef95601890afd80709")) + assert.Equal(t, digest, ev.Hashes[SHA1]) }) } @@ -254,13 +343,14 @@ func BenchmarkHashFile(b *testing.B) { defer os.Remove(f.Name()) zeros := make([]byte, 100) - iterations := 1024 * 1024 // 100 MiB + const iterations = 1024 * 1024 // 100 MiB for i := 0; i < iterations; i++ { if _, err = f.Write(zeros); err != nil { b.Fatal(err) } } - b.Logf("file size: %v bytes", len(zeros)*iterations) + size := uint64(iterations * len(zeros)) + b.Logf("file size: %v bytes", size) f.Sync() f.Close() b.ResetTimer() @@ -268,10 +358,11 @@ func BenchmarkHashFile(b *testing.B) { for _, hashType := range validHashes { b.Run(string(hashType), func(b *testing.B) { for i := 0; i < b.N; i++ { - _, err = hashFile(f.Name(), hashType) + _, nbytes, err := hashFile(f.Name(), size+1, hashType) if err != nil { b.Fatal(err) } + assert.Equal(b, size, nbytes) } }) } diff --git a/auditbeat/module/file_integrity/fileinfo_other_test.go b/auditbeat/module/file_integrity/fileinfo_other_test.go new file mode 100644 index 00000000000..d4acbb1781b --- /dev/null +++ b/auditbeat/module/file_integrity/fileinfo_other_test.go @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !windows + +package file_integrity + +import ( + "os" + "testing" +) + +func makeFileNonReadable(t testing.TB, path string) { + if err := os.Chmod(path, 0); err != nil { + t.Fatal(err) + } +} diff --git a/auditbeat/module/file_integrity/fileinfo_windows.go b/auditbeat/module/file_integrity/fileinfo_windows.go index 9a61ce19cdb..7c3ff0e3fdf 100644 --- a/auditbeat/module/file_integrity/fileinfo_windows.go +++ b/auditbeat/module/file_integrity/fileinfo_windows.go @@ -41,6 +41,8 @@ func NewMetadata(path string, info os.FileInfo) (*Metadata, error) { return nil, errors.Errorf("unexpected fileinfo sys type %T for %v", info.Sys(), path) } + var errs multierror.Errors + state := file.GetOSState(info) fileInfo := &Metadata{ @@ -65,24 +67,27 @@ func NewMetadata(path string, info os.FileInfo) (*Metadata, error) { // FILE_FLAG_BACKUP_SEMANTICS flag. var err error if !info.IsDir() { - fileInfo.SID, fileInfo.Owner, err = fileOwner(path) + if fileInfo.SID, fileInfo.Owner, err = fileOwner(path); err != nil { + errs = append(errs, errors.Wrap(err, "fileOwner failed")) + } + } - fileInfo.Origin, err = GetFileOrigin(path) - return fileInfo, err + if fileInfo.Origin, err = GetFileOrigin(path); err != nil { + errs = append(errs, errors.Wrap(err, "GetFileOrigin failed")) + } + return fileInfo, errs.Err() } // fileOwner returns the SID and name (domain\user) of the file's owner. func fileOwner(path string) (sid, owner string, err error) { - f, err := file.ReadOpen(path) - if err != nil { - return "", "", errors.Wrap(err, "failed to open file to get owner") - } - defer f.Close() - var securityID *syscall.SID var securityDescriptor *SecurityDescriptor - if err = GetSecurityInfo(syscall.Handle(f.Fd()), FileObject, + pathW, err := syscall.UTF16PtrFromString(path) + if err != nil { + return sid, owner, errors.Wrapf(err, "failed to convert path:'%s' to UTF16", path) + } + if err = GetNamedSecurityInfo(pathW, FileObject, OwnerSecurityInformation, &securityID, nil, nil, nil, &securityDescriptor); err != nil { return "", "", errors.Wrapf(err, "failed on GetSecurityInfo for %v", path) } diff --git a/auditbeat/module/file_integrity/fileinfo_windows_test.go b/auditbeat/module/file_integrity/fileinfo_windows_test.go new file mode 100644 index 00000000000..1b64c644778 --- /dev/null +++ b/auditbeat/module/file_integrity/fileinfo_windows_test.go @@ -0,0 +1,61 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package file_integrity + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/hectane/go-acl" + "github.com/stretchr/testify/assert" +) + +// TestFileInfoPermissions tests obtaining metadata of a file +// when we don't have permissions to open the file for reading. +// This prevents us to get the file owner of a file unless we use +// a method that doesn't need to open the file for reading. +// (GetNamedSecurityInfo vs CreateFile+GetSecurityInfo) +func TestFileInfoPermissions(t *testing.T) { + f, err := ioutil.TempFile("", "metadata") + if err != nil { + t.Fatal(err) + } + name := f.Name() + defer func() { + f.Close() + os.Remove(f.Name()) + }() + makeFileNonReadable(t, f.Name()) + info, err := os.Stat(name) + if err != nil { + t.Fatal(err) + } + meta, err := NewMetadata(name, info) + if !assert.NoError(t, err) { + t.Fatal(err) + } + t.Log(meta.Owner) + assert.NotEqual(t, "", meta.Owner) +} + +func makeFileNonReadable(t testing.TB, path string) { + if err := acl.Chmod(path, 0); err != nil { + t.Fatal(err) + } +} diff --git a/auditbeat/module/file_integrity/metricset.go b/auditbeat/module/file_integrity/metricset.go index eaa40f34e43..cd836f472d5 100644 --- a/auditbeat/module/file_integrity/metricset.go +++ b/auditbeat/module/file_integrity/metricset.go @@ -75,6 +75,9 @@ type MetricSet struct { scanStart time.Time scanChan <-chan Event fsnotifyChan <-chan Event + + // Used when a hash can't be calculated + nullHashes map[HashType]Digest } // New returns a new file.MetricSet. @@ -96,6 +99,13 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { log: logp.NewLogger(moduleName), } + ms.nullHashes = make(map[HashType]Digest, len(config.HashTypes)) + for _, hashType := range ms.config.HashTypes { + // One byte is enough so that the hashes are persisted to the datastore. + // The comparison function doesn't care if the lengths are not the expected + // for the given algorithms. + ms.nullHashes[hashType] = Digest{0x00} + } ms.log.Debugf("Initialized the file event reader. Running as euid=%v", os.Geteuid()) return ms, nil @@ -230,6 +240,19 @@ func (ms *MetricSet) reportEvent(reporter mb.PushReporterV2, event *Event) bool ms.log.Errorw("Failed during DB delete", "error", err) } } else { + if event.hashFailed { + // If hashing failed, persist the previous hashes, so it can detect + // a future change to the file. Otherwise the next update event will + // be reported as a config change. + // Hashing usually fails while the file is being updated under Windows + // if open in exclusive mode, and succeeds once the file is closed + // and its mtime is updated. + if lastEvent != nil { + event.Hashes = lastEvent.Hashes + } else { + event.Hashes = ms.nullHashes + } + } if err := store(ms.bucket, event); err != nil { ms.log.Errorw("Failed during DB store", "error", err) } @@ -245,15 +268,30 @@ func (ms *MetricSet) hasFileChangedSinceLastEvent(event *Event) (changed bool, l return true, lastEvent } + // Received a deleted event but the file now exists on disk (already re-created). + if event.Action&Deleted != 0 && event.Info != nil { + event.Action &= ^Action(Deleted) + event.Action |= Updated + } + // We receive a creation event for a deletion that we didn't observe due to the above. + if event.Action&Created != 0 && lastEvent != nil && lastEvent.Info != nil { + event.Action &= ^Action(Created) + event.Action |= Updated + } action, changed := diffEvents(lastEvent, event) - if event.Action == 0 { - event.Action = action + if uint8(event.Action)&^uint8(Updated) == 0 { + if event.hashFailed && !changed { + event.Action = Updated + } else { + event.Action = action + } + changed = event.Action != None } - if changed { ms.log.Debugw("File changed since it was last seen", "file_path", event.Path, "took", event.rtt, - logp.Namespace("event"), "old", lastEvent, "new", event) + logp.Namespace("event"), "action", event.Action, + "old", lastEvent, "new", event) } return changed, lastEvent } diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go index ab1b5ec40e6..841056f7aa1 100644 --- a/auditbeat/module/file_integrity/metricset_test.go +++ b/auditbeat/module/file_integrity/metricset_test.go @@ -22,7 +22,9 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" "runtime" + "strings" "testing" "time" @@ -31,6 +33,7 @@ import ( "github.com/elastic/beats/v7/auditbeat/core" "github.com/elastic/beats/v7/auditbeat/datastore" abtest "github.com/elastic/beats/v7/auditbeat/testing" + "github.com/elastic/beats/v7/metricbeat/mb" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) @@ -167,6 +170,7 @@ func TestActions(t *testing.T) { assert.Contains(t, actions, "created") case updatedFilepath: assert.Contains(t, actions, "updated") + assert.Contains(t, actions, "attributes_modified") default: assert.Fail(t, "unexpected path", "path %v", path) } @@ -293,6 +297,688 @@ func TestIncludedExcludedFiles(t *testing.T) { assert.Equal(t, wanted, got) } +func TestErrorReporting(t *testing.T) { + if runtime.GOOS == "darwin" { + // FSEvents doesn't generate write events during this test, + // either it needs the file to be closed before generating + // events or the non-readable permissions trick it somehow. + t.Skip("Skip this test on Darwin") + } + if runtime.GOOS != "windows" && os.Getuid() == 0 { + // There's no easy way to make a file unreadable by root + // in UNIX/Linux OS. + t.Skip("This test can't be run as root") + } + defer abtest.SetupDataDir(t)() + + dir, err := ioutil.TempDir("", "audit-file") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + dir, err = filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + + path := filepath.Join(dir, "unreadable.txt") + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + defer f.Close() + + makeFileNonReadable(t, path) + + config := getConfig(dir) + config["scan_at_start"] = false + ms := mbtest.NewPushMetricSetV2(t, config) + + done := make(chan struct{}, 1) + go func() { + for { + f.WriteString("can't read this\n") + f.Sync() + select { + case <-done: + return + default: + time.Sleep(time.Second / 10) + } + } + }() + + events := mbtest.RunPushMetricSetV2(10*time.Second, 10, ms) + close(done) + + getField := func(ev *mb.Event, field string) interface{} { + v, _ := ev.MetricSetFields.GetValue(field) + return v + } + match := func(ev *mb.Event) bool { + return ev != nil && + reflect.DeepEqual(getField(ev, "event.type"), []string{"change"}) && + getField(ev, "file.type") == "file" && + getField(ev, "file.extension") == "txt" + } + + var event *mb.Event + for idx, ev := range events { + t.Log("event[", idx, "] = ", ev) + if match(&ev) { + event = &ev + break + } + } + + if !assert.NotNil(t, event) { + t.Fatal("target event not found") + } + + if event.Error != nil { + t.Fatalf("received error: %+v", event.Error) + } + + errors := getField(event, "error.message") + if !assert.NotNil(t, errors) { + t.Fatal("no error.message in event") + } + + var errList []string + switch v := errors.(type) { + case string: + errList = []string{v} + case []interface{}: + for _, val := range v { + str, ok := val.(string) + if !ok { + t.Fatalf("Unexpected type %T in error.message list: %v", val, val) + } + errList = append(errList, str) + } + + case []string: + errList = v + + default: + t.Fatalf("Unexpected type %T in error.message: %v", v, v) + } + + found := false + assert.NotEmpty(t, errList) + for _, msg := range errList { + if strings.Contains(msg, "hashing") { + found = true + break + } + } + assert.True(t, found) +} + +type testReporter struct { + events []mb.Event + errors []error +} + +func (t *testReporter) Event(event mb.Event) bool { + t.events = append(t.events, event) + return true +} + +func (t *testReporter) Error(err error) bool { + t.errors = append(t.errors, err) + return true +} + +func (t *testReporter) Done() <-chan struct{} { + return nil +} + +func (t *testReporter) Clear() { + t.events = nil + t.errors = nil +} + +func checkExpectedEvent(t *testing.T, ms *MetricSet, title string, input *Event, expected map[string]interface{}) { + var reporter testReporter + if !ms.reportEvent(&reporter, input) { + t.Fatal("reportEvent failed", title) + } + if !assert.Empty(t, reporter.errors, title) { + t.Fatal("errors during reportEvent", reporter.errors, title) + } + if expected == nil { + assert.Empty(t, reporter.events) + return + } + if !assert.NotEmpty(t, reporter.events) { + t.Fatal("no event received", title) + } + if !assert.Len(t, reporter.events, 1) { + t.Fatal("more than one event received", title) + } + ev := reporter.events[0] + t.Log("got title=", title, "event=", ev) + for k, v := range expected { + iface, err := ev.MetricSetFields.GetValue(k) + if v == nil { + assert.Error(t, err, title) + continue + } + if err != nil { + t.Fatal("failed to fetch key", k, title) + } + assert.Equal(t, v, iface, title) + } +} + +type expectedEvent struct { + title string + input Event + expected map[string]interface{} +} + +func (e expectedEvent) validate(t *testing.T, ms *MetricSet) { + checkExpectedEvent(t, ms, e.title, &e.input, e.expected) +} + +type expectedEvents []expectedEvent + +func (e expectedEvents) validate(t *testing.T) { + store, err := ioutil.TempFile("", "bucket") + if err != nil { + t.Fatal(err) + } + defer store.Close() + defer os.Remove(store.Name()) + ds := datastore.New(store.Name(), 0644) + bucket, err := ds.OpenBucket(bucketName) + if err != nil { + t.Fatal(err) + } + defer bucket.Close() + config := getConfig("somepath") + config["hash_types"] = []string{"sha1"} + ms, ok := mbtest.NewPushMetricSetV2(t, config).(*MetricSet) + if !assert.True(t, ok) { + t.Fatal("can't create metricset") + } + ms.bucket = bucket.(datastore.BoltBucket) + for _, ev := range e { + ev.validate(t, ms) + } +} + +func TestEventFailedHash(t *testing.T) { + baseTime := time.Now() + t.Run("failed hash on update", func(t *testing.T) { + expectedEvents{ + expectedEvent{ + title: "creation event", + input: Event{ + Timestamp: baseTime, + Path: "/some/path", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Action: Created, + Source: SourceFSNotify, + Hashes: map[HashType]Digest{ + SHA1: []byte("11111111111111111111"), + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"created"}, + "event.type": []string{"creation"}, + "file.hash.sha1": Digest("11111111111111111111"), + }, + }, + expectedEvent{ + title: "update with hash", + input: Event{ + Timestamp: time.Now(), + Path: "/some/path", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Source: SourceFSNotify, + Action: Updated, + Hashes: map[HashType]Digest{ + SHA1: []byte("22222222222222222222"), + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"updated"}, + "event.type": []string{"change"}, + "file.hash.sha1": Digest("22222222222222222222"), + }, + }, + expectedEvent{ + title: "update with failed hash", + input: Event{ + Timestamp: time.Now(), + Path: "/some/path", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Source: SourceFSNotify, + Action: Updated, + hashFailed: true, + }, + expected: map[string]interface{}{ + "event.action": []string{"updated"}, + "event.type": []string{"change"}, + "file.hash.sha1": nil, + }, + }, + expectedEvent{ + title: "update again now with hash", + input: Event{ + Timestamp: time.Now(), + Path: "/some/path", + Info: &Metadata{ + CTime: baseTime, + MTime: baseTime, + Type: FileType, + }, + Source: SourceFSNotify, + Action: Updated, + Hashes: map[HashType]Digest{ + SHA1: []byte("33333333333333333333"), + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"updated"}, + "event.type": []string{"change"}, + "file.hash.sha1": Digest("33333333333333333333"), + }, + }, + expectedEvent{ + title: "new modification time", + input: Event{ + Timestamp: time.Now(), + Path: "/some/path", + Info: &Metadata{ + CTime: baseTime, + MTime: baseTime.Add(time.Second), + Type: FileType, + }, + Source: SourceFSNotify, + Action: Updated, + Hashes: map[HashType]Digest{ + SHA1: []byte("33333333333333333333"), + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"attributes_modified"}, + "event.type": []string{"change"}, + "file.hash.sha1": Digest("33333333333333333333"), + }, + }, + }.validate(t) + }) + t.Run("failed hash on creation", func(t *testing.T) { + expectedEvents{ + expectedEvent{ + title: "creation event with failed hash", + input: Event{ + Timestamp: baseTime, + Path: "/some/other/path", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Action: Created, + Source: SourceFSNotify, + hashFailed: true, + }, + expected: map[string]interface{}{ + "event.action": []string{"created"}, + "event.type": []string{"creation"}, + "file.hash.sha1": nil, + }, + }, + expectedEvent{ + title: "update with hash", + input: Event{ + Timestamp: time.Now(), + Path: "/some/other/path", + Info: &Metadata{ + MTime: baseTime.Add(time.Second), + CTime: baseTime, + Type: FileType, + }, + Source: SourceFSNotify, + Action: Updated, + Hashes: map[HashType]Digest{ + SHA1: []byte("22222222222222222222"), + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"updated", "attributes_modified"}, + "event.type": []string{"change"}, + "file.hash.sha1": Digest("22222222222222222222"), + }, + }, + }.validate(t) + }) + t.Run("delete", func(t *testing.T) { + expectedEvents{ + expectedEvent{ + title: "creation event", + input: Event{ + Timestamp: baseTime, + Path: "/some/other/path", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Action: Created, + Source: SourceFSNotify, + Hashes: map[HashType]Digest{ + SHA1: []byte("22222222222222222222"), + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"created"}, + "event.type": []string{"creation"}, + "file.hash.sha1": Digest("22222222222222222222"), + }, + }, + expectedEvent{ + title: "delete", + input: Event{ + Timestamp: time.Now(), + Path: "/some/other/path", + Info: nil, + Source: SourceFSNotify, + Action: Deleted, + Hashes: nil, + }, + expected: map[string]interface{}{ + "event.action": []string{"deleted"}, + "event.type": []string{"deletion"}, + "file.hash.sha1": nil, + }, + }, + }.validate(t) + }) + t.Run("move", func(t *testing.T) { + expectedEvents{ + expectedEvent{ + title: "creation event", + input: Event{ + Timestamp: baseTime, + Path: "/some/other/path", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Action: Created, + Source: SourceFSNotify, + Hashes: map[HashType]Digest{ + SHA1: []byte("22222222222222222222"), + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"created"}, + "event.type": []string{"creation"}, + "file.hash.sha1": Digest("22222222222222222222"), + }, + }, + expectedEvent{ + title: "delete", + input: Event{ + Timestamp: time.Now(), + Path: "/some/other/path", + Info: nil, + Source: SourceFSNotify, + // FSEvents likes to add extra flags to delete events. + Action: Moved, + Hashes: nil, + }, + expected: map[string]interface{}{ + "event.action": []string{"moved"}, + "event.type": []string{"change"}, + "file.hash.sha1": nil, + }, + }, + }.validate(t) + }) +} + +func TestEventDelete(t *testing.T) { + store, err := ioutil.TempFile("", "bucket") + if err != nil { + t.Fatal(err) + } + defer store.Close() + defer os.Remove(store.Name()) + ds := datastore.New(store.Name(), 0644) + bucket, err := ds.OpenBucket(bucketName) + if err != nil { + t.Fatal(err) + } + defer bucket.Close() + config := getConfig("somepath") + config["hash_types"] = []string{"sha1"} + ms, ok := mbtest.NewPushMetricSetV2(t, config).(*MetricSet) + if !assert.True(t, ok) { + t.Fatal("can't create metricset") + } + ms.bucket = bucket.(datastore.BoltBucket) + + baseTime := time.Now() + sha := Digest("22222222222222222222") + t.Run("delete event for file missing on disk", func(t *testing.T) { + expectedEvents{ + expectedEvent{ + title: "creation event", + input: Event{ + Timestamp: baseTime, + Path: "/file", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Action: Created, + Source: SourceFSNotify, + Hashes: map[HashType]Digest{ + SHA1: sha, + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"created"}, + "event.type": []string{"creation"}, + "file.hash.sha1": sha, + }, + }, + expectedEvent{ + title: "delete", + input: Event{ + Timestamp: time.Now(), + Path: "/file", + Source: SourceFSNotify, + Action: Deleted, + }, + expected: map[string]interface{}{ + "event.action": []string{"deleted"}, + "event.type": []string{"deletion"}, + }, + }, + expectedEvent{ + title: "creation event", + input: Event{ + Timestamp: baseTime, + Path: "/file", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Action: Created, + Source: SourceFSNotify, + Hashes: map[HashType]Digest{ + SHA1: sha, + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"created"}, + "event.type": []string{"creation"}, + "file.hash.sha1": sha, + }, + }, + }.validate(t) + }) + + // This tests getting a DELETE followed by a CREATE, but by the time we observe the former the file already + // exists on disk. + shaNext := Digest("22222222222222222223") + t.Run("delete event for file present on disk (different contents)", func(t *testing.T) { + expectedEvents{ + expectedEvent{ + title: "create", + input: Event{ + Timestamp: baseTime, + Path: "/file", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Action: Created, + Source: SourceFSNotify, + Hashes: map[HashType]Digest{ + SHA1: sha, + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"created"}, + "event.type": []string{"creation"}, + "file.hash.sha1": sha, + }, + }, + expectedEvent{ + title: "delete", + input: Event{ + Timestamp: time.Now(), + Path: "/file", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Source: SourceFSNotify, + Action: Deleted, + Hashes: map[HashType]Digest{ + SHA1: shaNext, + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"updated"}, + "event.type": []string{"change"}, + "file.hash.sha1": shaNext, + }, + }, + expectedEvent{ + title: "re-create", + input: Event{ + Timestamp: baseTime, + Path: "/file", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Action: Created, + Source: SourceFSNotify, + Hashes: map[HashType]Digest{ + SHA1: shaNext, + }, + }, + expected: nil, // Already observed during handling of previous event. + }, + }.validate(t) + }) + + t.Run("delete event for file present on disk (same contents)", func(t *testing.T) { + expectedEvents{ + expectedEvent{ + title: "create", + input: Event{ + Timestamp: baseTime, + Path: "/file", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Action: Created, + Source: SourceFSNotify, + Hashes: map[HashType]Digest{ + SHA1: sha, + }, + }, + expected: map[string]interface{}{ + "event.action": []string{"created"}, + "event.type": []string{"creation"}, + "file.hash.sha1": sha, + }, + }, + expectedEvent{ + title: "delete", + input: Event{ + Timestamp: time.Now(), + Path: "/file", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Source: SourceFSNotify, + Action: Deleted, + Hashes: map[HashType]Digest{ + SHA1: sha, + }, + }, + // No event because it has the same contents as before. + expected: nil, + }, + expectedEvent{ + title: "re-create", + input: Event{ + Timestamp: baseTime, + Path: "/file", + Info: &Metadata{ + MTime: baseTime, + CTime: baseTime, + Type: FileType, + }, + Action: Created, + Source: SourceFSNotify, + Hashes: map[HashType]Digest{ + SHA1: sha, + }, + }, + // No event because it has the same contents as before. + expected: nil, + }, + }.validate(t) + }) +} + func getConfig(path ...string) map[string]interface{} { return map[string]interface{}{ "module": "file_integrity", diff --git a/auditbeat/module/file_integrity/security_windows.go b/auditbeat/module/file_integrity/security_windows.go index 1ce4c86ec03..8bb512c6b8a 100644 --- a/auditbeat/module/file_integrity/security_windows.go +++ b/auditbeat/module/file_integrity/security_windows.go @@ -67,7 +67,7 @@ const ( // Use "GOOS=windows go generate -v -x ." to generate the source. // Add -trace to enable debug prints around syscalls. -//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsecurity_windows.go security_windows.go +//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsecurity_windows.go security_windows.go // Windows API calls -//sys GetSecurityInfo(handle syscall.Handle, objectType ObjectType, securityInformation SecurityInformation, ppsidOwner **syscall.SID, ppsidGroup **syscall.SID, ppDacl **ACL, ppSacl **ACL, ppSecurityDescriptor **SecurityDescriptor) (err error) [failretval!=0] = advapi32.GetSecurityInfo +//sys GetNamedSecurityInfo(name *uint16, objectType ObjectType, securityInformation SecurityInformation, ppsidOwner **syscall.SID, ppsidGroup **syscall.SID, ppDacl **ACL, ppSacl **ACL, ppSecurityDescriptor **SecurityDescriptor) (err error) [failretval!=0] = advapi32.GetNamedSecurityInfoW diff --git a/auditbeat/module/file_integrity/security_windows_test.go b/auditbeat/module/file_integrity/security_windows_test.go index cba8db8eecc..6da649e7f86 100644 --- a/auditbeat/module/file_integrity/security_windows_test.go +++ b/auditbeat/module/file_integrity/security_windows_test.go @@ -27,7 +27,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestGetSecurityInfo(t *testing.T) { +func TestGetNamedSecurityInfo(t *testing.T) { // Create a temp file that we will use in checking the owner. file, err := ioutil.TempFile("", "go") if err != nil { @@ -36,10 +36,15 @@ func TestGetSecurityInfo(t *testing.T) { defer os.Remove(file.Name()) defer file.Close() + name, err := syscall.UTF16PtrFromString(file.Name()) + if err != nil { + t.Fatal(err) + } + // Get the file owner. var securityID *syscall.SID var securityDescriptor *SecurityDescriptor - if err = GetSecurityInfo(syscall.Handle(file.Fd()), FileObject, + if err = GetNamedSecurityInfo(name, FileObject, OwnerSecurityInformation, &securityID, nil, nil, nil, &securityDescriptor); err != nil { t.Fatal(err) } diff --git a/auditbeat/module/file_integrity/zsecurity_windows.go b/auditbeat/module/file_integrity/zsecurity_windows.go index b9e4c8895b1..5f8bc47483b 100644 --- a/auditbeat/module/file_integrity/zsecurity_windows.go +++ b/auditbeat/module/file_integrity/zsecurity_windows.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT +// Code generated by 'go generate'; DO NOT EDIT. package file_integrity @@ -28,17 +28,42 @@ import ( var _ unsafe.Pointer +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + var ( modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") - procGetSecurityInfo = modadvapi32.NewProc("GetSecurityInfo") + procGetNamedSecurityInfoW = modadvapi32.NewProc("GetNamedSecurityInfoW") ) -func GetSecurityInfo(handle syscall.Handle, objectType ObjectType, securityInformation SecurityInformation, ppsidOwner **syscall.SID, ppsidGroup **syscall.SID, ppDacl **ACL, ppSacl **ACL, ppSecurityDescriptor **SecurityDescriptor) (err error) { - r1, _, e1 := syscall.Syscall9(procGetSecurityInfo.Addr(), 8, uintptr(handle), uintptr(objectType), uintptr(securityInformation), uintptr(unsafe.Pointer(ppsidOwner)), uintptr(unsafe.Pointer(ppsidGroup)), uintptr(unsafe.Pointer(ppDacl)), uintptr(unsafe.Pointer(ppSacl)), uintptr(unsafe.Pointer(ppSecurityDescriptor)), 0) +func GetNamedSecurityInfo(name *uint16, objectType ObjectType, securityInformation SecurityInformation, ppsidOwner **syscall.SID, ppsidGroup **syscall.SID, ppDacl **ACL, ppSacl **ACL, ppSecurityDescriptor **SecurityDescriptor) (err error) { + r1, _, e1 := syscall.Syscall9(procGetNamedSecurityInfoW.Addr(), 8, uintptr(unsafe.Pointer(name)), uintptr(objectType), uintptr(securityInformation), uintptr(unsafe.Pointer(ppsidOwner)), uintptr(unsafe.Pointer(ppsidGroup)), uintptr(unsafe.Pointer(ppDacl)), uintptr(unsafe.Pointer(ppSacl)), uintptr(unsafe.Pointer(ppSecurityDescriptor)), 0) if r1 != 0 { if e1 != 0 { - err = error(e1) + err = errnoErr(e1) } else { err = syscall.EINVAL } diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 7c1801f5fe3..53a6573bfd5 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -108,6 +108,18 @@ shared: source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' mode: 0644 skip_on_missing: true + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + skip_on_missing: true + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + skip_on_missing: true + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc: + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' + mode: 0644 + skip_on_missing: true @@ -201,6 +213,18 @@ shared: source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' mode: 0644 skip_on_missing: true + /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + skip_on_missing: true + /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + skip_on_missing: true + /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc: + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' + mode: 0644 + skip_on_missing: true - &agent_binary_files '{{.BeatName}}{{.BinaryExt}}': @@ -283,6 +307,18 @@ shared: source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' mode: 0644 skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz': + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512': + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc': + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' + mode: 0644 + skip_on_missing: true # Binary package spec (zip for windows) for community beats. @@ -334,6 +370,18 @@ shared: source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc' mode: 0644 skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip': + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512': + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc': + source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc' + mode: 0644 + skip_on_missing: true - &agent_docker_spec <<: *agent_binary_spec diff --git a/filebeat/docs/inputs/input-common-tcp-options.asciidoc b/filebeat/docs/inputs/input-common-tcp-options.asciidoc index 25d736f5324..ad57d4893d5 100644 --- a/filebeat/docs/inputs/input-common-tcp-options.asciidoc +++ b/filebeat/docs/inputs/input-common-tcp-options.asciidoc @@ -16,6 +16,17 @@ The maximum size of the message received over TCP. The default is `20MiB`. The host and TCP port to listen on for event streams. +[float] +[id="{beatname_lc}-input-{type}-tcp-framing"] +==== `framing` + +Specify the framing used to split incoming events. Can be one of +`delimiter` or `rfc6587`. `delimiter` uses the characters specified +in `line_delimiter` to split the incoming events. `rfc6587` supports +octet counting and non-transparent framing as described in +https://tools.ietf.org/html/rfc6587[RFC6587]. `line_delimiter` is +used to split the events in non-transparent framing. The default is `delimiter`. + [float] [id="{beatname_lc}-input-{type}-tcp-line-delimiter"] ==== `line_delimiter` diff --git a/filebeat/docs/inputs/input-common-unix-options.asciidoc b/filebeat/docs/inputs/input-common-unix-options.asciidoc index 9f97d84017e..2580b31b920 100644 --- a/filebeat/docs/inputs/input-common-unix-options.asciidoc +++ b/filebeat/docs/inputs/input-common-unix-options.asciidoc @@ -39,6 +39,17 @@ The file mode of the Unix socket that will be created by Filebeat. This is expected to be a file mode as an octal string. The default value is the system default (generally `0755`). +[float] +[id="{beatname_lc}-input-{type}-unix-framing"] +==== `framing` + +Specify the framing used to split incoming events. Can be one of +`delimiter` or `rfc6587`. `delimiter` uses the characters specified +in `line_delimiter` to split the incoming events. `rfc6587` supports +octet counting and non-transparent framing as described in +https://tools.ietf.org/html/rfc6587[RFC6587]. `line_delimiter` is +used to split the events in non-transparent framing. The default is `delimiter`. + [float] [id="{beatname_lc}-input-{type}-unix-line-delimiter"] ==== `line_delimiter` diff --git a/filebeat/input/syslog/config.go b/filebeat/input/syslog/config.go index ff97abd8e14..dd2803ecc09 100644 --- a/filebeat/input/syslog/config.go +++ b/filebeat/input/syslog/config.go @@ -47,7 +47,8 @@ var defaultConfig = config{ type syslogTCP struct { tcp.Config `config:",inline"` - LineDelimiter string `config:"line_delimiter" validate:"nonzero"` + LineDelimiter string `config:"line_delimiter" validate:"nonzero"` + Framing streaming.FramingType `config:"framing"` } var defaultTCP = syslogTCP{ @@ -90,9 +91,9 @@ func factory( return nil, err } - splitFunc := streaming.SplitFunc([]byte(config.LineDelimiter)) - if splitFunc == nil { - return nil, fmt.Errorf("error creating splitFunc from delimiter %s", config.LineDelimiter) + splitFunc, err := streaming.SplitFunc(config.Framing, []byte(config.LineDelimiter)) + if err != nil { + return nil, err } logger := logp.NewLogger("input.syslog.tcp").With("address", config.Config.Host) diff --git a/filebeat/input/tcp/config.go b/filebeat/input/tcp/config.go index 3bc928ac8c0..36e72aa8e44 100644 --- a/filebeat/input/tcp/config.go +++ b/filebeat/input/tcp/config.go @@ -23,6 +23,7 @@ import ( "github.com/dustin/go-humanize" "github.com/elastic/beats/v7/filebeat/harvester" + "github.com/elastic/beats/v7/filebeat/inputsource/common/streaming" "github.com/elastic/beats/v7/filebeat/inputsource/tcp" ) @@ -30,7 +31,8 @@ type config struct { tcp.Config `config:",inline"` harvester.ForwarderConfig `config:",inline"` - LineDelimiter string `config:"line_delimiter" validate:"nonzero"` + LineDelimiter string `config:"line_delimiter" validate:"nonzero"` + Framing streaming.FramingType `config:"framing"` } var defaultConfig = config{ diff --git a/filebeat/input/tcp/input.go b/filebeat/input/tcp/input.go index 598650d2a9c..865eff3d0f8 100644 --- a/filebeat/input/tcp/input.go +++ b/filebeat/input/tcp/input.go @@ -18,7 +18,6 @@ package tcp import ( - "fmt" "sync" "time" @@ -75,9 +74,9 @@ func NewInput( forwarder.Send(event) } - splitFunc := streaming.SplitFunc([]byte(config.LineDelimiter)) - if splitFunc == nil { - return nil, fmt.Errorf("unable to create splitFunc for delimiter %s", config.LineDelimiter) + splitFunc, err := streaming.SplitFunc(config.Framing, []byte(config.LineDelimiter)) + if err != nil { + return nil, err } logger := logp.NewLogger("input.tcp").With("address", config.Config.Host) diff --git a/filebeat/inputsource/common/streaming/listener.go b/filebeat/inputsource/common/streaming/listener.go index cff66b28b6c..16e708f554b 100644 --- a/filebeat/inputsource/common/streaming/listener.go +++ b/filebeat/inputsource/common/streaming/listener.go @@ -21,6 +21,7 @@ import ( "bufio" "bytes" "context" + "fmt" "net" "sync" @@ -47,6 +48,21 @@ type Listener struct { listenerFactory ListenerFactory } +// FramingType are supported framing options for the SplitFunc +type FramingType int + +const ( + FramingDelimiter = iota + FramingRFC6587 +) + +var ( + framingTypes = map[string]FramingType{ + "delimiter": FramingDelimiter, + "rfc6587": FramingRFC6587, + } +) + // NewListener creates a new Listener func NewListener(family inputsource.Family, location string, handlerFactory HandlerFactory, listenerFactory ListenerFactory, config *ListenerConfig) *Listener { return &Listener{ @@ -168,18 +184,43 @@ func (l *Listener) unregisterHandler() { l.clientsCount.Dec() } -// SplitFunc allows to create a `bufio.SplitFunc` based on a delimiter provided. -func SplitFunc(lineDelimiter []byte) bufio.SplitFunc { +// SplitFunc allows to create a `bufio.SplitFunc` based on a framing & +// delimiter provided. +func SplitFunc(framing FramingType, lineDelimiter []byte) (bufio.SplitFunc, error) { if len(lineDelimiter) == 0 { - return nil + return nil, fmt.Errorf("line delimiter required") } + switch framing { + case FramingDelimiter: + // This will work for most usecases and will also + // strip \r if present. CustomDelimiter, need to + // match completely and the delimiter will be + // completely removed from the returned byte slice + if bytes.Equal(lineDelimiter, []byte("\n")) { + return bufio.ScanLines, nil + } + return FactoryDelimiter(lineDelimiter), nil + case FramingRFC6587: + return FactoryRFC6587Framing(lineDelimiter), nil + default: + return nil, fmt.Errorf("unknown SplitFunc for framing %d and line delimiter %s", framing, string(lineDelimiter)) + } + +} + +// Unpack for config +func (f *FramingType) Unpack(value string) error { + ft, ok := framingTypes[value] + if !ok { + availableTypes := make([]string, len(framingTypes)) + i := 0 + for t := range framingTypes { + availableTypes[i] = t + i++ + } + return fmt.Errorf("invalid framing type '%s', supported types: %v", value, availableTypes) - ld := []byte(lineDelimiter) - if bytes.Equal(ld, []byte("\n")) { - // This will work for most usecases and will also strip \r if present. - // CustomDelimiter, need to match completely and the delimiter will be completely removed from - // the returned byte slice - return bufio.ScanLines } - return FactoryDelimiter(ld) + *f = ft + return nil } diff --git a/filebeat/inputsource/common/streaming/scan.go b/filebeat/inputsource/common/streaming/scan.go index de2b342049a..5b826e73592 100644 --- a/filebeat/inputsource/common/streaming/scan.go +++ b/filebeat/inputsource/common/streaming/scan.go @@ -20,6 +20,7 @@ package streaming import ( "bufio" "bytes" + "strconv" ) // FactoryDelimiter return a function to split line using a custom delimiter supporting multibytes @@ -46,3 +47,43 @@ func dropDelimiter(data []byte, delimiter []byte) []byte { } return data } + +// FactoryRFC6587Framing returns a function that splits based on octet +// counting or non-transparent framing as defined in RFC6587. Allows +// for custom delimter for non-transparent framing. +func FactoryRFC6587Framing(delimiter []byte) bufio.SplitFunc { + return func(data []byte, eof bool) (int, []byte, error) { + if eof && len(data) == 0 { + return 0, nil, nil + } + // need at least one character to see if octet or + // non transparent framing + if len(data) <= 1 { + return 0, nil, nil + } + // It can be assumed that octet-counting framing is + // used if a syslog frame starts with a digit RFC6587 + if bytes.ContainsAny(data[0:1], "0123456789") { + if i := bytes.IndexByte(data, ' '); i > 0 { + length, err := strconv.Atoi(string(data[0:i])) + if err != nil { + return 0, nil, err + } + end := length + i + 1 + if len(data) >= end { + return end, data[i+1 : end], nil + } + } + //request more data + return 0, nil, nil + } + if i := bytes.Index(data, delimiter); i >= 0 { + return i + len(delimiter), dropDelimiter(data[0:i], delimiter), nil + } + if eof { + return len(data), dropDelimiter(data, delimiter), nil + } + // request more data + return 0, nil, nil + } +} diff --git a/filebeat/inputsource/common/streaming/scan_test.go b/filebeat/inputsource/common/streaming/scan_test.go index eccd2b66300..c58e038a73c 100644 --- a/filebeat/inputsource/common/streaming/scan_test.go +++ b/filebeat/inputsource/common/streaming/scan_test.go @@ -106,3 +106,75 @@ func TestCustomDelimiter(t *testing.T) { }) } } + +func TestOctetCounting(t *testing.T) { + tests := []struct { + name string + input string + expected []string + delimiter []byte + }{ + { + name: "non-transparent", + input: "<9> message 0\n<6> msg 1\n<3> message 2", + expected: []string{ + "<9> message 0", + "<6> msg 1", + "<3> message 2", + }, + delimiter: []byte("\n"), + }, + { + name: "octet counting", + input: "13 <9> message 09 <6> msg 113 <3> message 2", + expected: []string{ + "<9> message 0", + "<6> msg 1", + "<3> message 2", + }, + delimiter: []byte("\n"), + }, + { + name: "octet counting, embedded newline", + input: "14 <9> message \n010 <6> msg \n114 <3> message \n2", + expected: []string{ + "<9> message \n0", + "<6> msg \n1", + "<3> message \n2", + }, + delimiter: []byte("\n"), + }, + { + name: "octet, non-transparent, octet", + input: "14 <9> message \n0<6> msg 1\n14 <3> message \n2", + expected: []string{ + "<9> message \n0", + "<6> msg 1", + "<3> message \n2", + }, + delimiter: []byte("\n"), + }, + { + name: "non-transparent, octet, non-transparent", + input: "<9> message 0\n10 <6> msg \n1<3> message 2", + expected: []string{ + "<9> message 0", + "<6> msg \n1", + "<3> message 2", + }, + delimiter: []byte("\n"), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + buf := strings.NewReader(test.input) + scanner := bufio.NewScanner(buf) + scanner.Split(FactoryRFC6587Framing(test.delimiter)) + var elements []string + for scanner.Scan() { + elements = append(elements, scanner.Text()) + } + assert.EqualValues(t, test.expected, elements) + }) + } +} diff --git a/filebeat/inputsource/tcp/server_test.go b/filebeat/inputsource/tcp/server_test.go index 2d05ed77d57..cb63eb88451 100644 --- a/filebeat/inputsource/tcp/server_test.go +++ b/filebeat/inputsource/tcp/server_test.go @@ -62,6 +62,8 @@ func TestReceiveEventsAndMetadata(t *testing.T) { tests := []struct { name string cfg map[string]interface{} + framing streaming.FramingType + delimiter []byte splitFunc bufio.SplitFunc expectedMessages []string messageSent string @@ -69,76 +71,87 @@ func TestReceiveEventsAndMetadata(t *testing.T) { { name: "NewLine", cfg: map[string]interface{}{}, - splitFunc: streaming.SplitFunc([]byte("\n")), + framing: streaming.FramingDelimiter, + delimiter: []byte("\n"), expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, "\n"), }, { name: "NewLineWithCR", cfg: map[string]interface{}{}, - splitFunc: streaming.SplitFunc([]byte("\r\n")), + framing: streaming.FramingDelimiter, + delimiter: []byte("\r\n"), expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, "\r\n"), }, { name: "CustomDelimiter", cfg: map[string]interface{}{}, - splitFunc: streaming.SplitFunc([]byte(";")), + framing: streaming.FramingDelimiter, + delimiter: []byte(";"), expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, ";"), }, { name: "MultipleCharsCustomDelimiter", cfg: map[string]interface{}{}, - splitFunc: streaming.SplitFunc([]byte("")), + framing: streaming.FramingDelimiter, + delimiter: []byte(""), expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, ""), }, { name: "SingleCharCustomDelimiterMessageWithoutBoundaries", cfg: map[string]interface{}{}, - splitFunc: streaming.SplitFunc([]byte(";")), + framing: streaming.FramingDelimiter, + delimiter: []byte(";"), expectedMessages: []string{"hello"}, messageSent: "hello", }, { name: "MultipleCharCustomDelimiterMessageWithoutBoundaries", cfg: map[string]interface{}{}, - splitFunc: streaming.SplitFunc([]byte("")), + framing: streaming.FramingDelimiter, + delimiter: []byte(""), expectedMessages: []string{"hello"}, messageSent: "hello", }, { name: "NewLineMessageWithoutBoundaries", cfg: map[string]interface{}{}, - splitFunc: streaming.SplitFunc([]byte("\n")), + framing: streaming.FramingDelimiter, + delimiter: []byte("\n"), expectedMessages: []string{"hello"}, messageSent: "hello", }, { name: "NewLineLargeMessagePayload", cfg: map[string]interface{}{}, - splitFunc: streaming.SplitFunc([]byte("\n")), + framing: streaming.FramingDelimiter, + delimiter: []byte("\n"), expectedMessages: largeMessages, messageSent: strings.Join(largeMessages, "\n"), }, { name: "CustomLargeMessagePayload", cfg: map[string]interface{}{}, - splitFunc: streaming.SplitFunc([]byte(";")), + framing: streaming.FramingDelimiter, + delimiter: []byte(";"), expectedMessages: largeMessages, messageSent: strings.Join(largeMessages, ";"), }, { name: "ReadRandomLargePayload", cfg: map[string]interface{}{}, - splitFunc: streaming.SplitFunc([]byte("\n")), + framing: streaming.FramingDelimiter, + delimiter: []byte("\n"), expectedMessages: []string{randomGeneratedText}, messageSent: randomGeneratedText, }, { name: "MaxReadBufferReachedUserConfigured", - splitFunc: streaming.SplitFunc([]byte("\n")), + framing: streaming.FramingDelimiter, + delimiter: []byte("\n"), cfg: map[string]interface{}{ "max_message_size": 50000, }, @@ -147,13 +160,50 @@ func TestReceiveEventsAndMetadata(t *testing.T) { }, { name: "MaxBufferSizeSet", - splitFunc: streaming.SplitFunc([]byte("\n")), + framing: streaming.FramingDelimiter, + delimiter: []byte("\n"), cfg: map[string]interface{}{ "max_message_size": 66 * 1024, }, expectedMessages: extraLargeMessages, messageSent: strings.Join(extraLargeMessages, "\n"), }, + { + name: "rfc6587 framing non-transparent", + framing: streaming.FramingRFC6587, + delimiter: []byte("\n"), + cfg: map[string]interface{}{}, + expectedMessages: []string{ + "<9> message 0", + "<6> msg 1", + "<3> message 2", + }, + messageSent: "<9> message 0\n<6> msg 1\n<3> message 2", + }, + { + name: "rfc6587 framing octet", + cfg: map[string]interface{}{}, + framing: streaming.FramingRFC6587, + delimiter: []byte("\n"), + expectedMessages: []string{ + "<9> message 0", + "<6> msg 1", + "<3> message 2", + }, + messageSent: "13 <9> message 09 <6> msg 113 <3> message 2", + }, + { + name: "rfc6587 framing octet embedded newline", + cfg: map[string]interface{}{}, + framing: streaming.FramingRFC6587, + delimiter: []byte("\n"), + expectedMessages: []string{ + "<9> message \n0", + "<6> msg \n1", + "<3> message \n2", + }, + messageSent: "14 <9> message \n010 <6> msg \n114 <3> message \n2", + }, } for _, test := range tests { @@ -171,7 +221,12 @@ func TestReceiveEventsAndMetadata(t *testing.T) { return } - factory := streaming.SplitHandlerFactory(inputsource.FamilyTCP, logp.NewLogger("test"), MetadataCallback, to, test.splitFunc) + splitFunc, err := streaming.SplitFunc(test.framing, test.delimiter) + if !assert.NoError(t, err) { + return + } + + factory := streaming.SplitHandlerFactory(inputsource.FamilyTCP, logp.NewLogger("test"), MetadataCallback, to, splitFunc) server, err := New(&config, factory) if !assert.NoError(t, err) { return diff --git a/filebeat/inputsource/unix/config.go b/filebeat/inputsource/unix/config.go index f6a2e4fa83a..7b4dfa74612 100644 --- a/filebeat/inputsource/unix/config.go +++ b/filebeat/inputsource/unix/config.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + "github.com/elastic/beats/v7/filebeat/inputsource/common/streaming" "github.com/elastic/beats/v7/libbeat/common/cfgtype" ) @@ -45,14 +46,15 @@ var socketTypes = map[string]SocketType{ // Config exposes the unix configuration. type Config struct { - Path string `config:"path"` - Group *string `config:"group"` - Mode *string `config:"mode"` - Timeout time.Duration `config:"timeout" validate:"nonzero,positive"` - MaxMessageSize cfgtype.ByteSize `config:"max_message_size" validate:"nonzero,positive"` - MaxConnections int `config:"max_connections"` - LineDelimiter string `config:"line_delimiter"` - SocketType SocketType `config:"socket_type"` + Path string `config:"path"` + Group *string `config:"group"` + Mode *string `config:"mode"` + Timeout time.Duration `config:"timeout" validate:"nonzero,positive"` + MaxMessageSize cfgtype.ByteSize `config:"max_message_size" validate:"nonzero,positive"` + MaxConnections int `config:"max_connections"` + LineDelimiter string `config:"line_delimiter"` + Framing streaming.FramingType `config:"framing"` + SocketType SocketType `config:"socket_type"` } // Validate validates the Config option for the unix input. diff --git a/filebeat/inputsource/unix/server.go b/filebeat/inputsource/unix/server.go index e85ced4cf49..5fd5486e331 100644 --- a/filebeat/inputsource/unix/server.go +++ b/filebeat/inputsource/unix/server.go @@ -52,9 +52,9 @@ type datagramServer struct { func New(log *logp.Logger, config *Config, nf inputsource.NetworkFunc) (Server, error) { switch config.SocketType { case StreamSocket: - splitFunc := streaming.SplitFunc([]byte(config.LineDelimiter)) - if splitFunc == nil { - return nil, fmt.Errorf("unable to create splitFunc for delimiter %s", config.LineDelimiter) + splitFunc, err := streaming.SplitFunc(config.Framing, []byte(config.LineDelimiter)) + if err != nil { + return nil, err } factory := streaming.SplitHandlerFactory(inputsource.FamilyUnix, log, MetadataCallback, nf, splitFunc) server := &streamServer{config: config} diff --git a/filebeat/tests/system/test_tcp.py b/filebeat/tests/system/test_tcp.py index fbfaa01e6d5..dea22e7388a 100644 --- a/filebeat/tests/system/test_tcp.py +++ b/filebeat/tests/system/test_tcp.py @@ -25,6 +25,18 @@ def test_tcp_with_custom_word_delimiter(self): """ self.send_events_with_delimiter("") + def test_tcp_with_rfc6587_non_transparent(self): + """ + Test TCP input with rfc6587 non_transparent framing + """ + self.send_events_with_rfc6587_framing("non-transparent") + + def test_tcp_with_rfc6587_octet(self): + """ + Test TCP input with rfc6587 octet counting framing + """ + self.send_events_with_rfc6587_framing("octet") + def send_events_with_delimiter(self, delimiter): host = "127.0.0.1" port = 8080 @@ -65,3 +77,47 @@ def send_events_with_delimiter(self, delimiter): assert output[0]["input.type"] == "tcp" sock.close() + + def send_events_with_rfc6587_framing(self, framing): + host = "127.0.0.1" + port = 8080 + delimiter = "\n" + input_raw = """ +- type: tcp + host: "{}:{}" + enabled: true + framing: rfc6587 +""" + + input_raw += "\n line_delimiter: {}".format(delimiter) + + input_raw = input_raw.format(host, port) + + self.render_config_template( + input_raw=input_raw, + inputs=False, + ) + + filebeat = self.start_beat() + + self.wait_until(lambda: self.log_contains("Started listening for TCP connection")) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP + sock.connect((host, port)) + + for n in range(0, 2): + if framing == "non-transparent": + sock.send(bytes("Hello World: " + str(n) + "\n", "utf-8")) + if framing == "octet": + sock.send(bytes("14 Hello World: " + str(n), "utf-8")) + + self.wait_until(lambda: self.output_count(lambda x: x >= 2)) + + filebeat.check_kill_and_wait() + + output = self.read_output() + + assert len(output) == 2 + assert output[0]["input.type"] == "tcp" + + sock.close() diff --git a/filebeat/tests/system/test_tcp_tls.py b/filebeat/tests/system/test_tcp_tls.py index 845ade6b288..b4edd0144b5 100644 --- a/filebeat/tests/system/test_tcp_tls.py +++ b/filebeat/tests/system/test_tcp_tls.py @@ -284,3 +284,62 @@ def test_tcp_tls_with_a_plain_text_socket(self): def assert_output(self, output): assert len(output) == 2 assert output[0]["input.type"] == "tcp" + + def test_tcp_over_tls_mutual_auth_rfc6587_framing(self): + """ + Test filebeat TCP with TLS when enforcing client auth with good client certificates and rfc6587 framing. + """ + input_raw = """ +- type: tcp + host: "{host}:{port}" + enabled: true + framing: rfc6587 + ssl.certificate_authorities: {cacert} + ssl.certificate: {certificate} + ssl.key: {key} + ssl.client_authentication: required +""" + config = { + "host": "127.0.0.1", + "port": 8080, + "cacert": CACERT, + "certificate": CLIENT1, + "key": CLIENTKEY1, + } + + input_raw = input_raw.format(**config) + + self.render_config_template( + input_raw=input_raw, + inputs=False, + ) + + filebeat = self.start_beat() + + self.wait_until(lambda: self.log_contains( + "Started listening for TCP connection")) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CACERT) + context.load_cert_chain(certfile=CLIENT2, keyfile=CLIENTKEY2) + + tls = context.wrap_socket(sock, server_side=False) + + tls.connect((config.get('host'), config.get('port'))) + + for n in range(0, NUMBER_OF_EVENTS): + tls.send(bytes("14 Hello World: " + str(n), "utf-8")) + + self.wait_until(lambda: self.output_count( + lambda x: x >= NUMBER_OF_EVENTS)) + + filebeat.check_kill_and_wait() + + output = self.read_output() + + self.assert_output(output) + + sock.close() diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 569341de1b4..f1f8c39d7fd 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -64,3 +64,4 @@ - Log level reloadable from fleet {pull}22690[22690] - Push log level downstream {pull}22815[22815] - Add metrics collection for Agent {pull}22793[22793] +- Add support for Fleet Server {pull}23736[23736] diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_decorator.go b/x-pack/elastic-agent/pkg/agent/application/fleet_decorator.go index 63e99d87a37..e2c4a2941bf 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_decorator.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_decorator.go @@ -54,7 +54,13 @@ func injectFleet(cfg *config.Config, hostInfo types.HostInfo, agentInfo *info.Ag transpiler.NewKey("id", transpiler.NewStrVal(hostInfo.UniqueID)), })) - fleet := transpiler.NewDict([]transpiler.Node{agent, token, kbn, host}) + nodes := []transpiler.Node{agent, token, kbn, host} + server, ok := transpiler.Lookup(ast, "fleet.server") + if ok { + nodes = append(nodes, server) + } + fleet := transpiler.NewDict(nodes) + err = transpiler.Insert(rootAst, fleet, "fleet") if err != nil { return err diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go index 262c3e5cc8c..19eb5499197 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/program_test.go @@ -384,8 +384,8 @@ func TestConfiguration(t *testing.T) { err bool }{ "single_config": { - programs: []string{"filebeat", "heartbeat", "metricbeat", "endpoint", "packetbeat"}, - expected: 5, + programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"}, + expected: 6, }, // "audit_config": { // programs: []string{"auditbeat"}, diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index b49de1bce6f..3ad9fbb8463 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -20,10 +20,11 @@ func init() { // spec/apm-server.yml // spec/endpoint.yml // spec/filebeat.yml + // spec/fleet-server.yml // spec/heartbeat.yml // spec/metricbeat.yml // spec/packetbeat.yml - unpacked := packer.MustUnpack("eJzMWVt3qziWfp+fUa89Fy5xupm1+sGQQoAdckxOJKE3JNmALTAd4wvMmv8+S4Ax4OTkVNdUzTw5xkLakvb+Ljv/9cuhWLP/iIrs3w7r99P6/d+rTPzyn7/QzC7J9328gqa/hL5gOREsLrYUrR5dYJ/pq1oT7GkEu4sQe0qESBLqH/6Ws3ofo8otg1f34FpeGaJZQjRYEjRTlhk8hsg7ELQyuOOppBnz2Vj1RMCbsbbUc4j89yUiB4Kg4qbn2E1Vu/nMRusfCbCVEBo1dzwRIrW+X8/jVu6pFMD6Jd7HrqXEJBMPWPcVlsGEft/Ha11ZuNa8dAEpqBMIJgwtRBeV4OdHK53HrjWPXccXFMAtB0b1kpoFzU2VO8+L5jdrHkcanL2kprLGpri9QxLqQMHqfT+uXcc+01Ufy4EDWLWxyd/NmqCLCPXgxPLB+tY8Xr7er9vOZ6jcMVU+L/QIwONLah4ImuUcxHvPKbt3fGPxOv+L+zSPQzTbuSBJmFKK9Wu8W2vd+45ycC0uKLBrDsSWaTBhmb/3qt3il39tE2md82Kf5uUkjQI02zFgFDRfxW8a3HLsFdzZLUJN3b2kpqBZcKaaOHJLrQnyVZYJZb0qEpYHBcnsLX/ax+Q2R0kA1Ky8Scsi1N4e3adQf3mKFxQYOdbNhIOkvU4QJCznBd3uYzc1niPkVSH2ZksFHkLsKxF6Pg1iOzE9SDh4O8l5lho8Esc8RTL1XvdH+cxt5rwUNIcPL+k8XWrGmVuGfT2SpTJ4R/eVEAdiqV1OpDIGe1T+sczkM3fhWqYeodmO6ryW863qgmHbrKjGqxApcZCJA8E+w79e423+7tfA9qXmwFYIvLBm7/blw3VCLRGhVm4iNJPjD/Rpv1i+mmIN4BZrpKDgrUsl8xziYC9jGZ43u91Z2o1LWMbra1ouX+cpz2AVITJzu2cciJIgQ5V391zPFwwYNbflfL4SosvhJd6XLoAPBPkbmebke1cGjry/+NG1vI/z4tVtxwG7InpfAqVref3c7iCu5auqMiDPMxCT5xXB/oljb0vwczqY55N1R+OP60ycP9qrv52frdysCDRUmomjzCkKzo9WqsQEJyJUjSxCF3E9OwZsJXrax242yB3si1CHVYSD/jw7KF3cyn2ecjS72/N9LD28NRC2tq9wqDy6TmlY3fOPoW0AN9t9HKHZmeOg7n57J3j36DrBjIG3xceQpgzrdtGcQQdpWJ+M/RGEglKsvzcYkITZRZB5t9fMPnAEB3loKiyHotnTdb4m54JTf94aPBDkK1T36pfUpETOh1fHEPlbgv0aa/Y5gobc28EFLcUss7IIM/sYQmWco/3v9jlaNTVVhng+qSUzowAK3sXMcngYUEfpOp6gyNBIu+YI0jvaSELNF0z3NyE2C6yVYr3q91sRpJ54BjfN2J4mbmcWaSKVNDq65+39mY/urD2T/vv43oeUMogDk4Jp4kTj/YJriZCYSwE8Ej3YL6zgr+2cwYRmJF5xJbIkzYgjc6DCdKVwnx7iZ8tMaLaKI2DXrxqcyTmoDhU5ZvN6jj2tx8SaILsKtThfrIot1WYyvxNZR1J6eE5ZcTRTIqSKZSbHJYZr/Wq4Fk9Ypmi+xXr62qRiTdfRHX1JeEKeCPHqSllNCYcZTPi8aKEwNWmvQnJfcAeel5k40NeZoJmdUgB335BMeV+MFMt1bB4Iis1DQxkDlUIy+8C0t3RpzdPlW/tJkX0MERcUwSO3ZiXVAvENxyUD9jaq1DbdrB8pqR+qrgPVeB6hWb7MLoJn8PANBSLMYe4KZaL25JkE9bKhEJgSZCtfwk3apNM/ZAliTRwJgA/XtOWOOMvzllTGzk05FTQrJPxsmB5UBNkl1iU9NuVw6tMYGEes+SeakUOEfKWFDykPgk2IiCKVaJvuDVQ9uuByIvpzA0cU2ecpFE9gruLoMoK0UDPOa2gkFFw2HBgbCkTNn4YqzlRovY+vMbPzsCzvYj1SzTgPy57gZEuwqTQ5lXdKFD83dx+hVfPZQ2Fzz96ZZUYDXxLW5D1NYlWoahwi7CtjiBAKae5lcKb58z+7j9uZZzCjutfBsKTTpo66uyIV1ZRHF3Tlfr7S1t9uz/R+z4uONhUmZaDd7gFrMm61/uzepvFGOBBSyU+fj9Y8fwrfYypyhm6ho5zMPjLtIqXjCL6vcTV5vRqenZowx7zBcP/8ciKdHGv+Hp53kxdESJnayaWmTobruZYp6/XILaPmIGhkL9ODXYQeJutArcEBPdgyGR/wz5/MoxJn/ug6cMfm41haiRycQq2U+4gJMLaRBqvJPAeqsRPL4C7C/oZplxOXcljmVPPs+X7/lVGvsS/fe3QdfybfuZ7Dz9Adx77A2gf09MV7Y6d4qx+WwZLqRDS0+31U4638AdIy2D0+LbNZQhGsJRaT30DTk/Vb54p9KRtkXkq+UQj2NlPJc5Mz7l1N3btOc+pcpdvUWaVuKZB3zRMO/P3kt/r5dv7JOocVeVXb/ACJGt7qoJ0DkBOXki/fDTHoyIFd0KzPj9IFXe7c3s9DfV4yB6ZMhzc5DhKFO+ZmKCN7WeIECgNFTbWH2zPNziLt1/77oEZKF6gX7tzepxlUSHY58dv+Ts91qIZY5uZqmAODXFUmNSW/i3q0jhNs2S33ZA31Y6VkX99+k/ZP5vqfK42BOEZZI42lJjg2Ejkjp5fU/OumlbFbqpt9npLcO0n8nPBiww1koI2Ge/hQbt9iGein29lzHJz5UHZqcMak9cjePpGU/drHa2yb1138LZ2fXWAfiWXuQ+wvCd7tPafs5g+MpTXPCbokTA+KUPdFiL1tZLGDa/GKoKBgFTvIPXpamZCsTLxKyk2pGaSlH3c8knX0Xn6gGV8BTFgetJoo80si+Wn0rLWvrn3odR3ToMLx/BihS/mVBryO5QCWDDTcc+xz4EnNQnSpf383TU1oZucEqZJ3hvM3uTQeK7mLFzRjR9rwy9kgAKYcsRRPWjnNvTvPp9F5/NNdOH/DNaFEtlERxMXamd84+MoJI/41K3k/OPdnNA/2BMsceD4t00OP7Z9j6g9q9AssvvFFf6/9e9Na7bFCavM/pBM4XuerOvu/qq1sXb6n7IPi+o6gwjKx7QxY15ZWBXe8ItQ6o3bfeq4JDlRmzQoKlK+K5TpWCs0zBbZCvjJyk2KhyNiR7+rDEkv/fig7T/0jI3ebHwcVRxPTB4ycSMNUzQ6NEHxSdwR5Kqk8LsGEA5GFrRBvCopVRklwUEXI7wrMPDE9GPUAO1JshMmoBzfqS6kn4jQ9iyOxGnEmTcNxjdS+5yTJQJ43watHCTRUC5piXmarkyRHCVLLXJTUmkkheDUui1sv5OOCHxrACM12BMdXcd6ImpfUvO6xnhDbteW/YY53CjVYM83oi4dqs40UJSS7FK2RFUemwaohxDzoRUlvSLt860xCJXOHor5nmrHMKO+NQnC6PfP7f0G0caoJe5r0Oz8wP58YjoacsWYfqP2JsWvXvq05AIf7vc9OVJ8Pha1YA18wZ9WQU2+aqqYuis5w9rnaNjFGBjLFq0msenDC2qVg+mrc37oas8EdjUzmb9pHf4cpQaQBsz/ZPN6JdKzzgoNkwzKYE5z0jYYPhHlLSunD+1LrcEx/3v2wB/t7+7ZTIvqgd9sR0P+CMYafk/SPjLLjyRpfL56M1be2efOXZXoo7s+oI1K5xtM+9oY96VbAH0OkirHx6hoRo7E3YSvxm6OLGJiHJNLgJsReFU77tF2O9DihQWUUV5Mr15h9cSPmnzGYg/d+i6GdGII/1wQ332syNlF/ipGeNABGHNJwbd5wSSbvWHKMxIimSfZTDaqxxpDvTnPrapboUKNIzXHPDT1P/pbe/GjenzRPVs73BD08jvvto70cr/n+e/rvyyaem9BzLf4eIvIevjZ/H6jGpfaoI4sVVvz3v1+FXhGx3fojF/UG7G2kQWUk9BwpqErBwVTosTJoF/pC6Mkxd2N/KPQalVqpdqtWf0ro5VIJL9/ems8vhN547KdCj38m9BoHR/CnTur3OZYJQH7mVlh3V1a37q3LL89q1nb7s79d/+n4R7ia/xfupUns//6X/wkAAP//iTjyjQ==") + unpacked := packer.MustUnpack("eJzMWVt3ozqafZ+fcV57LlzidDNr9YMhjQA7pEwqSOgNSTZgC+yO8QVmzX+fJYExYKdSVedMnX5yAkLX79vf3lv/89t+t6T/Fe/y/9gv34/L9/+scv7bf/9GcrvEX7fJIjT9eehzWmBOk92awMWjC+wTeVVrjDwNI3cWIU+JIU4j/e67gtbbBFZuGby6e9fyyghOUqyFJYYTZZ6Hhwh6ewwXBnM8Fcs2H7VVjxi8GUtLPUXQf59DvMcwVNzslLiZasvffDD+AQNbiUKjZo7HI6jWt+N5zCo8lYCwfkm2iWspCc75A9J9heZhSr5uk6WuzFxrWroA74gTcMoNLYJnFaPnRyubJq41TVzH5wSEawaM6iUzd6QwVeY8z+Q7a5rEWjh5yUxliUx+/QanxAk5rbddu2Yc+0QW3Vz2DIRVMzfx3qwxPPNID4606I1vTZP56+24TX+GyhxTZdOdHoPw8JKZewwnBQPJ1nPK9hvfmL1O/+I+TZMITjYuSFOqlHz5mmyWWvu9o+xdi3EC7JoBvqZamNLc33rVZvbbvzeBtCzYbpsV5SiMAjjZUGDsSLFI3rRwzZC3Y85mFmnq5iUzOcmDE9H4gVlqjaGv0pwry8UupUWww7m9Zk/bBF/7KDEINauQYbmLtLdH9ynSX56SGQFGgXQzZSBtjhMEKS3Yjqy3iZsZzzH0qgh5k7kS7iPkKzF8PvbmdqR6kDLwdhT9zLXwgB3zGIvQe90exDNX9nnekSJ8eMmm2VwzTswy7MuWzJXeN7qvRCjgc+18xJXRW6Pyz3kunrkz1zL1GE42RGe16G9R7yiyzYporIqgkgQ532PkU/SPy3zl390YyD7XDNgKDs9Urt0+3x0n0lIeaeUqhhPRfk+etrP5q8mXIFwjDe8IeGtDyTxFKNiKufT3m17PLGvbpTRn9SUs56/TjOVhFUM8cdtnDPASQ0MVZ/dcT2cUGDWzRX++EsHz/iXZli4IHzD0VyLM8dc2DRxxfsmja3n34+LVbdoBu8J6lwKla3ld325vXvNXVaVA7GfAR88rjPwjQ94ao+es188H4w7aH5Y5P91bq7+enqzCrHBoqCTnBxFTBJwerUxJMEp5pBp5DM/8sncU2Er8tE3cvBc7yOeRHlYxCrr9bKF0dk33acbg5GbNt3Pp4E1C2NK+wKHy6DqlYbXP70NbD27W2ySGkxNDQd2+e8do8+g6wYSCt9l9SFP6eTuTe9BCGtJHbb8FoaDky68SA9IoP3M8bdea23sGw14cmgotQi7XdOlPxlxw7PZbC/cY+grRvfolMwkW/aHFIYL+GiO/Rpp9ikNDrG3vgqbEzPNyF+X2IQqVYYx27+1TvJA5VUZoOsolMycg5KydMy3Cfa90lK7jcQINDTdjDiC9LRtppPmc6v4qQuYOaSVfLrr1VhiqR5aHK9m2KxPXPYs1noky+gPjdeW7f1539++674M13pYiJSF6qFAnVPrzYDmvY2gcrORuaWr6dJSd+/SQPFtmSvJFQkB4wHqwnVnBXy/xuHrdJF+y6ckF9gFb5jZC/hyjjejjyFAg2hhza1pgeE6pHuwi3ecR8taxRXdW7h9FDtHcFnEicHiz1D2VFMGOwLeDbOcoiftVSTzNrsjXSPFO279fSt4q40uyjG9KnoA06PEILS5lTqZ9lIcpm+4a+MxM0jGXwufMCU/znO/J64ST3M4ICDdfoEgTnw9YzqVtEXCCzL0sMz1mg3N7T7W3bG5Ns/lb80ugfYgg4wSGB2ZNSqIF/AtKSgrsdVypzVZb32Jf32Rqe6KxIoaTYp6fOcvD/RcY8KgIC5crI4Yo9iSo57LshBmGtvIpRGWKgJ5/irBDGj9gED5cQo85/CT2W5Q/epIpuCP5TkDWiupBhaFdIl2UVJlCAsZaSDEOSPOPJMf7GPpKAzmCUgSrCGJFhH+T6hLeHl1wPmL9WaYEgfZpDN8jaKwYPA9gMNKM0zI0UgLOKwaMFQG8Zk995mcqpN4mlznTUz+Vb+Z6IJpx6qcuRukaI1ORMVW07BU9y7OP4UL+dvApz9k70dyQkCdSWZzTaK4KUY19jHxlCCtcwfJcentaPP/sOq57noc50b0WukUJlnnUnhWuiKY8uoAfBIRc+mTgb9dnerfmWVtqRUoL+JdrQJqYt1p/dG7j+cYo4IL9j58Pxjx9CPlD+HP6CqMtU7l9oNpZ0M0BBF/mJeN60d87NaWOybv47Z6fj7ilcPLv/n7LuMBcUNuWYsk86Y/nWqbI1wOzjJqBQFJlqgebGD6Mxgk1iQN6sKZifsA/fdCPip3po+uEGzodzqWh1cEx0kqxjgQDYx1rYTXqZ080eqR5uImRv6La+cgEhRYxJZ89366/Muol8sV3j67jT8Q3l334nhLJkM/RReVMv/+7obq85g/Nw5LomMtS/XWQ4w1lAkJm2B0+zfNJSmBYCyzGP1DaR+M3ahf5ogyLuBT1RsHIW41p0rWEuzc5datUzbHaFQpVp5W6JkCcNUsZ8Lejd/Xzdf/TZRFW+FVt4gOkanTNg6YPgI9M0MRi08egAwP2juRdfJQuaGPn+n0R6dOSOmFG9fBK4UGqMMdc9annpd/YCRQKdjXRHq7PNDuPtX9cqco1R0oXqGfmXL8neajg/Hxk1/Udn+tIjZCIzUU/BnqxqoxySvzP68E4TrCm19gTOdS1FTR/eX0nJKOI9V9LpwE/xLmk04ITHCStzvHxJTP/umqo75roZhenuPCOAj9HdVHWBtzjRj9CMa/86br3Db3rna8WTqiQK/nbjGkpF3jWp4y9sQ9/AH3cuxarMAx2tKJ7sUZPK1Ocl6lXnRJP4JC0AYYuyYovl+V9wy1o1EjypoWVUNRW7pf4SWyZfC6UePu8Ub+uvRdKtjkSS5W07pZytSoFdUZen558pAoHBtRNeH0CI3fV4Z0Q61TieH1/1Pigo62fzkHS2kW7Jx+ppDZ1OjXTzvMyF0FPBUW4Zww2RmirlKxJZ1Je+prnnctTY2hXkZYkqKMAHbXoQl6MMaQft8r0+nxIQT4vVQNF2MmRkeH4cYkanIG9jrSwpqqRUsBl25/rp1Xhi59aQ4WReaRC1eV0aJB+QxH3z3wcm30Z8CH1u1X8P7ZmJ6xxaIxg7sfmfZ+m/K4+DhKvkH+KoM9/7izHFEj+X+OfXGOPpg/N+vU9vPA+wp+RcT7GlOnQyfgzXYvk750DkS7j9/KOBfEKwpQWQSOx2zoSD571akhrE1AtVBiaHmJ4Lj+zFC5tGQhLCqSUOXSU4knNI3iuf/+FjpqS3C4wVIWM6fcvqcmwrZBCTOT3gUi5cjIwCDMGaYZGtwmSRjjPx8F+/PRFkL9iGldi26gwZHzpTG/qxVDOmZU4H1T4E1IEW4zE2T4f59n+O2rbn1IT/x8uo4bjfEbb/iyqli/L94zeSa6vMFRoztetn9fejKqcOd4u0lrf7/b2s8YoUKk12RGgfJYsl7YKhuqJAFvBn/mCo2Qh0Njgr+rDHJlppO3LFuS+5Qte+0dBxeDIQwRGgTV+wNVkL32FJ3WDoafiymMCTBjgedT4OjKhaGWUGAVVDP02wcwj1YPBNVSrsWTBGFwDDa5G1CN2ZBE9YEtqfQVD5bCEanftIbSF2G+MFo8CaIgWyGSe54uj0FoCpOYFL4k12cTIv/hgs2txvp/wfT8xhpMNRsnF65Ea+SUzL2usRzrpcuu8oo53lARIM7rkIdpkJTQuzs+7hqTzgyQYQl8VQadxO3+zjbfWc6pE7BDYXdvlNDfKW98pOF6f+d0teDNPNaVPoyu3O17aPfJ40XpIs/fE/sAnbMa+jtkDh9u1T45En/Z9Er4EPqfOQhanrrhXMi92rX/ZxWrjiQ/8yAwtRnPVgyPSzjuqL4ZXHhefr3dGg4L/Q+vozjDDEEsw+8Ve5I3ng3S2YyBd0TwsMEo7wnrH52mKUvbwPtdaHNOfN9+8Bvy9V4fjQnTn+vAiyn6/zxp+XKS/5bs6nsjx5ezJWHxpxOxf5tl+d7tHbSEVYzxtE28gfKVIOURQ5UMfr/W1RyL54pMI/BZCvedFpbEWriLkVdH46q6NkQ4ntJ7462LlMmef96/mPvcre9/9iD868pd+rac6FhS/zpcd+cmDGiJrbSFrSS7OWNQYgRHyzuW77juGHEN8O46ti/dG+hxFcI7b2tDVyUEur289xH5ODvr9Ti/OKtgWw4dHF5w5yZkSW8lmqQ3WcrjEO9WH4ikGdv2qhRNBBC/ia/UqSNuNYVLM5XyuRM+12HsE8Xv0Kv/eE40J7lGPRdQuppvlPRX1Bux1rIXKgOg5glCVnIEx0aNl0Az0CdETbW7afpPoSZZaqXbDVr+L6BWCCc/f3uTvJ0Rv2PZDosc+InpSwWH0oZL6JS4ibc/q4tBcL43FXk2ay+P8b3cdzj9I1fxLqBcZ2P/7b/8XAAD//xrkHvo=") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-fleet-server.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-fleet-server.yml new file mode 100644 index 00000000000..cb8a3e10659 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-fleet-server.yml @@ -0,0 +1,13 @@ +fleet: + agent: + id: fleet-agent-id + +output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword + +inputs: + - id: fleet-server-id + type: fleet-server diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml index c7460776151..fb585dae996 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml @@ -8,6 +8,12 @@ fleet: protocol: https host: localhost:5601 timeout: 30s + server: + output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword outputs: default: @@ -25,6 +31,11 @@ outputs: ca_sha256: "7lHLiyp4J8m9kw38SJ7SURJP4bXRZv/BNxyyXkCcE/M=" inputs: +- id: fleet-server-id + type: fleet-server + use_output: default + data_stream: + type: default - type: docker/metrics use_output: default streams: diff --git a/x-pack/elastic-agent/pkg/core/app/descriptor.go b/x-pack/elastic-agent/pkg/core/app/descriptor.go index a5445e6178d..9c1ef10c3ad 100644 --- a/x-pack/elastic-agent/pkg/core/app/descriptor.go +++ b/x-pack/elastic-agent/pkg/core/app/descriptor.go @@ -5,7 +5,6 @@ package app import ( - "path" "path/filepath" "strings" @@ -29,7 +28,7 @@ func NewDescriptor(spec program.Spec, version string, config *artifact.Config, t spec: spec, directory: dir, executionCtx: NewExecutionContext(spec.ServicePort, spec.Cmd, version, tags), - process: specification(dir, spec.Cmd), + process: specification(dir, spec), } } @@ -78,23 +77,12 @@ func (p *Descriptor) Directory() string { return p.directory } -func defaultSpec(dir string, binaryName string) ProcessSpec { - if !isKnownBeat(binaryName) { - return ProcessSpec{ - BinaryPath: path.Join(dir, binaryName), - } - } - +func specification(dir string, spec program.Spec) ProcessSpec { return ProcessSpec{ - BinaryPath: path.Join(dir, binaryName), - Args: []string{}, + BinaryPath: filepath.Join(dir, spec.Cmd), + Args: spec.Args, + Configuration: nil, } - -} - -func specification(directory, binaryName string) ProcessSpec { - defaultSpec := defaultSpec(directory, binaryName) - return populateSpec(directory, binaryName, defaultSpec) } func directory(spec program.Spec, version string, config *artifact.Config) string { @@ -114,42 +102,3 @@ func directory(spec program.Spec, version string, config *artifact.Config) strin return strings.TrimSuffix(path, suffix) } - -func isKnownBeat(name string) bool { - switch name { - case "filebeat": - fallthrough - case "metricbeat": - return true - } - - return false -} - -func populateSpec(dir, binaryName string, spec ProcessSpec) ProcessSpec { - var programSpec program.Spec - var found bool - binaryName = strings.ToLower(binaryName) - for _, prog := range program.Supported { - if binaryName != strings.ToLower(prog.Name) { - continue - } - found = true - programSpec = prog - break - } - - if !found { - return spec - } - - if programSpec.Cmd != "" { - spec.BinaryPath = filepath.Join(dir, programSpec.Cmd) - } - - if len(programSpec.Args) > 0 { - spec.Args = programSpec.Args - } - - return spec -} diff --git a/x-pack/elastic-agent/pkg/core/authority/ca.go b/x-pack/elastic-agent/pkg/core/authority/ca.go index f558e8a9eb3..4d8394045e5 100644 --- a/x-pack/elastic-agent/pkg/core/authority/ca.go +++ b/x-pack/elastic-agent/pkg/core/authority/ca.go @@ -42,6 +42,7 @@ func NewCA() (*CertificateAuthority, error) { SerialNumber: big.NewInt(1653), Subject: pkix.Name{ Organization: []string{"elastic-fleet"}, + CommonName: "localhost", }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(10, 0, 0), @@ -108,6 +109,7 @@ func (c *CertificateAuthority) GeneratePairWithName(name string) (*Pair, error) DNSNames: []string{name}, Subject: pkix.Name{ Organization: []string{"elastic-fleet"}, + CommonName: name, }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(10, 0, 0), diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/start.go b/x-pack/elastic-agent/pkg/core/plugin/process/start.go index 915cbe155ff..e177a9882df 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/start.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/start.go @@ -140,6 +140,8 @@ func injectLogLevel(logLevel string, args []string) []string { level = "info" case "debug": level = "debug" + case "warning": + level = "warning" case "error": level = "error" } diff --git a/x-pack/elastic-agent/spec/endpoint.yml b/x-pack/elastic-agent/spec/endpoint.yml index a435c2f9ed4..7265ba2e693 100644 --- a/x-pack/elastic-agent/spec/endpoint.yml +++ b/x-pack/elastic-agent/spec/endpoint.yml @@ -53,6 +53,8 @@ rules: key: access_api_key - remove_key: key: kibana + - remove_key: + key: server - filter: selectors: diff --git a/x-pack/elastic-agent/spec/fleet-server.yml b/x-pack/elastic-agent/spec/fleet-server.yml new file mode 100644 index 00000000000..1c444f8990e --- /dev/null +++ b/x-pack/elastic-agent/spec/fleet-server.yml @@ -0,0 +1,61 @@ +name: Fleet Server +cmd: fleet-server +args: ["--agent-mode"] +artifact: fleet-server +rules: + - fix_stream: {} + + - filter_values: + selector: inputs + key: type + values: + - fleet-server + + - filter_values: + selector: inputs + key: enabled + values: + - true + + - remove_key: + key: output + + - select_into: + selectors: [ fleet.server.output.elasticsearch ] + path: output + + - map: + path: fleet + rules: + - remove_key: + key: enabled + - remove_key: + key: access_api_key + - remove_key: + key: kibana + - remove_key: + key: reporting + - remove_key: + key: server + + - map: + path: inputs + rules: + - remove_key: + key: use_output + - remove_key: + key: data_stream + - remove_key: + key: data_stream.namespace + - remove_key: + key: data_stream.dataset + - remove_key: + key: streams + + - filter: + selectors: + - fleet + - inputs + - output + +when: length(${fleet}) > 0 and length(${inputs}) > 0 and hasKey(${output}, 'elasticsearch') diff --git a/x-pack/filebeat/module/o365/audit/config/pipeline.js b/x-pack/filebeat/module/o365/audit/config/pipeline.js index b7be6418c18..5d314822af6 100644 --- a/x-pack/filebeat/module/o365/audit/config/pipeline.js +++ b/x-pack/filebeat/module/o365/audit/config/pipeline.js @@ -157,7 +157,7 @@ function makeDictFromModifiedPropertyArray(options) { if (src[i] == null || (name=src[i].Name) == null || (newValue=src[i].NewValue) == null - || (oldValue=src[i].OldValue)) continue; + || (oldValue=src[i].OldValue) == null) continue; name = validFieldName(name); if (name in dict) { if (dict[name].NewValue instanceof Array) { diff --git a/x-pack/filebeat/module/o365/audit/test/08-azuread.log-expected.json b/x-pack/filebeat/module/o365/audit/test/08-azuread.log-expected.json index 78cfca3dbfb..ee64d193719 100644 --- a/x-pack/filebeat/module/o365/audit/test/08-azuread.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/08-azuread.log-expected.json @@ -91,6 +91,8 @@ "o365.audit.Id": "8f6eb24b-6e61-4ee2-a376-31368c300613", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -235,6 +237,8 @@ "o365.audit.Id": "8f6eb24b-6e61-4ee2-a376-31368c300613", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -379,6 +383,8 @@ "o365.audit.Id": "8f6eb24b-6e61-4ee2-a376-31368c300613", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -2939,6 +2945,8 @@ "o365.audit.Id": "fe115c66-3e08-4ab4-8a00-84ae25a59078", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -3083,6 +3091,8 @@ "o365.audit.Id": "fe115c66-3e08-4ab4-8a00-84ae25a59078", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -3382,6 +3392,8 @@ "o365.audit.Id": "d6ad8dba-dd88-499e-a1e1-e649bf8eeb71", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -3526,6 +3538,8 @@ "o365.audit.Id": "d6ad8dba-dd88-499e-a1e1-e649bf8eeb71", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -3670,6 +3684,8 @@ "o365.audit.Id": "d6ad8dba-dd88-499e-a1e1-e649bf8eeb71", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -5582,6 +5598,8 @@ "o365.audit.Id": "83c924c1-f2e2-4b39-8eda-b80c3823a875", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "StrongAuthenticationPhoneAppDetail", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.StrongAuthenticationPhoneAppDetail.NewValue": "[\r\n {\r\n \"DeviceName\": \"NO_DEVICE\",\r\n \"DeviceToken\": \"NO_DEVICE_TOKEN\",\r\n \"DeviceTag\": \"SoftwareTokenActivated\",\r\n \"PhoneAppVersion\": \"NO_PHONE_APP_VERSION\",\r\n \"OathTokenTimeDrift\": -1,\r\n \"DeviceId\": null,\r\n \"Id\": \"3b539b10-3846-4f9b-877d-55b0b8e76147\",\r\n \"TimeInterval\": null,\r\n \"AuthenticationType\": 2,\r\n \"NotificationType\": 1,\r\n \"SecuredPartitionId\": 0,\r\n \"SecuredKeyId\": 0\r\n }\r\n]", + "o365.audit.ModifiedProperties.StrongAuthenticationPhoneAppDetail.OldValue": "[\r\n {\r\n \"DeviceName\": \"NO_DEVICE\",\r\n \"DeviceToken\": \"NO_DEVICE_TOKEN\",\r\n \"DeviceTag\": \"SoftwareTokenActivated\",\r\n \"PhoneAppVersion\": \"NO_PHONE_APP_VERSION\",\r\n \"OathTokenTimeDrift\": 0,\r\n \"DeviceId\": null,\r\n \"Id\": \"3b539b10-3846-4f9b-877d-55b0b8e76147\",\r\n \"TimeInterval\": null,\r\n \"AuthenticationType\": 2,\r\n \"NotificationType\": 1,\r\n \"SecuredPartitionId\": 0,\r\n \"SecuredKeyId\": 0\r\n }\r\n]", "o365.audit.ModifiedProperties.TargetId_UserType.NewValue": "Member", "o365.audit.ModifiedProperties.TargetId_UserType.OldValue": "", "o365.audit.ObjectId": "asr@testsiem.onmicrosoft.com", @@ -9932,8 +9950,16 @@ "o365.audit.ExtendedProperties.teamName": "MSODS.", "o365.audit.ExtendedProperties.version": "2", "o365.audit.Id": "689aaff0-b34f-4077-9244-0563b9f9c03b", + "o365.audit.ModifiedProperties.AppId.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.AppId.OldValue": "[]", + "o365.audit.ModifiedProperties.AvailableToOtherTenants.NewValue": "[\r\n false\r\n]", + "o365.audit.ModifiedProperties.AvailableToOtherTenants.OldValue": "[]", + "o365.audit.ModifiedProperties.DisplayName.NewValue": "[\r\n \"siem2\"\r\n]", + "o365.audit.ModifiedProperties.DisplayName.OldValue": "[]", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Add application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -10076,8 +10102,16 @@ "o365.audit.ExtendedProperties.teamName": "MSODS.", "o365.audit.ExtendedProperties.version": "2", "o365.audit.Id": "689aaff0-b34f-4077-9244-0563b9f9c03b", + "o365.audit.ModifiedProperties.AppId.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.AppId.OldValue": "[]", + "o365.audit.ModifiedProperties.AvailableToOtherTenants.NewValue": "[\r\n false\r\n]", + "o365.audit.ModifiedProperties.AvailableToOtherTenants.OldValue": "[]", + "o365.audit.ModifiedProperties.DisplayName.NewValue": "[\r\n \"siem2\"\r\n]", + "o365.audit.ModifiedProperties.DisplayName.OldValue": "[]", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Add application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -10220,8 +10254,16 @@ "o365.audit.ExtendedProperties.teamName": "MSODS.", "o365.audit.ExtendedProperties.version": "2", "o365.audit.Id": "689aaff0-b34f-4077-9244-0563b9f9c03b", + "o365.audit.ModifiedProperties.AppId.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.AppId.OldValue": "[]", + "o365.audit.ModifiedProperties.AvailableToOtherTenants.NewValue": "[\r\n false\r\n]", + "o365.audit.ModifiedProperties.AvailableToOtherTenants.OldValue": "[]", + "o365.audit.ModifiedProperties.DisplayName.NewValue": "[\r\n \"siem2\"\r\n]", + "o365.audit.ModifiedProperties.DisplayName.OldValue": "[]", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Add application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -10364,8 +10406,16 @@ "o365.audit.ExtendedProperties.teamName": "MSODS.", "o365.audit.ExtendedProperties.version": "2", "o365.audit.Id": "689aaff0-b34f-4077-9244-0563b9f9c03b", + "o365.audit.ModifiedProperties.AppId.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.AppId.OldValue": "[]", + "o365.audit.ModifiedProperties.AvailableToOtherTenants.NewValue": "[\r\n false\r\n]", + "o365.audit.ModifiedProperties.AvailableToOtherTenants.OldValue": "[]", + "o365.audit.ModifiedProperties.DisplayName.NewValue": "[\r\n \"siem2\"\r\n]", + "o365.audit.ModifiedProperties.DisplayName.OldValue": "[]", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Add application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -10663,8 +10713,18 @@ "o365.audit.ExtendedProperties.teamName": "MSODS.", "o365.audit.ExtendedProperties.version": "2", "o365.audit.Id": "48403af8-b712-4e63-a999-686b631240ac", + "o365.audit.ModifiedProperties.AccountEnabled.NewValue": "[\r\n true\r\n]", + "o365.audit.ModifiedProperties.AccountEnabled.OldValue": "[]", + "o365.audit.ModifiedProperties.AppPrincipalId.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.AppPrincipalId.OldValue": "[]", + "o365.audit.ModifiedProperties.Credential.NewValue": "[\r\n {\r\n \"CredentialType\": 2,\r\n \"KeyStoreId\": \"291154f0-a9f5-45bb-87be-9c8ee5b6d62c\",\r\n \"KeyGroupId\": \"291154f0-a9f5-45bb-87be-9c8ee5b6d62c\"\r\n }\r\n]", + "o365.audit.ModifiedProperties.Credential.OldValue": "[]", + "o365.audit.ModifiedProperties.DisplayName.NewValue": "[\r\n \"siem2\"\r\n]", + "o365.audit.ModifiedProperties.DisplayName.OldValue": "[]", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.ServicePrincipalName.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.ServicePrincipalName.OldValue": "[]", "o365.audit.ModifiedProperties.TargetId_ServicePrincipalNames.NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40", "o365.audit.ModifiedProperties.TargetId_ServicePrincipalNames.OldValue": "", "o365.audit.ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40", @@ -10818,8 +10878,18 @@ "o365.audit.ExtendedProperties.teamName": "MSODS.", "o365.audit.ExtendedProperties.version": "2", "o365.audit.Id": "48403af8-b712-4e63-a999-686b631240ac", + "o365.audit.ModifiedProperties.AccountEnabled.NewValue": "[\r\n true\r\n]", + "o365.audit.ModifiedProperties.AccountEnabled.OldValue": "[]", + "o365.audit.ModifiedProperties.AppPrincipalId.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.AppPrincipalId.OldValue": "[]", + "o365.audit.ModifiedProperties.Credential.NewValue": "[\r\n {\r\n \"CredentialType\": 2,\r\n \"KeyStoreId\": \"291154f0-a9f5-45bb-87be-9c8ee5b6d62c\",\r\n \"KeyGroupId\": \"291154f0-a9f5-45bb-87be-9c8ee5b6d62c\"\r\n }\r\n]", + "o365.audit.ModifiedProperties.Credential.OldValue": "[]", + "o365.audit.ModifiedProperties.DisplayName.NewValue": "[\r\n \"siem2\"\r\n]", + "o365.audit.ModifiedProperties.DisplayName.OldValue": "[]", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.ServicePrincipalName.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.ServicePrincipalName.OldValue": "[]", "o365.audit.ModifiedProperties.TargetId_ServicePrincipalNames.NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40", "o365.audit.ModifiedProperties.TargetId_ServicePrincipalNames.OldValue": "", "o365.audit.ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40", @@ -10973,8 +11043,18 @@ "o365.audit.ExtendedProperties.teamName": "MSODS.", "o365.audit.ExtendedProperties.version": "2", "o365.audit.Id": "48403af8-b712-4e63-a999-686b631240ac", + "o365.audit.ModifiedProperties.AccountEnabled.NewValue": "[\r\n true\r\n]", + "o365.audit.ModifiedProperties.AccountEnabled.OldValue": "[]", + "o365.audit.ModifiedProperties.AppPrincipalId.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.AppPrincipalId.OldValue": "[]", + "o365.audit.ModifiedProperties.Credential.NewValue": "[\r\n {\r\n \"CredentialType\": 2,\r\n \"KeyStoreId\": \"291154f0-a9f5-45bb-87be-9c8ee5b6d62c\",\r\n \"KeyGroupId\": \"291154f0-a9f5-45bb-87be-9c8ee5b6d62c\"\r\n }\r\n]", + "o365.audit.ModifiedProperties.Credential.OldValue": "[]", + "o365.audit.ModifiedProperties.DisplayName.NewValue": "[\r\n \"siem2\"\r\n]", + "o365.audit.ModifiedProperties.DisplayName.OldValue": "[]", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.ServicePrincipalName.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.ServicePrincipalName.OldValue": "[]", "o365.audit.ModifiedProperties.TargetId_ServicePrincipalNames.NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40", "o365.audit.ModifiedProperties.TargetId_ServicePrincipalNames.OldValue": "", "o365.audit.ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40", @@ -11128,8 +11208,18 @@ "o365.audit.ExtendedProperties.teamName": "MSODS.", "o365.audit.ExtendedProperties.version": "2", "o365.audit.Id": "48403af8-b712-4e63-a999-686b631240ac", + "o365.audit.ModifiedProperties.AccountEnabled.NewValue": "[\r\n true\r\n]", + "o365.audit.ModifiedProperties.AccountEnabled.OldValue": "[]", + "o365.audit.ModifiedProperties.AppPrincipalId.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.AppPrincipalId.OldValue": "[]", + "o365.audit.ModifiedProperties.Credential.NewValue": "[\r\n {\r\n \"CredentialType\": 2,\r\n \"KeyStoreId\": \"291154f0-a9f5-45bb-87be-9c8ee5b6d62c\",\r\n \"KeyGroupId\": \"291154f0-a9f5-45bb-87be-9c8ee5b6d62c\"\r\n }\r\n]", + "o365.audit.ModifiedProperties.Credential.OldValue": "[]", + "o365.audit.ModifiedProperties.DisplayName.NewValue": "[\r\n \"siem2\"\r\n]", + "o365.audit.ModifiedProperties.DisplayName.OldValue": "[]", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.ServicePrincipalName.NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]", + "o365.audit.ModifiedProperties.ServicePrincipalName.OldValue": "[]", "o365.audit.ModifiedProperties.TargetId_ServicePrincipalNames.NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40", "o365.audit.ModifiedProperties.TargetId_ServicePrincipalNames.OldValue": "", "o365.audit.ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40", @@ -11426,6 +11516,8 @@ "o365.audit.Id": "20a82fa1-625b-491a-a3e8-54d779a9b17e", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "KeyDescription", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.KeyDescription.NewValue": "[\r\n \"[KeyIdentifier=6d944a5f-234c-4879-8de4-39f089d8b96b,KeyType=AsymmetricX509Cert,KeyUsage=Verify,DisplayName=E=asr@example.net, CN=testsiem.onmicrosoft.com, OU=SIEM, O=Elastic, L=Barcelona, S=Barce]\"\r\n]", + "o365.audit.ModifiedProperties.KeyDescription.OldValue": "[]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application \u2013 Certificates and secrets management ", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -11570,6 +11662,8 @@ "o365.audit.Id": "20a82fa1-625b-491a-a3e8-54d779a9b17e", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "KeyDescription", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.KeyDescription.NewValue": "[\r\n \"[KeyIdentifier=6d944a5f-234c-4879-8de4-39f089d8b96b,KeyType=AsymmetricX509Cert,KeyUsage=Verify,DisplayName=E=asr@example.net, CN=testsiem.onmicrosoft.com, OU=SIEM, O=Elastic, L=Barcelona, S=Barce]\"\r\n]", + "o365.audit.ModifiedProperties.KeyDescription.OldValue": "[]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application \u2013 Certificates and secrets management ", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -12179,6 +12273,8 @@ "o365.audit.Id": "d23b201c-5436-4ecc-a789-18d3f00ea76c", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -12323,6 +12419,8 @@ "o365.audit.Id": "d23b201c-5436-4ecc-a789-18d3f00ea76c", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", @@ -12467,6 +12565,8 @@ "o365.audit.Id": "d23b201c-5436-4ecc-a789-18d3f00ea76c", "o365.audit.ModifiedProperties.Included_Updated_Properties.NewValue": "RequiredResourceAccess", "o365.audit.ModifiedProperties.Included_Updated_Properties.OldValue": "", + "o365.audit.ModifiedProperties.RequiredResourceAccess.NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", + "o365.audit.ModifiedProperties.RequiredResourceAccess.OldValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]", "o365.audit.ObjectId": "Not Available", "o365.audit.Operation": "Update application.", "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", diff --git a/x-pack/filebeat/module/zoom/webhook/config/webhook.yml b/x-pack/filebeat/module/zoom/webhook/config/webhook.yml index e58a9cb6045..6c2ed13fdba 100644 --- a/x-pack/filebeat/module/zoom/webhook/config/webhook.yml +++ b/x-pack/filebeat/module/zoom/webhook/config/webhook.yml @@ -3,10 +3,11 @@ type: http_endpoint listen_address: {{ .listen_address }} listen_port: {{ .listen_port }} +{{ if .url }}url: {{ .url}}{{ end }} prefix: {{ .prefix }} basic_auth: {{ .basic_auth }} username: {{ .username }} -username: {{ .password }} +password: {{ .password }} content_type: "{{ .content_type }}" secret: {{ .secret | tojson }} ssl: {{ .ssl | tojson }}