From 7e68c798fb5a04aebd64433603c4b212e86afa93 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Mon, 6 Aug 2018 14:11:38 +0100 Subject: [PATCH] Auditbeat: Add include_paths to file integrity module (#7829) Add include_paths config option to file_integrity module. Closes: elastic/beats#7707 --- auditbeat/auditbeat.reference.yml | 5 ++ .../docs/modules/file_integrity.asciidoc | 15 ++++- .../file_integrity/_meta/config.yml.tmpl | 10 +++ .../module/file_integrity/_meta/docs.asciidoc | 15 ++++- auditbeat/module/file_integrity/config.go | 15 +++++ .../module/file_integrity/config_test.go | 3 + .../file_integrity/eventreader_fsevents.go | 3 +- .../file_integrity/eventreader_fsnotify.go | 3 +- .../module/file_integrity/metricset_test.go | 65 +++++++++++++++++++ 9 files changed, 130 insertions(+), 4 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 43f0017d73b..cda70c00080 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -84,6 +84,11 @@ auditbeat.modules: - '~$' - '/\.git($|/)' + # List of regular expressions used to explicitly include files. When configured, + # Auditbeat will ignore files unless they match a pattern. + #include_files: + #- '/\.ssh($|/)' + # Scan over the configured file paths at startup and send events for new or # modified files since the last time Auditbeat was running. scan_at_start: true diff --git a/auditbeat/docs/modules/file_integrity.asciidoc b/auditbeat/docs/modules/file_integrity.asciidoc index 6c203a92c14..df896a7dd01 100644 --- a/auditbeat/docs/modules/file_integrity.asciidoc +++ b/auditbeat/docs/modules/file_integrity.asciidoc @@ -58,6 +58,7 @@ Linux. - '(?i)\.sw[nop]$' - '~$' - '/\.git($|/)' + include_files: [] scan_at_start: true scan_rate_per_sec: 50 MiB max_file_size: 100 MiB @@ -70,7 +71,19 @@ not supported. The specified paths should exist when the metricset is started. *`exclude_files`*:: A list of regular expressions used to filter out events for unwanted files. The expressions are matched against the full path of every -file and directory. By default, no files are excluded. See <> +file and directory. When used in conjunction with `include_files`, file paths need +to match both `include_files` and not match `exclude_files` to be selected. +By default, no files are excluded. See <> +for a list of supported regexp patterns. It is recommended to wrap regular +expressions in single quotation marks to avoid issues with YAML escaping +rules. + +*`include_files`*:: A list of regular expressions used to specify which files to +select. When configured, only files matching the pattern will be monitored. +The expressions are matched against the full path of every file and directory. +When used in conjunction with `exclude_files`, file paths need +to match both `include_files` and not match `exclude_files` to be selected. +By default, all files are selected. See <> for a list of supported regexp patterns. It is recommended to wrap regular expressions in single quotation marks to avoid issues with YAML escaping rules. diff --git a/auditbeat/module/file_integrity/_meta/config.yml.tmpl b/auditbeat/module/file_integrity/_meta/config.yml.tmpl index 774a430eba3..beaa9be26f8 100644 --- a/auditbeat/module/file_integrity/_meta/config.yml.tmpl +++ b/auditbeat/module/file_integrity/_meta/config.yml.tmpl @@ -44,6 +44,16 @@ - '/\.git($|/)' {{- end }} + # List of regular expressions used to explicitly include files. When configured, + # Auditbeat will ignore files unless they match a pattern. + {{ if eq .GOOS "windows" -}} + #include_files: + #- '\\\.ssh($|\\)' + {{ else -}} + #include_files: + #- '/\.ssh($|/)' + {{- end }} + # Scan over the configured file paths at startup and send events for new or # modified files since the last time Auditbeat was running. scan_at_start: true diff --git a/auditbeat/module/file_integrity/_meta/docs.asciidoc b/auditbeat/module/file_integrity/_meta/docs.asciidoc index f5f25abcd2e..74b6a19332e 100644 --- a/auditbeat/module/file_integrity/_meta/docs.asciidoc +++ b/auditbeat/module/file_integrity/_meta/docs.asciidoc @@ -53,6 +53,7 @@ Linux. - '(?i)\.sw[nop]$' - '~$' - '/\.git($|/)' + include_files: [] scan_at_start: true scan_rate_per_sec: 50 MiB max_file_size: 100 MiB @@ -65,7 +66,19 @@ not supported. The specified paths should exist when the metricset is started. *`exclude_files`*:: A list of regular expressions used to filter out events for unwanted files. The expressions are matched against the full path of every -file and directory. By default, no files are excluded. See <> +file and directory. When used in conjunction with `include_files`, file paths need +to match both `include_files` and not match `exclude_files` to be selected. +By default, no files are excluded. See <> +for a list of supported regexp patterns. It is recommended to wrap regular +expressions in single quotation marks to avoid issues with YAML escaping +rules. + +*`include_files`*:: A list of regular expressions used to specify which files to +select. When configured, only files matching the pattern will be monitored. +The expressions are matched against the full path of every file and directory. +When used in conjunction with `exclude_files`, file paths need +to match both `include_files` and not match `exclude_files` to be selected. +By default, all files are selected. See <> for a list of supported regexp patterns. It is recommended to wrap regular expressions in single quotation marks to avoid issues with YAML escaping rules. diff --git a/auditbeat/module/file_integrity/config.go b/auditbeat/module/file_integrity/config.go index 17962e71eeb..6b76deb6ac8 100644 --- a/auditbeat/module/file_integrity/config.go +++ b/auditbeat/module/file_integrity/config.go @@ -79,6 +79,7 @@ type Config struct { ScanRateBytesPerSec uint64 `config:",ignore"` Recursive bool `config:"recursive"` // Recursive enables recursive monitoring of directories. ExcludeFiles []match.Matcher `config:"exclude_files"` + IncludeFiles []match.Matcher `config:"include_files"` } // Validate validates the config data and return an error explaining all the @@ -147,6 +148,20 @@ func (c *Config) IsExcludedPath(path string) bool { return false } +// IsIncludedPath checks if a path matches the include_files regular expressions. +func (c *Config) IsIncludedPath(path string) bool { + if len(c.IncludeFiles) == 0 { + return true + } + + for _, matcher := range c.IncludeFiles { + if matcher.MatchString(path) { + return true + } + } + return false +} + var defaultConfig = Config{ HashTypes: []HashType{SHA1}, MaxFileSize: "100 MiB", diff --git a/auditbeat/module/file_integrity/config_test.go b/auditbeat/module/file_integrity/config_test.go index 67c1a12b829..202f186e29d 100644 --- a/auditbeat/module/file_integrity/config_test.go +++ b/auditbeat/module/file_integrity/config_test.go @@ -37,6 +37,7 @@ func TestConfig(t *testing.T) { "max_file_size": "1 GiB", "scan_rate_per_sec": "10MiB", "exclude_files": []string{`\.DS_Store$`, `\.swp$`}, + "include_files": []string{`\.ssh/$`}, }) if err != nil { t.Fatal(err) @@ -53,6 +54,8 @@ func TestConfig(t *testing.T) { assert.Len(t, c.ExcludeFiles, 2) assert.EqualValues(t, `\.DS_Store(?-m:$)`, c.ExcludeFiles[0].String()) assert.EqualValues(t, `\.swp(?-m:$)`, c.ExcludeFiles[1].String()) + assert.Len(t, c.IncludeFiles, 1) + assert.EqualValues(t, `\.ssh/(?-m:$)`, c.IncludeFiles[0].String()) } func TestConfigInvalid(t *testing.T) { diff --git a/auditbeat/module/file_integrity/eventreader_fsevents.go b/auditbeat/module/file_integrity/eventreader_fsevents.go index 5de8b82497f..e9f5384e733 100644 --- a/auditbeat/module/file_integrity/eventreader_fsevents.go +++ b/auditbeat/module/file_integrity/eventreader_fsevents.go @@ -153,7 +153,8 @@ func (r *fsreader) consumeEvents(done <-chan struct{}) { return case events := <-r.stream.Events: for _, event := range events { - if !r.isWatched(event.Path) || r.config.IsExcludedPath(event.Path) { + if !r.isWatched(event.Path) || r.config.IsExcludedPath(event.Path) || + !r.config.IsIncludedPath(event.Path) { continue } r.log.Debugw("Received FSEvents event", diff --git a/auditbeat/module/file_integrity/eventreader_fsnotify.go b/auditbeat/module/file_integrity/eventreader_fsnotify.go index 7417147e9f9..c08e8186a6e 100644 --- a/auditbeat/module/file_integrity/eventreader_fsnotify.go +++ b/auditbeat/module/file_integrity/eventreader_fsnotify.go @@ -89,7 +89,8 @@ func (r *reader) consumeEvents(done <-chan struct{}) { r.log.Debug("fsnotify reader terminated") return case event := <-r.watcher.EventChannel(): - if event.Name == "" || r.config.IsExcludedPath(event.Name) { + if event.Name == "" || r.config.IsExcludedPath(event.Name) || + !r.config.IsIncludedPath(event.Name) { continue } r.log.Debugw("Received fsnotify event", diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go index f4f8c8ba501..38da93f5ff6 100644 --- a/auditbeat/module/file_integrity/metricset_test.go +++ b/auditbeat/module/file_integrity/metricset_test.go @@ -176,6 +176,71 @@ func TestExcludedFiles(t *testing.T) { } } +func TestIncludedExcludedFiles(t *testing.T) { + defer setup(t)() + + bucket, err := datastore.OpenBucket(bucketName) + if err != nil { + t.Fatal(err) + } + defer bucket.Close() + + dir, err := ioutil.TempDir("", "audit-file") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + dir, err = filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + + err = os.Mkdir(filepath.Join(dir, ".ssh"), 0700) + if err != nil { + t.Fatal(err) + } + + config := getConfig(dir) + config["include_files"] = []string{`\.ssh/`} + config["recursive"] = true + ms := mbtest.NewPushMetricSetV2(t, config) + + go func() { + for _, f := range []string{"FILE.TXT", ".ssh/known_hosts", ".ssh/known_hosts.swp"} { + file := filepath.Join(dir, f) + err := ioutil.WriteFile(file, []byte("hello world"), 0600) + if err != nil { + t.Fatal(err) + } + } + }() + + events := mbtest.RunPushMetricSetV2(10*time.Second, 3, ms) + for _, e := range events { + if e.Error != nil { + t.Fatalf("received error: %+v", e.Error) + } + } + + wanted := map[string]bool{ + dir: true, + filepath.Join(dir, ".ssh"): true, + filepath.Join(dir, ".ssh/known_hosts"): true, + } + if !assert.Len(t, events, len(wanted)) { + return + } + for _, e := range events { + event := e.MetricSetFields + path, err := event.GetValue("file.path") + if assert.NoError(t, err) { + _, ok := wanted[path.(string)] + assert.True(t, ok) + } + } +} + func setup(t testing.TB) func() { // path.data should be set so that the DB is written to a predictable location. var err error