Auditbeat: Add include_paths to file integrity module (elastic#7829)

Add include_paths config option to file_integrity module.

Closes: elastic#7707

auditbeat/auditbeat.reference.yml

@@ -84,6 +84,11 @@ auditbeat.modules:
   - '~$'
   - '/\.git($|/)'
 
+  # List of regular expressions used to explicitly include files. When configured,
+  # Auditbeat will ignore files unless they match a pattern.
+  #include_files:
+  #- '/\.ssh($|/)'
+
   # Scan over the configured file paths at startup and send events for new or
   # modified files since the last time Auditbeat was running.
   scan_at_start: true

auditbeat/docs/modules/file_integrity.asciidoc

@@ -58,6 +58,7 @@ Linux.
   - '(?i)\.sw[nop]$'
   - '~$'
   - '/\.git($|/)'
+  include_files: []
   scan_at_start: true
   scan_rate_per_sec: 50 MiB
   max_file_size: 100 MiB
@@ -70,7 +71,19 @@ not supported. The specified paths should exist when the metricset is started.
 
 *`exclude_files`*:: A list of regular expressions used to filter out events
 for unwanted files. The expressions are matched against the full path of every
-file and directory. By default, no files are excluded. See <<regexp-support>>
+file and directory. When used in conjunction with `include_files`, file paths need
+to match both `include_files` and not match `exclude_files` to be selected.
+By default, no files are excluded. See <<regexp-support>>
+for a list of supported regexp patterns. It is recommended to wrap regular
+expressions in single quotation marks to avoid issues with YAML escaping
+rules.
+
+*`include_files`*:: A list of regular expressions used to specify which files to
+select. When configured, only files matching the pattern will be monitored.
+The expressions are matched against the full path of every file and directory.
+When used in conjunction with `exclude_files`, file paths need
+to match both `include_files` and not match `exclude_files` to be selected.
+By default, all files are selected. See <<regexp-support>>
 for a list of supported regexp patterns. It is recommended to wrap regular
 expressions in single quotation marks to avoid issues with YAML escaping
 rules.

auditbeat/module/file_integrity/_meta/config.yml.tmpl

@@ -44,6 +44,16 @@
   - '/\.git($|/)'
   {{- end }}
 
+  # List of regular expressions used to explicitly include files. When configured,
+  # Auditbeat will ignore files unless they match a pattern.
+  {{ if eq .GOOS "windows" -}}
+  #include_files:
+  #- '\\\.ssh($|\\)'
+  {{ else -}}
+  #include_files:
+  #- '/\.ssh($|/)'
+  {{- end }}
+
   # Scan over the configured file paths at startup and send events for new or
   # modified files since the last time Auditbeat was running.
   scan_at_start: true

auditbeat/module/file_integrity/_meta/docs.asciidoc

@@ -53,6 +53,7 @@ Linux.
   - '(?i)\.sw[nop]$'
   - '~$'
   - '/\.git($|/)'
+  include_files: []
   scan_at_start: true
   scan_rate_per_sec: 50 MiB
   max_file_size: 100 MiB
@@ -65,7 +66,19 @@ not supported. The specified paths should exist when the metricset is started.
 
 *`exclude_files`*:: A list of regular expressions used to filter out events
 for unwanted files. The expressions are matched against the full path of every
-file and directory. By default, no files are excluded. See <<regexp-support>>
+file and directory. When used in conjunction with `include_files`, file paths need
+to match both `include_files` and not match `exclude_files` to be selected.
+By default, no files are excluded. See <<regexp-support>>
+for a list of supported regexp patterns. It is recommended to wrap regular
+expressions in single quotation marks to avoid issues with YAML escaping
+rules.
+
+*`include_files`*:: A list of regular expressions used to specify which files to
+select. When configured, only files matching the pattern will be monitored.
+The expressions are matched against the full path of every file and directory.
+When used in conjunction with `exclude_files`, file paths need
+to match both `include_files` and not match `exclude_files` to be selected.
+By default, all files are selected. See <<regexp-support>>
 for a list of supported regexp patterns. It is recommended to wrap regular
 expressions in single quotation marks to avoid issues with YAML escaping
 rules.

auditbeat/module/file_integrity/config.go

@@ -79,6 +79,7 @@ type Config struct {
 	ScanRateBytesPerSec uint64          `config:",ignore"`
 	Recursive           bool            `config:"recursive"` // Recursive enables recursive monitoring of directories.
 	ExcludeFiles        []match.Matcher `config:"exclude_files"`
+	IncludeFiles        []match.Matcher `config:"include_files"`
 }
 
 // Validate validates the config data and return an error explaining all the
@@ -147,6 +148,20 @@ func (c *Config) IsExcludedPath(path string) bool {
 	return false
 }
 
+// IsIncludedPath checks if a path matches the include_files regular expressions.
+func (c *Config) IsIncludedPath(path string) bool {
+	if len(c.IncludeFiles) == 0 {
+		return true
+	}
+
+	for _, matcher := range c.IncludeFiles {
+		if matcher.MatchString(path) {
+			return true
+		}
+	}
+	return false
+}
+
 var defaultConfig = Config{
 	HashTypes:        []HashType{SHA1},
 	MaxFileSize:      "100 MiB",

auditbeat/module/file_integrity/config_test.go

@@ -37,6 +37,7 @@ func TestConfig(t *testing.T) {
 		"max_file_size":     "1 GiB",
 		"scan_rate_per_sec": "10MiB",
 		"exclude_files":     []string{`\.DS_Store$`, `\.swp$`},
+		"include_files":     []string{`\.ssh/$`},
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -53,6 +54,8 @@ func TestConfig(t *testing.T) {
 	assert.Len(t, c.ExcludeFiles, 2)
 	assert.EqualValues(t, `\.DS_Store(?-m:$)`, c.ExcludeFiles[0].String())
 	assert.EqualValues(t, `\.swp(?-m:$)`, c.ExcludeFiles[1].String())
+	assert.Len(t, c.IncludeFiles, 1)
+	assert.EqualValues(t, `\.ssh/(?-m:$)`, c.IncludeFiles[0].String())
 }
 
 func TestConfigInvalid(t *testing.T) {

auditbeat/module/file_integrity/eventreader_fsevents.go

@@ -153,7 +153,8 @@ func (r *fsreader) consumeEvents(done <-chan struct{}) {
 			return
 		case events := <-r.stream.Events:
 			for _, event := range events {
-				if !r.isWatched(event.Path) || r.config.IsExcludedPath(event.Path) {
+				if !r.isWatched(event.Path) || r.config.IsExcludedPath(event.Path) ||
+					!r.config.IsIncludedPath(event.Path) {
 					continue
 				}
 				r.log.Debugw("Received FSEvents event",

auditbeat/module/file_integrity/eventreader_fsnotify.go

@@ -89,7 +89,8 @@ func (r *reader) consumeEvents(done <-chan struct{}) {
 			r.log.Debug("fsnotify reader terminated")
 			return
 		case event := <-r.watcher.EventChannel():
-			if event.Name == "" || r.config.IsExcludedPath(event.Name) {
+			if event.Name == "" || r.config.IsExcludedPath(event.Name) ||
+				!r.config.IsIncludedPath(event.Name) {
 				continue
 			}
 			r.log.Debugw("Received fsnotify event",

auditbeat/module/file_integrity/metricset_test.go

@@ -176,6 +176,71 @@ func TestExcludedFiles(t *testing.T) {
 	}
 }
 
+func TestIncludedExcludedFiles(t *testing.T) {
+	defer setup(t)()
+
+	bucket, err := datastore.OpenBucket(bucketName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer bucket.Close()
+
+	dir, err := ioutil.TempDir("", "audit-file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	dir, err = filepath.EvalSymlinks(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.Mkdir(filepath.Join(dir, ".ssh"), 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	config := getConfig(dir)
+	config["include_files"] = []string{`\.ssh/`}
+	config["recursive"] = true
+	ms := mbtest.NewPushMetricSetV2(t, config)
+
+	go func() {
+		for _, f := range []string{"FILE.TXT", ".ssh/known_hosts", ".ssh/known_hosts.swp"} {
+			file := filepath.Join(dir, f)
+			err := ioutil.WriteFile(file, []byte("hello world"), 0600)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}()
+
+	events := mbtest.RunPushMetricSetV2(10*time.Second, 3, ms)
+	for _, e := range events {
+		if e.Error != nil {
+			t.Fatalf("received error: %+v", e.Error)
+		}
+	}
+
+	wanted := map[string]bool{
+		dir: true,
+		filepath.Join(dir, ".ssh"):             true,
+		filepath.Join(dir, ".ssh/known_hosts"): true,
+	}
+	if !assert.Len(t, events, len(wanted)) {
+		return
+	}
+	for _, e := range events {
+		event := e.MetricSetFields
+		path, err := event.GetValue("file.path")
+		if assert.NoError(t, err) {
+			_, ok := wanted[path.(string)]
+			assert.True(t, ok)
+		}
+	}
+}
+
 func setup(t testing.TB) func() {
 	// path.data should be set so that the DB is written to a predictable location.
 	var err error