Mage: Validate modules.d dir in Filebeat + Disabled filesets

dev-tools/mage/modules.go

@@ -18,10 +18,15 @@
 package mage
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/joeshaw/multierror"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
 )
 
 var modulesDConfigTemplate = `
@@ -71,3 +76,68 @@ func GenerateDirModulesD() error {
 	}
 	return nil
 }
+
+type datasetDefinition struct {
+	Enabled *bool
+}
+
+type moduleDefinition struct {
+	Name     string                       `yaml:"module"`
+	Filesets map[string]datasetDefinition `yaml:",inline"`
+}
+
+// ValidateDirModulesD validates a modules.d directory containing the
+// <module>.yml.disabled files. It checks that the files are valid
+// yaml and conform to module definitions.
+func ValidateDirModulesD() error {
+	_, err := loadModulesD()
+	return err
+}
+
+// ValidateDirModulesDDatasetsDisabled ensures that all the datasets
+// are disabled by default.
+func ValidateDirModulesDDatasetsDisabled() error {
+	cfgs, err := loadModulesD()
+	if err != nil {
+		return err
+	}
+	var errs multierror.Errors
+	for path, cfg := range cfgs {
+		// A config.yml is a list of module configurations.
+		for modIdx, mod := range cfg {
+			// A module config is a map of datasets.
+			for dsName, ds := range mod.Filesets {
+				if ds.Enabled == nil || *ds.Enabled {
+					var entry string
+					if len(cfg) > 1 {
+						entry = fmt.Sprintf(" (entry #%d)", modIdx+1)
+					}
+					err = fmt.Errorf("in file '%s': %s module%s dataset %s must be explicitly disabled (needs `enabled: false`)",
+						path, mod.Name, entry, dsName)
+					errs = append(errs, err)
+				}
+			}
+		}
+	}
+	return errs.Err()
+}
+
+func loadModulesD() (modules map[string][]moduleDefinition, err error) {
+	files, err := filepath.Glob("modules.d/*.disabled")
+	if err != nil {
+		return nil, err
+	}
+	modules = make(map[string][]moduleDefinition, len(files))
+	for _, file := range files {
+		contents, err := ioutil.ReadFile(file)
+		if err != nil {
+			return nil, errors.Wrapf(err, "reading %s", file)
+		}
+		var cfg []moduleDefinition
+		if err = yaml.Unmarshal(contents, &cfg); err != nil {
+			return nil, errors.Wrapf(err, "parsing %s as YAML", file)
+		}
+		modules[file] = cfg
+	}
+	return modules, nil
+}

filebeat/magefile.go

@@ -123,6 +123,7 @@ func Update() {
 // modules.d directory.
 func Config() {
 	mg.Deps(devtools.GenerateDirModulesD, configYML)
+	mg.SerialDeps(devtools.ValidateDirModulesD, devtools.ValidateDirModulesDDatasetsDisabled)
 }
 
 func configYML() error {

x-pack/filebeat/magefile.go

@@ -130,6 +130,7 @@ func ExportDashboard() error {
 // Config generates both the short and reference configs.
 func Config() {
 	mg.Deps(configYML, devtools.GenerateDirModulesD)
+	mg.SerialDeps(devtools.ValidateDirModulesD, devtools.ValidateDirModulesDDatasetsDisabled)
 }
 
 func configYML() error {