Merge feature arch v2

NOTICE.txt

@@ -9867,11 +9867,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-a
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-client/v7
-Version: v7.0.0-20210727140539-f0905d9377f6
+Version: v7.0.0-20221028150015-05e494d37ccd
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.0.0-20210727140539-f0905d9377f6/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.0.0-20221028150015-05e494d37ccd/LICENSE.txt:
 
 ELASTIC LICENSE AGREEMENT
 
@@ -10100,11 +10100,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.2.11
+Version: v0.2.13
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.11/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.13/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -17908,11 +17908,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/mitchellh/mapstructure
-Version: v1.4.3
+Version: v1.5.0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/mitchellh/mapstructure@v1.4.3/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/mitchellh/mapstructure@v1.5.0/LICENSE:
 
 The MIT License (MIT)
 

filebeat/beater/filebeat.go

@@ -362,10 +362,9 @@ func (fb *Filebeat) Run(b *beat.Beat) error {
 
 	// Register reloadable list of inputs and modules
 	inputs := cfgfile.NewRunnerList(management.DebugK, inputLoader, fb.pipeline)
-	reload.Register.MustRegisterList("filebeat.inputs", inputs)
+	reload.RegisterV2.MustRegisterInput(inputs)
 
 	modules := cfgfile.NewRunnerList(management.DebugK, moduleLoader, fb.pipeline)
-	reload.Register.MustRegisterList("filebeat.modules", modules)
 
 	var adiscover *autodiscover.Autodiscover
 	if fb.config.Autodiscover != nil {

go.mod

@@ -70,7 +70,7 @@ require (
 	github.com/dustin/go-humanize v1.0.0
 	github.com/eapache/go-resiliency v1.2.0
 	github.com/eclipse/paho.mqtt.golang v1.3.5
-	github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6
+	github.com/elastic/elastic-agent-client/v7 v7.0.0-20221028150015-05e494d37ccd
 	github.com/elastic/go-concert v0.2.0
 	github.com/elastic/go-libaudit/v2 v2.3.2
 	github.com/elastic/go-licenser v0.4.0
@@ -124,7 +124,7 @@ require (
 	github.com/miekg/dns v1.1.42
 	github.com/mitchellh/gox v1.0.1
 	github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51
-	github.com/mitchellh/mapstructure v1.4.3
+	github.com/mitchellh/mapstructure v1.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/osquery/osquery-go v0.0.0-20210622151333-99b4efa62ec5
 	github.com/otiai10/copy v1.2.0
@@ -193,7 +193,7 @@ require (
 	github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/elastic-agent-autodiscover v0.4.0
-	github.com/elastic/elastic-agent-libs v0.2.11
+	github.com/elastic/elastic-agent-libs v0.2.13
 	github.com/elastic/elastic-agent-shipper-client v0.4.0
 	github.com/elastic/elastic-agent-system-metrics v0.4.5-0.20220927192933-25a985b07d51
 	github.com/elastic/go-elasticsearch/v8 v8.2.0

go.sum

@@ -615,10 +615,11 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/elastic-agent-autodiscover v0.4.0 h1:R1JMLHQpH2KP3GXY8zmgV4dj39uoe1asyPPWGQbGgSk=
 github.com/elastic/elastic-agent-autodiscover v0.4.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4=
-github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo=
-github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc=
-github.com/elastic/elastic-agent-libs v0.2.11 h1:ZeYn35Kxt+IdtMPmE01TaDeaahCg/z7MkGPVWUo6Lp4=
+github.com/elastic/elastic-agent-client/v7 v7.0.0-20221028150015-05e494d37ccd h1:IuAuac3vcucBrjAXKPQlTJ22H7mBUsSnNWxa7GZYFEg=
+github.com/elastic/elastic-agent-client/v7 v7.0.0-20221028150015-05e494d37ccd/go.mod h1:FEXUbFMfaV62S0CtJgD+FFHGY7+4o4fXkDicyONPSH8=
 github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
+github.com/elastic/elastic-agent-libs v0.2.13 h1:YQzhO8RaLosGlyt7IHtj/ZxigWiwLcXXlv3gS4QY9CA=
+github.com/elastic/elastic-agent-libs v0.2.13/go.mod h1:0J9lzJh+BjttIiVjYDLncKYCEWUUHiiqnuI64y6C6ss=
 github.com/elastic/elastic-agent-shipper-client v0.4.0 h1:nsTJF9oo4RHLl+zxFUZqNHaE86C6Ba5aImfegcEf6Sk=
 github.com/elastic/elastic-agent-shipper-client v0.4.0/go.mod h1:OyI2W+Mv3JxlkEF3OeT7K0dbuxvwew8ke2Cf4HpLa9Q=
 github.com/elastic/elastic-agent-system-metrics v0.4.5-0.20220927192933-25a985b07d51 h1:ZFk7hC6eRPJkJNtOSG+GYbRlsgLjSD8rTj4gQq+7rsA=
@@ -1366,8 +1367,9 @@ github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
 github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=

heartbeat/beater/heartbeat.go

@@ -222,10 +222,8 @@ func (bt *Heartbeat) RunCentralMgmtMonitors(b *beat.Beat) {
 		return nil
 	})
 
-	mons := cfgfile.NewRunnerList(management.DebugK, bt.monitorFactory, b.Publisher)
-	reload.Register.MustRegisterList(b.Info.Beat+".monitors", mons)
 	inputs := cfgfile.NewRunnerList(management.DebugK, bt.monitorFactory, b.Publisher)
-	reload.Register.MustRegisterList("inputs", inputs)
+	reload.RegisterV2.MustRegisterInput(inputs)
 }
 
 // RunReloadableMonitors runs the `heartbeat.config.monitors` portion of the yaml config if present.

libbeat/cfgfile/cfgfile.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/fleetmode"
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
 )
@@ -101,13 +102,13 @@ func HandleFlags() error {
 	home, err := filepath.Abs(filepath.Dir(os.Args[0]))
 	if err != nil {
 		if *homePath == "" {
-			return fmt.Errorf("The absolute path to %s could not be obtained. %v",
+			return fmt.Errorf("The absolute path to %s could not be obtained. %w",
 				os.Args[0], err)
 		}
 		home = *homePath
 	}
 
-	defaults.SetString("path.home", -1, home)
+	_ = defaults.SetString("path.home", -1, home)
 
 	if len(overwrites.GetFields()) > 0 {
 		common.PrintConfigDebugf(overwrites, "CLI setting overwrites (-E flag):")
@@ -133,30 +134,36 @@ func Read(out interface{}, path string) error {
 // Load reads the configuration from a YAML file structure. If path is empty
 // this method reads from the configuration file specified by the '-c' command
 // line flag.
+// This function cares about the underlying fleet setting, and if beats is running with
+// the management.enabled flag, Load() will bypass reading a config file, and merely merge any overrides.
 func Load(path string, beatOverrides []ConditionalOverride) (*config.C, error) {
 	var c *config.C
 	var err error
 
 	cfgpath := GetPathConfig()
 
-	if path == "" {
-		list := []string{}
-		for _, cfg := range configfiles.List() {
-			if !filepath.IsAbs(cfg) {
-				list = append(list, filepath.Join(cfgpath, cfg))
-			} else {
-				list = append(list, cfg)
+	if !fleetmode.Enabled() {
+		if path == "" {
+			list := []string{}
+			for _, cfg := range configfiles.List() {
+				if !filepath.IsAbs(cfg) {
+					list = append(list, filepath.Join(cfgpath, cfg))
+				} else {
+					list = append(list, cfg)
+				}
+			}
+			c, err = common.LoadFiles(list...)
+		} else {
+			if !filepath.IsAbs(path) {
+				path = filepath.Join(cfgpath, path)
 			}
+			c, err = common.LoadFile(path)
 		}
-		c, err = common.LoadFiles(list...)
-	} else {
-		if !filepath.IsAbs(path) {
-			path = filepath.Join(cfgpath, path)
+		if err != nil {
+			return nil, err
 		}
-		c, err = common.LoadFile(path)
-	}
-	if err != nil {
-		return nil, err
+	} else {
+		c = config.NewConfig()
 	}
 
 	if beatOverrides != nil {
@@ -183,6 +190,9 @@ func Load(path string, beatOverrides []ConditionalOverride) (*config.C, error) {
 			c,
 			overwrites,
 		)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	common.PrintConfigDebugf(c, "Complete configuration loaded:")
@@ -194,13 +204,13 @@ func LoadList(file string) ([]*config.C, error) {
 	logp.Debug("cfgfile", "Load config from file: %s", file)
 	rawConfig, err := common.LoadFile(file)
 	if err != nil {
-		return nil, fmt.Errorf("invalid config: %s", err)
+		return nil, fmt.Errorf("invalid config: %w", err)
 	}
 
 	var c []*config.C
 	err = rawConfig.Unpack(&c)
 	if err != nil {
-		return nil, fmt.Errorf("error reading configuration from file %s: %s", file, err)
+		return nil, fmt.Errorf("error reading configuration from file %s: %w", file, err)
 	}
 
 	return c, nil

libbeat/cmd/instance/beat.go

@@ -352,7 +352,7 @@ func (b *Beat) createBeater(bt beat.Creator) (beat.Beater, error) {
 		return nil, fmt.Errorf("error initializing publisher: %w", err)
 	}
 
-	reload.Register.MustRegister("output", b.makeOutputReloader(publisher.OutputReloader()))
+	reload.RegisterV2.MustRegisterOutput(b.makeOutputReloader(publisher.OutputReloader()))
 
 	// TODO: some beats race on shutdown with publisher.Stop -> do not call Stop yet,
 	//       but refine publisher to disconnect clients on stop automatically
@@ -721,7 +721,7 @@ func (b *Beat) configure(settings Settings) error {
 	logp.Info("Beat ID: %v", b.Info.ID)
 
 	// initialize config manager
-	b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.Register, b.Beat.Info.ID)
+	b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.RegisterV2, b.Beat.Info.ID)
 	if err != nil {
 		return err
 	}

libbeat/common/reload/reload.go

@@ -25,8 +25,14 @@ import (
 	"github.com/elastic/elastic-agent-libs/mapstr"
 )
 
-// Register holds a registry of reloadable objects
-var Register = NewRegistry()
+// RegisterV2 is the special registry used for the V2 controller
+var RegisterV2 = NewRegistry()
+
+// InputRegName is the registation name for V2 inputs
+const InputRegName = "input"
+
+// OutputRegName is the registation name for V2 Outputs
+const OutputRegName = "output"
 
 // ConfigWithMeta holds a pair of config.C and optional metadata for it
 type ConfigWithMeta struct {
@@ -106,13 +112,38 @@ func (r *Registry) MustRegister(name string, obj Reloadable) {
 	}
 }
 
-// MustRegisterList declares a reloadable object list
-func (r *Registry) MustRegisterList(name string, list ReloadableList) {
-	if err := r.RegisterList(name, list); err != nil {
+// MustRegisterOutput is a V2-specific registration function
+// That declares a reloadable output
+func (r *Registry) MustRegisterOutput(obj Reloadable) {
+	if err := r.Register(OutputRegName, obj); err != nil {
 		panic(err)
 	}
 }
 
+// MustRegisterInput is a V2-specific registration function
+// that declares a reloadable object list for a beat input
+func (r *Registry) MustRegisterInput(list ReloadableList) {
+	if err := r.RegisterList(InputRegName, list); err != nil {
+		panic(err)
+	}
+}
+
+// GetInputList is a V2-specific function
+// That returns the reloadable list created for an input
+func (r *Registry) GetInputList() ReloadableList {
+	r.RLock()
+	defer r.RUnlock()
+	return r.confsLists[InputRegName]
+}
+
+// GetReloadableOutput is a V2-specific function
+// That returns the reloader for the registered output
+func (r *Registry) GetReloadableOutput() Reloadable {
+	r.RLock()
+	defer r.RUnlock()
+	return r.confs[OutputRegName]
+}
+
 // GetRegisteredNames returns the list of names registered
 func (r *Registry) GetRegisteredNames() []string {
 	r.RLock()

metricbeat/beater/metricbeat.go

@@ -218,7 +218,7 @@ func (bt *Metricbeat) Run(b *beat.Beat) error {
 	// Centrally managed modules
 	factory := module.NewFactory(b.Info, bt.moduleOptions...)
 	modules := cfgfile.NewRunnerList(management.DebugK, factory, b.Publisher)
-	reload.Register.MustRegisterList(b.Info.Beat+".modules", modules)
+	reload.RegisterV2.MustRegisterInput(modules)
 	wg.Add(1)
 	go func() {
 		defer wg.Done()

packetbeat/beater/packetbeat.go

@@ -153,7 +153,7 @@ func (pb *packetbeat) runStatic(b *beat.Beat, factory *processorFactory) error {
 // the runner by starting the beat's manager. It returns on the first fatal error.
 func (pb *packetbeat) runManaged(b *beat.Beat, factory *processorFactory) error {
 	runner := newReloader(management.DebugK, factory, b.Publisher)
-	reload.Register.MustRegisterList("inputs", runner)
+	reload.RegisterV2.MustRegisterInput(runner)
 	logp.Debug("main", "Waiting for the runner to finish")
 
 	// Start the manager after all the hooks are registered and terminates when

x-pack/auditbeat/cmd/root.go

@@ -5,8 +5,15 @@
 package cmd
 
 import (
+	"fmt"
+	"strings"
+
 	auditbeatcmd "github.com/elastic/beats/v7/auditbeat/cmd"
 	"github.com/elastic/beats/v7/libbeat/cmd"
+	"github.com/elastic/beats/v7/libbeat/common/reload"
+	"github.com/elastic/beats/v7/x-pack/libbeat/management"
+	"github.com/elastic/elastic-agent-client/v7/pkg/client"
+	"github.com/elastic/elastic-agent-client/v7/pkg/proto"
 
 	// Register Auditbeat x-pack modules.
 	_ "github.com/elastic/beats/v7/x-pack/auditbeat/include"
@@ -19,7 +26,33 @@ var Name = auditbeatcmd.Name
 // RootCmd to handle beats CLI.
 var RootCmd *cmd.BeatsRootCmd
 
+// auditbeatCfg is a callback registered with central management to perform any needed config transformations
+// before agent configs are sent to a beat
+func auditbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) {
+	modules, err := management.CreateInputsFromStreams(rawIn, "metrics", agentInfo)
+	if err != nil {
+		return nil, fmt.Errorf("error creating input list from raw expected config: %w", err)
+	}
+
+	// Extract the type field that has "audit/auditd", treat this
+	// as the module config key
+	module := strings.Split(rawIn.Type, "/")[1]
+
+	for iter := range modules {
+		modules[iter]["module"] = module
+	}
+
+	// Format for the reloadable list needed bythe cm.Reload() method.
+	configList, err := management.CreateReloadConfigFromInputs(modules)
+	if err != nil {
+		return nil, fmt.Errorf("error creating reloader config: %w", err)
+	}
+
+	return configList, nil
+}
+
 func init() {
+	management.ConfigTransform.SetTransform(auditbeatCfg)
 	settings := auditbeatcmd.AuditbeatSettings()
 	settings.ElasticLicensed = true
 	RootCmd = auditbeatcmd.Initialize(settings)

x-pack/filebeat/cmd/agent.go

@@ -0,0 +1,28 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/elastic/beats/v7/libbeat/common/reload"
+	"github.com/elastic/beats/v7/x-pack/libbeat/management"
+	"github.com/elastic/elastic-agent-client/v7/pkg/client"
+	"github.com/elastic/elastic-agent-client/v7/pkg/proto"
+)
+
+func filebeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) {
+	modules, err := management.CreateInputsFromStreams(rawIn, "logs", agentInfo)
+	if err != nil {
+		return nil, fmt.Errorf("error creating input list from raw expected config: %w", err)
+	}
+
+	// format for the reloadable list needed bythe cm.Reload() method
+	configList, err := management.CreateReloadConfigFromInputs(modules)
+	if err != nil {
+		return nil, fmt.Errorf("error creating config for reloader: %w", err)
+	}
+	return configList, nil
+}