[Elastic Agent] Fix issues with dynamic inputs and conditions (#23886) (

#24005)

* Fix loading of configurations to not parse variables in inputs. Fix issue with EQL string methods.

* Run fmt.

* Add changelog entry.

* Cleanup the config loading more.

* Fix NewConfigFrom to handle maps.

(cherry picked from commit de7906b)

x-pack/elastic-agent/CHANGELOG.next.asciidoc

@@ -36,6 +36,7 @@
 - Fix issue of missing log messages from filebeat monitor {pull}23514[23514]
 - Increase checkin grace period to 30 seconds {pull}23568[23568]
 - Fix libbeat from reporting back degraded on config update {pull}23537[23537]
+- Fix issues with dynamic inputs and conditions {pull}23886[23886]
 
 ==== New features
 

x-pack/elastic-agent/pkg/agent/application/application.go

@@ -35,7 +35,7 @@ func New(log *logger.Logger, pathConfigFile string, reexec reexecManager, uc upg
 	// Load configuration from disk to understand in which mode of operation
 	// we must start the elastic-agent, the mode of operation cannot be changed without restarting the
 	// elastic-agent.
-	rawConfig, err := LoadConfigFromFile(pathConfigFile)
+	rawConfig, err := config.LoadFile(pathConfigFile)
 	if err != nil {
 		return nil, err
 	}

x-pack/elastic-agent/pkg/agent/application/config.go

@@ -5,16 +5,8 @@
 package application
 
 import (
-	"io/ioutil"
-
-	"github.com/elastic/go-ucfg"
-
-	"gopkg.in/yaml.v2"
-
-	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
 )
 
@@ -34,53 +26,3 @@ func createFleetConfigFromEnroll(accessAPIKey string, kbn *kibana.Config) (*conf
 	}
 	return cfg, nil
 }
-
-// LoadConfigFromFile loads the Agent configuration from a file.
-//
-// This must be used to load the Agent configuration, so that variables defined in the inputs are not
-// parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler.
-func LoadConfigFromFile(path string) (*config.Config, error) {
-	in, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	var m map[string]interface{}
-	if err := yaml.Unmarshal(in, &m); err != nil {
-		return nil, err
-	}
-	return LoadConfig(m)
-}
-
-// LoadConfig loads the Agent configuration from a map.
-//
-// This must be used to load the Agent configuration, so that variables defined in the inputs are not
-// parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler.
-func LoadConfig(in map[string]interface{}) (*config.Config, error) {
-	// make copy of a map so we dont affect a caller
-	m := common.MapStr(in).Clone()
-
-	inputs, ok := m["inputs"]
-	if ok {
-		// remove the inputs
-		delete(m, "inputs")
-	}
-	cfg, err := config.NewConfigFrom(m)
-	if err != nil {
-		return nil, err
-	}
-	if ok {
-		inputsOnly := map[string]interface{}{
-			"inputs": inputs,
-		}
-		// convert to config without variable substitution
-		inputsCfg, err := config.NewConfigFrom(inputsOnly, ucfg.PathSep("."), ucfg.ResolveNOOP)
-		if err != nil {
-			return nil, err
-		}
-		err = cfg.Merge(inputsCfg, ucfg.PathSep("."), ucfg.ResolveNOOP)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return cfg, err
-}

x-pack/elastic-agent/pkg/agent/application/config_test.go

@@ -6,8 +6,6 @@ package application
 
 import (
 	"io/ioutil"
-	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -20,44 +18,6 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 )
 
-func TestLoadConfig(t *testing.T) {
-	contents := map[string]interface{}{
-		"outputs": map[string]interface{}{
-			"default": map[string]interface{}{
-				"type":     "elasticsearch",
-				"hosts":    []interface{}{"127.0.0.1:9200"},
-				"username": "elastic",
-				"password": "changeme",
-			},
-		},
-		"inputs": []interface{}{
-			map[string]interface{}{
-				"type": "logfile",
-				"streams": []interface{}{
-					map[string]interface{}{
-						"paths": []interface{}{"/var/log/${host.name}"},
-					},
-				},
-			},
-		},
-	}
-
-	tmp, err := ioutil.TempDir("", "config")
-	require.NoError(t, err)
-	defer os.RemoveAll(tmp)
-
-	cfgPath := filepath.Join(tmp, "config.yml")
-	dumpToYAML(t, cfgPath, contents)
-
-	cfg, err := LoadConfigFromFile(cfgPath)
-	require.NoError(t, err)
-
-	cfgData, err := cfg.ToMapStr()
-	require.NoError(t, err)
-
-	assert.Equal(t, contents, cfgData)
-}
-
 func TestConfig(t *testing.T) {
 	testMgmtMode(t)
 	testLocalConfig(t)

x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go

@@ -44,7 +44,7 @@ func (h *handlerPolicyChange) Handle(ctx context.Context, a action, acker fleetA
 		return fmt.Errorf("invalid type, expected ActionPolicyChange and received %T", a)
 	}
 
-	c, err := LoadConfig(action.Policy)
+	c, err := config.NewConfigFrom(action.Policy)
 	if err != nil {
 		return errors.New(err, "could not parse the configuration from the policy", errors.TypeConfig)
 	}

x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go

@@ -61,7 +61,7 @@ func (c *InspectConfigCmd) inspectConfig() error {
 }
 
 func loadConfig(configPath string) (*config.Config, error) {
-	rawConfig, err := LoadConfigFromFile(configPath)
+	rawConfig, err := config.LoadFile(configPath)
 	if err != nil {
 		return nil, err
 	}

x-pack/elastic-agent/pkg/agent/cmd/enroll.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 
 	"github.com/spf13/cobra"
 
@@ -92,7 +93,7 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args
 	}
 
 	pathConfigFile := flags.Config()
-	rawConfig, err := application.LoadConfigFromFile(pathConfigFile)
+	rawConfig, err := config.LoadFile(pathConfigFile)
 	if err != nil {
 		return errors.New(err,
 			fmt.Sprintf("could not read configuration file %s", pathConfigFile),

x-pack/elastic-agent/pkg/agent/cmd/run.go

@@ -79,7 +79,7 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { // Windows: Mark se
 	service.HandleSignals(stopBeat, cancel)
 
 	pathConfigFile := flags.Config()
-	rawConfig, err := application.LoadConfigFromFile(pathConfigFile)
+	rawConfig, err := config.LoadFile(pathConfigFile)
 	if err != nil {
 		return errors.New(err,
 			fmt.Sprintf("could not read configuration file %s", pathConfigFile),

x-pack/elastic-agent/pkg/agent/cmd/watch.go

@@ -21,6 +21,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
 )
@@ -182,7 +183,7 @@ func gracePeriod(marker *upgrade.UpdateMarker) (bool, time.Duration) {
 
 func configuredLogger(flags *globalFlags) (*logger.Logger, error) {
 	pathConfigFile := flags.Config()
-	rawConfig, err := application.LoadConfigFromFile(pathConfigFile)
+	rawConfig, err := config.LoadFile(pathConfigFile)
 	if err != nil {
 		return nil, errors.New(err,
 			fmt.Sprintf("could not read configuration file %s", pathConfigFile),