Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Elastic Agent] Fix issues with dynamic inputs and conditions #23886

Merged
merged 5 commits into from
Feb 11, 2021
Merged

[Elastic Agent] Fix issues with dynamic inputs and conditions #23886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ba0683f
Fix loading of configurations to not parse variables in inputs. Fix i…
blakerouse Feb 5, 2021
91a0ba6
Run fmt.
blakerouse Feb 5, 2021
b2baa3b
Add changelog entry.
blakerouse Feb 5, 2021
c62de28
Cleanup the config loading more.
blakerouse Feb 8, 2021
df7a3c5
Fix NewConfigFrom to handle maps.
blakerouse Feb 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions x-pack/elastic-agent/CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
- Fix issue of missing log messages from filebeat monitor {pull}23514[23514]
- Increase checkin grace period to 30 seconds {pull}23568[23568]
- Fix libbeat from reporting back degraded on config update {pull}23537[23537]
- Fix issues with dynamic inputs and conditions {pull}23886[23886]

==== New features

Expand Down
2 changes: 1 addition & 1 deletion x-pack/elastic-agent/pkg/agent/application/application.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func New(log *logger.Logger, pathConfigFile string, reexec reexecManager, uc upg
// Load configuration from disk to understand in which mode of operation
// we must start the elastic-agent, the mode of operation cannot be changed without restarting the
// elastic-agent.
rawConfig, err := LoadConfigFromFile(pathConfigFile)
rawConfig, err := config.LoadFile(pathConfigFile)
if err != nil {
return nil, err
}
Expand Down
58 changes: 0 additions & 58 deletions x-pack/elastic-agent/pkg/agent/application/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,8 @@
package application

import (
"io/ioutil"

"github.com/elastic/go-ucfg"

"gopkg.in/yaml.v2"

"github.com/elastic/beats/v7/libbeat/common"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
)

Expand All @@ -34,53 +26,3 @@ func createFleetConfigFromEnroll(accessAPIKey string, kbn *kibana.Config) (*conf
}
return cfg, nil
}

// LoadConfigFromFile loads the Agent configuration from a file.
//
// This must be used to load the Agent configuration, so that variables defined in the inputs are not
// parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler.
func LoadConfigFromFile(path string) (*config.Config, error) {
in, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
var m map[string]interface{}
if err := yaml.Unmarshal(in, &m); err != nil {
return nil, err
}
return LoadConfig(m)
}

// LoadConfig loads the Agent configuration from a map.
//
// This must be used to load the Agent configuration, so that variables defined in the inputs are not
// parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler.
func LoadConfig(in map[string]interface{}) (*config.Config, error) {
// make copy of a map so we dont affect a caller
m := common.MapStr(in).Clone()

inputs, ok := m["inputs"]
if ok {
// remove the inputs
delete(m, "inputs")
}
cfg, err := config.NewConfigFrom(m)
if err != nil {
return nil, err
}
if ok {
inputsOnly := map[string]interface{}{
"inputs": inputs,
}
// convert to config without variable substitution
inputsCfg, err := config.NewConfigFrom(inputsOnly, ucfg.PathSep("."), ucfg.ResolveNOOP)
if err != nil {
return nil, err
}
err = cfg.Merge(inputsCfg, ucfg.PathSep("."), ucfg.ResolveNOOP)
if err != nil {
return nil, err
}
}
return cfg, err
}
40 changes: 0 additions & 40 deletions x-pack/elastic-agent/pkg/agent/application/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ package application

import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"

Expand All @@ -20,44 +18,6 @@ import (
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
)

func TestLoadConfig(t *testing.T) {
contents := map[string]interface{}{
"outputs": map[string]interface{}{
"default": map[string]interface{}{
"type": "elasticsearch",
"hosts": []interface{}{"127.0.0.1:9200"},
"username": "elastic",
"password": "changeme",
},
},
"inputs": []interface{}{
map[string]interface{}{
"type": "logfile",
"streams": []interface{}{
map[string]interface{}{
"paths": []interface{}{"/var/log/${host.name}"},
},
},
},
},
}

tmp, err := ioutil.TempDir("", "config")
require.NoError(t, err)
defer os.RemoveAll(tmp)

cfgPath := filepath.Join(tmp, "config.yml")
dumpToYAML(t, cfgPath, contents)

cfg, err := LoadConfigFromFile(cfgPath)
require.NoError(t, err)

cfgData, err := cfg.ToMapStr()
require.NoError(t, err)

assert.Equal(t, contents, cfgData)
}

func TestConfig(t *testing.T) {
testMgmtMode(t)
testLocalConfig(t)
Expand Down
2 changes: 1 addition & 1 deletion x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func (h *handlerPolicyChange) Handle(ctx context.Context, a action, acker fleetA
return fmt.Errorf("invalid type, expected ActionPolicyChange and received %T", a)
}

c, err := LoadConfig(action.Policy)
c, err := config.NewConfigFrom(action.Policy)
if err != nil {
return errors.New(err, "could not parse the configuration from the policy", errors.TypeConfig)
}
Expand Down
2 changes: 1 addition & 1 deletion x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func (c *InspectConfigCmd) inspectConfig() error {
}

func loadConfig(configPath string) (*config.Config, error) {
rawConfig, err := LoadConfigFromFile(configPath)
rawConfig, err := config.LoadFile(configPath)
if err != nil {
return nil, err
}
Expand Down
3 changes: 2 additions & 1 deletion x-pack/elastic-agent/pkg/agent/cmd/enroll.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"time"

"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"

"github.com/spf13/cobra"

Expand Down Expand Up @@ -92,7 +93,7 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args
}

pathConfigFile := flags.Config()
rawConfig, err := application.LoadConfigFromFile(pathConfigFile)
rawConfig, err := config.LoadFile(pathConfigFile)
if err != nil {
return errors.New(err,
fmt.Sprintf("could not read configuration file %s", pathConfigFile),
Expand Down
2 changes: 1 addition & 1 deletion x-pack/elastic-agent/pkg/agent/cmd/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { // Windows: Mark se
service.HandleSignals(stopBeat, cancel)

pathConfigFile := flags.Config()
rawConfig, err := application.LoadConfigFromFile(pathConfigFile)
rawConfig, err := config.LoadFile(pathConfigFile)
if err != nil {
return errors.New(err,
fmt.Sprintf("could not read configuration file %s", pathConfigFile),
Expand Down
3 changes: 2 additions & 1 deletion x-pack/elastic-agent/pkg/agent/cmd/watch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
)
Expand Down Expand Up @@ -182,7 +183,7 @@ func gracePeriod(marker *upgrade.UpdateMarker) (bool, time.Duration) {

func configuredLogger(flags *globalFlags) (*logger.Logger, error) {
pathConfigFile := flags.Config()
rawConfig, err := application.LoadConfigFromFile(pathConfigFile)
rawConfig, err := config.LoadFile(pathConfigFile)
if err != nil {
return nil, errors.New(err,
fmt.Sprintf("could not read configuration file %s", pathConfigFile),
Expand Down
Loading