From 09abf38013f3080ad056807c84486e5d892316d3 Mon Sep 17 00:00:00 2001 From: Amir Ben Nun <34831306+amirbenun@users.noreply.github.com> Date: Wed, 9 Nov 2022 16:18:13 +0200 Subject: [PATCH 1/7] Set OPA default log level (#505) --- beater/cloudbeat.go | 2 +- cmd/root.go | 25 +++++++++++++++++++++++++ go.mod | 8 +++++--- go.sum | 15 ++++++++++----- launcher/launcher_test.go | 2 +- 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/beater/cloudbeat.go b/beater/cloudbeat.go index df6ed61484..eb80c50bdf 100644 --- a/beater/cloudbeat.go +++ b/beater/cloudbeat.go @@ -71,7 +71,7 @@ func New(b *beat.Beat, cfg *agentconfig.C) (beat.Beater, error) { return nil, err } - reload.Register.MustRegisterList("inputs", reloader) + reload.RegisterV2.MustRegisterInput(reloader) return s, nil } diff --git a/cmd/root.go b/cmd/root.go index 3d8ccb594e..fc892900a3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -18,12 +18,18 @@ package cmd import ( + "fmt" + "github.com/elastic/cloudbeat/beater" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" cmd "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" + "github.com/elastic/beats/v7/libbeat/common/reload" _ "github.com/elastic/beats/v7/x-pack/libbeat/include" + "github.com/elastic/beats/v7/x-pack/libbeat/management" ) // Name of this beat @@ -31,3 +37,22 @@ var Name = "cloudbeat" // RootCmd to handle beats cli var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{Name: Name, Version: defaultBeatVersion}) + +func cloudbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + modules, err := management.CreateInputsFromStreams(rawIn, "logs", agentInfo) + if err != nil { + return nil, fmt.Errorf("error creating input list from raw expected config: %w", err) + } + + // format for the reloadable list needed bythe cm.Reload() method + configList, err := management.CreateReloadConfigFromInputs(modules) + if err != nil { + return nil, fmt.Errorf("error creating reloader config: %w", err) + } + + return configList, nil +} + +func init() { + management.ConfigTransform.SetTransform(cloudbeatCfg) +} diff --git a/go.mod b/go.mod index 036eabcfbe..88598b5dad 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/magefile/mage v1.14.0 github.com/mailru/easyjson v0.7.6 // indirect github.com/mitchellh/gox v1.0.1 - github.com/mitchellh/mapstructure v1.4.3 + github.com/mitchellh/mapstructure v1.5.0 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0 @@ -165,12 +165,12 @@ require ( github.com/dop251/goja_nodejs v0.0.0-20220712185256-8cad205bf387 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/eapache/go-resiliency v1.2.0 // indirect - github.com/elastic/elastic-agent-client/v7 v7.0.0-20220607160924-1a71765a8bbe // indirect + github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 // indirect github.com/elastic/elastic-agent-system-metrics v0.4.4 // indirect github.com/elastic/go-concert v0.2.0 // indirect github.com/elastic/go-lumber v0.1.0 // indirect github.com/elastic/go-seccomp-bpf v1.2.0 // indirect - github.com/elastic/go-structform v0.0.9 // indirect + github.com/elastic/go-structform v0.0.10 // indirect github.com/elastic/go-sysinfo v1.8.1 // indirect github.com/elastic/go-ucfg v0.8.6 github.com/elastic/go-windows v1.0.1 // indirect @@ -262,6 +262,8 @@ replace ( github.com/apoydence/eachers => github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 //indirect, see https://github.com/elastic/beats/pull/29780 for details. github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 + //github.com/elastic/beats/v7 => github.com/fearful-symmetry/beats/v7 v1.2.2-0.20220825171527-b0d25807e2ab + github.com/elastic/beats/v7 => github.com/eyalkraft/beats/v7 v7.0.0-testv2 // forked from ^ github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v1.4.8-0.20211018144411-a81f2b630e7c github.com/golang/glog => github.com/elastic/glog v1.0.1-0.20210831205241-7d8b5c89dfc4 github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a diff --git a/go.sum b/go.sum index 51e6608e7d..1630d84fb2 100644 --- a/go.sum +++ b/go.sum @@ -539,8 +539,8 @@ github.com/elastic/e2e-testing v1.99.2-0.20220117192005-d3365c99b9c4 h1:uYT+Krd8 github.com/elastic/e2e-testing v1.99.2-0.20220117192005-d3365c99b9c4/go.mod h1:UcNuf4pX/qDVNQr0zybm1NL2YoWik+jKBaINZqQCA40= github.com/elastic/elastic-agent-autodiscover v0.2.1 h1:Nbeayh3vq2FNm6xaFo34mhUdOu0EVlpj53CqCsbU0E4= github.com/elastic/elastic-agent-autodiscover v0.2.1/go.mod h1:gPnzzfdYNdgznAb+iG9eyyXaQXBbAMHa+Y6Z8hXfcGY= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20220607160924-1a71765a8bbe h1:knHO5fYAT3+sLf64NxcOxdjGysPxsSMkQGB27vMS8TE= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20220607160924-1a71765a8bbe/go.mod h1:fkvyUfFwyAG5OnMF0h+FV9sC0Xn9YLITwQpSuwungQs= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 h1:uJIMfLgCenJvxsVmEjBjYGxt0JddCgw2IxgoNfcIXOk= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484/go.mod h1:fkvyUfFwyAG5OnMF0h+FV9sC0Xn9YLITwQpSuwungQs= github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-libs v0.2.15 h1:hdAbrZZ2mCPcQLRCE3E8xw3mHKl8HFMt36w7jan/XGo= github.com/elastic/elastic-agent-libs v0.2.15/go.mod h1:0J9lzJh+BjttIiVjYDLncKYCEWUUHiiqnuI64y6C6ss= @@ -563,8 +563,9 @@ github.com/elastic/go-lumber v0.1.0 h1:HUjpyg36v2HoKtXlEC53EJ3zDFiDRn65d7B8dBHNi github.com/elastic/go-lumber v0.1.0/go.mod h1:8YvjMIRYypWuPvpxx7WoijBYdbB7XIh/9FqSYQZTtxQ= github.com/elastic/go-seccomp-bpf v1.2.0 h1:K5fToUAMzm0pmdlYORmw0FP0DloRa1SfqRYkum647Yk= github.com/elastic/go-seccomp-bpf v1.2.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE+vxCFzzvQwN7T8= -github.com/elastic/go-structform v0.0.9 h1:HpcS7xljL4kSyUfDJ8cXTJC6rU5ChL1wYb6cx3HLD+o= github.com/elastic/go-structform v0.0.9/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= +github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w= +github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-sysinfo v1.8.1 h1:4Yhj+HdV6WjbCRgGdZpPJ8lZQlXZLKDAeIkmQ/VRvi4= @@ -606,6 +607,8 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/eyalkraft/beats/v7 v7.0.0-testv2 h1:sgLR01TfyP8RSRkUJK5fu7ItQP664odTrZSpk0KZk/A= +github.com/eyalkraft/beats/v7 v7.0.0-testv2/go.mod h1:klCBW4dWSbh8b/YTSLcSWVXSdxObCd+8/b3AqvcD84E= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -1154,8 +1157,9 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= @@ -1544,8 +1548,9 @@ github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.elastic.co/apm v1.10.0/go.mod h1:qoOSi09pnzJDh5fKnfY7bPmQgl8yl2tULdOu03xhui0= -go.elastic.co/apm v1.13.0 h1:SUCDscFfpshuxoxuoSh+r+2E3v1uvd5HrlRK5ZHpyvs= go.elastic.co/apm v1.13.0/go.mod h1:dylGv2HKR0tiCV+wliJz1KHtDyuD8SPe69oV7VyK6WY= +go.elastic.co/apm v1.15.0 h1:uPk2g/whK7c7XiZyz/YCUnAUBNPiyNeE3ARX3G6Gx7Q= +go.elastic.co/apm v1.15.0/go.mod h1:dylGv2HKR0tiCV+wliJz1KHtDyuD8SPe69oV7VyK6WY= go.elastic.co/apm/module/apmelasticsearch v1.10.0/go.mod h1:lwoaGDfZzfb9e6TXd3h8/KNmLAONOas7o5NLVNmv8Xk= go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0 h1:5UaI4agfuGoGRhpFVb6s63Hj/9xtqYxF7kd6T77tNAw= go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0/go.mod h1:GmYz+KDp2LDAa2nd/qJ+OkrjWVEoCRIPAWarbUvyt6Y= diff --git a/launcher/launcher_test.go b/launcher/launcher_test.go index b415c1272f..b84ae34454 100644 --- a/launcher/launcher_test.go +++ b/launcher/launcher_test.go @@ -174,7 +174,7 @@ func (s *LauncherTestSuite) MockBeatManager(mocks *launcherMocks) { } b, err := instance.NewInitializedBeat(settings) s.NoError(err) - b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.Register, b.Beat.Info.ID) + b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.RegisterV2, b.Beat.Info.ID) s.NoError(err) mocks.beat = &b.Beat } From 348450da359119970adf38f1a71380afbda59130 Mon Sep 17 00:00:00 2001 From: Oleg Sucharevich Date: Sun, 30 Oct 2022 16:30:43 +0200 Subject: [PATCH 2/7] feat: align to elastic-agent v2 api --- beater/validator.go | 7 +- beater/validator_test.go | 49 ++---- config/config.go | 22 +-- config/config_test.go | 162 ++++++++---------- resources/fetchersManager/factory_aws_test.go | 10 +- resources/fetchersManager/factory_test.go | 43 +++-- 6 files changed, 120 insertions(+), 173 deletions(-) diff --git a/beater/validator.go b/beater/validator.go index 70c070c717..d2bcf0ed7a 100644 --- a/beater/validator.go +++ b/beater/validator.go @@ -27,14 +27,9 @@ import ( agentconfig "github.com/elastic/elastic-agent-libs/config" ) -type validator struct { -} +type validator struct{} func (v *validator) Validate(cfg *agentconfig.C) error { - if !cfg.HasField("streams") { - return fmt.Errorf("no streams in config") - } - c, err := config.New(cfg) if err != nil { return fmt.Errorf("could not parse reconfiguration %v, skipping with error: %v", cfg.FlattenedKeys(), err) diff --git a/beater/validator_test.go b/beater/validator_test.go index 19f4f7c6c8..4df458801e 100644 --- a/beater/validator_test.go +++ b/beater/validator_test.go @@ -45,39 +45,15 @@ func TestValidatorTestSuite(t *testing.T) { } func (s *ValidatorTestSuite) TestConfig() { - configNoStreams := config.MustNewConfigFrom(` -not_streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d - - e -`) - - configNoRuntimeCfg := config.MustNewConfigFrom(` -streams: - - not_runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d - - e -`) configWithRuntimeCfg := config.MustNewConfigFrom(` -streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d - - e +runtime_cfg: + activated_rules: + cis_k8s: + - a + - b + - c + - d + - e `) testcases := []struct { @@ -87,13 +63,8 @@ streams: { true, config.NewConfig(), - }, { - true, - configNoRuntimeCfg, - }, { - true, - configNoStreams, - }, { + }, + { false, configWithRuntimeCfg, }, diff --git a/config/config.go b/config/config.go index 9a28e9a13d..c5529a89bb 100644 --- a/config/config.go +++ b/config/config.go @@ -44,11 +44,6 @@ type Fetcher struct { Name string `config:"name"` // Name of the fetcher } -type AgentInput struct { - Streams []Stream `config:"streams"` - Type string `config:"type"` -} - type Stream struct { AWSConfig aws.ConfigAWS `config:",inline"` RuntimeCfg *RuntimeConfig `config:"runtime_cfg"` @@ -77,23 +72,22 @@ type Benchmarks struct { CisEks []string `config:"cis_eks,omitempty" yaml:"cis_eks,omitempty" json:"cis_eks,omitempty"` } -var DefaultConfig = AgentInput{ - Type: InputTypeVanillaK8s, - Streams: []Stream{{ - Period: 4 * time.Hour, - }}, +var DefaultConfig = Stream{ + Period: 4 * time.Hour, } func New(cfg *config.C) (Config, error) { c := DefaultConfig - if err := cfg.Unpack(&c); err != nil { return Config{}, err } - + inputType := InputTypeVanillaK8s + if c.RuntimeCfg != nil && c.RuntimeCfg.ActivatedRules != nil && len(c.RuntimeCfg.ActivatedRules.CisEks) > 0 { + inputType = InputTypeEks + } return Config{ - Stream: c.Streams[0], - Type: c.Type, + Stream: c, + Type: inputType, }, nil } diff --git a/config/config_test.go b/config/config_test.go index 2b96f30642..dc4b22b217 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -56,51 +56,48 @@ func (s *ConfigTestSuite) TestNew() { }{ { ` - type : cloudbeat/cis_k8s - streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d - - e - fetchers: - - name: a - directory: b - - name: b - directory: b +runtime_cfg: + activated_rules: + cis_k8s: + - a + - b + - c + - d + - e +fetchers: + - name: a + directory: b + - name: b + directory: b `, &Benchmarks{CisK8s: []string{"a", "b", "c", "d", "e"}}, "cloudbeat/cis_k8s", aws.ConfigAWS{}, 2, - }, { + }, + { ` - type : cloudbeat/cis_eks - streams: - - runtime_cfg: - activated_rules: - cis_eks: - - a - - b - - c - - d - - e - access_key_id: key - secret_access_key: secret - session_token: session - shared_credential_file: shared_credential_file - credential_profile_name: credential_profile_name - role_arn: role_arn - fetchers: - - name: a - directory: b - - name: b - directory: b - - name: c - directory: c +runtime_cfg: + activated_rules: + cis_eks: + - a + - b + - c + - d + - e +access_key_id: key +secret_access_key: secret +session_token: session +shared_credential_file: shared_credential_file +credential_profile_name: credential_profile_name +role_arn: role_arn +fetchers: + - name: a + directory: b + - name: b + directory: b + - name: c + directory: c `, &Benchmarks{CisEks: []string{"a", "b", "c", "d", "e"}}, "cloudbeat/cis_eks", @@ -137,23 +134,21 @@ func (s *ConfigTestSuite) TestRuntimeCfgExists() { }{ { ` - streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d - - e +runtime_cfg: + activated_rules: + cis_k8s: + - a + - b + - c + - d + - e `, true, }, { ` - streams: - - not_runtime_cfg: - something: true +not_runtime_cfg: + something: true `, false, }, @@ -177,14 +172,13 @@ func (s *ConfigTestSuite) TestRuntimeConfig() { }{ { ` - streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d +runtime_cfg: + activated_rules: + cis_k8s: + - a + - b + - c + - d `, []string{"a", "b", "c", "d"}, }, } @@ -207,10 +201,10 @@ func (s *ConfigTestSuite) TestRuntimeEvaluatorConfig() { config string expected EvaluatorConfig }{ - {` - streams: - - evaluator: - decision_logs: true + { + ` +evaluator: + decision_logs: true `, EvaluatorConfig{ DecisionLogs: true, @@ -237,19 +231,19 @@ func (s *ConfigTestSuite) TestConfigPeriod() { {"", 4 * time.Hour}, { ` - streams: - - period: 50s -`, 50 * time.Second}, + period: 50s +`, 50 * time.Second, + }, { ` - streams: - - period: 5m -`, 5 * time.Minute}, + period: 5m +`, 5 * time.Minute, + }, { ` - streams: - - period: 2h -`, 2 * time.Hour}, + period: 2h +`, 2 * time.Hour, + }, } for _, test := range tests { @@ -272,13 +266,11 @@ func (s *ConfigTestSuite) TestActivatedRulesFrameWork() { }{ { ` -type: cloudbeat/cis_k8s -streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b +runtime_cfg: + activated_rules: + cis_k8s: + - a + - b `, []string{"a", "b"}, nil, @@ -286,13 +278,11 @@ streams: }, { ` -type: cloudbeat/cis_eks -streams: - - runtime_cfg: - activated_rules: - cis_eks: - - a - - b +runtime_cfg: + activated_rules: + cis_eks: + - a + - b `, nil, []string{"a", "b"}, diff --git a/resources/fetchersManager/factory_aws_test.go b/resources/fetchersManager/factory_aws_test.go index 68ee1f7207..2f2c243b6f 100644 --- a/resources/fetchersManager/factory_aws_test.go +++ b/resources/fetchersManager/factory_aws_test.go @@ -19,6 +19,7 @@ package fetchersManager import ( "context" + "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/cloudbeat/config" "github.com/elastic/cloudbeat/resources/fetching" @@ -78,8 +79,6 @@ func (n *awsTestFactory) Create(log *logp.Logger, c *agentconfig.C, ch chan fetc func awsMockedFetcherConfig(s *FactoriesTestSuite, awsConfig aws.ConfigAWS) *agentconfig.C { c := agentconfig.NewConfig() - conf := config.DefaultConfig - conf.Type = config.InputTypeEks err := c.Merge(awsConfig) s.NoError(err) @@ -87,11 +86,12 @@ func awsMockedFetcherConfig(s *FactoriesTestSuite, awsConfig aws.ConfigAWS) *age } func (s *FactoriesTestSuite) TestCreateFetcherWithAwsCredentials() { - var tests = []struct { + tests := []struct { fetcherName string awsConfig aws.ConfigAWS }{ - {"some_fetcher", + { + "some_fetcher", aws.ConfigAWS{ AccessKeyID: "key", SecretAccessKey: "secret", @@ -120,7 +120,7 @@ func (s *FactoriesTestSuite) TestCreateFetcherWithAwsCredentials() { } func (s *FactoriesTestSuite) TestRegisterFetchersWithAwsCredentials() { - var tests = []struct { + tests := []struct { fetcherName string awsConfig aws.ConfigAWS }{ diff --git a/resources/fetchersManager/factory_test.go b/resources/fetchersManager/factory_test.go index b5719fa5fe..3c0da8b184 100644 --- a/resources/fetchersManager/factory_test.go +++ b/resources/fetchersManager/factory_test.go @@ -19,9 +19,10 @@ package fetchersManager import ( "context" + "testing" + "github.com/elastic/cloudbeat/resources/utils/testhelper" "github.com/elastic/elastic-agent-libs/logp" - "testing" "github.com/elastic/cloudbeat/config" "github.com/elastic/cloudbeat/resources/fetching" @@ -98,7 +99,7 @@ func (s *FactoriesTestSuite) TearDownTest() { } func (s *FactoriesTestSuite) TestListFetcher() { - var tests = []struct { + tests := []struct { key string }{ {"process"}, @@ -114,7 +115,7 @@ func (s *FactoriesTestSuite) TestListFetcher() { } func (s *FactoriesTestSuite) TestCreateFetcher() { - var tests = []struct { + tests := []struct { key string value int }{ @@ -138,7 +139,7 @@ func (s *FactoriesTestSuite) TestCreateFetcher() { } func (s *FactoriesTestSuite) TestCreateFetcherCollision() { - var tests = []struct { + tests := []struct { key string }{ {"process"}, @@ -153,7 +154,7 @@ func (s *FactoriesTestSuite) TestCreateFetcherCollision() { } func (s *FactoriesTestSuite) TestRegisterFetchers() { - var tests = []struct { + tests := []struct { key string value int integrationType string @@ -192,7 +193,7 @@ func (s *FactoriesTestSuite) TestRegisterFetchers() { } func (s *FactoriesTestSuite) TestRegisterNotFoundFetchers() { - var tests = []struct { + tests := []struct { key string value int }{ @@ -213,31 +214,27 @@ func (s *FactoriesTestSuite) TestRegisterNotFoundFetchers() { } func (s *FactoriesTestSuite) TestRegisterFromFullConfig() { - var tests = []struct { + tests := []struct { config string }{ { ` -type: cloudbeat/cis_k8s -streams: - - not_data_yaml: - activated_rules: - cis_k8s: - - a - fetchers: - - name: process +not_data_yaml: + activated_rules: + cis_k8s: + - a +fetchers: + - name: process `, }, { ` -type: cloudbeat/cis_eks -streams: - - not_data_yaml: - activated_rules: - cis_k8s: - - a - fetchers: - - name: aws-eks +not_data_yaml: + activated_rules: + cis_k8s: + - a +fetchers: + - name: aws-eks `, }, } From 58661348025a9f4bf971ca9fbdd6a2b4bfbf7ef7 Mon Sep 17 00:00:00 2001 From: Oleg Sucharevich Date: Sun, 30 Oct 2022 20:38:31 +0200 Subject: [PATCH 3/7] chore: update relace script --- scripts/common.sh | 69 ++++++++++------------------- scripts/remote_replace_cloudbeat.sh | 45 ++++++++++++++----- 2 files changed, 58 insertions(+), 56 deletions(-) diff --git a/scripts/common.sh b/scripts/common.sh index 6cde4408c5..b0093a6568 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -1,57 +1,34 @@ #!/bin/bash -# Based on the kubeconfig on the running host, -# the function remotely searches for a file with a given pattern on a given folder -# $1 is the pod on which it should run -# $2 is the folder on which it should search -# $3 is a regex pattern of the file which you want to find -# The function returns the file name -find_in_folder() { - POD=$1 - FOLDER=$2 - PATTERN=$3 - RES=$(kubectl -n kube-system exec $POD -- ls -1 $FOLDER) - if [ -z "$RES" ] - then - return - fi - - RES=$(echo "$RES" | grep $PATTERN) - if [ -z "$RES" ] - then - return - fi - - echo "${RES}" +exec_pod() { + pod=$1 + cmd=$2 + kubectl -n kube-system exec $pod -- $cmd } -# Based on the kubeconfig on the running host, -# the function remotely searches for a file with a given pattern on cloudbeat installation folder -# /usr/share/elastic-agent/data/elastic-agent-*/install/cloudbeat-*/ -# $1 is the pod on which it should run -# $2 is a regex pattern of the file which you want to find -# The function returns the file full path -find_in_cloudbeat_folder() { - POD=$1 +cp_to_pod() { + pod=$1 + source=$2 + dest=$3 + kubectl cp $2 kube-system/$1:$dest +} - PREFIX="/usr/share/elastic-agent/data" - PATH_PARTS=("elastic-agent-" "install" "cloudbeat-" "$2") - for NEXT in ${PATH_PARTS[@]}; do - FOUND=$(find_in_folder $POD $PREFIX $NEXT) - if [ -z "$FOUND" ] - then - return - fi - PREFIX="${PREFIX}/${FOUND}" - done +get_agents() { + kubectl -n kube-system get pod -l app=elastic-agent -o name +} - echo $PREFIX +find_target_os() { + _kubectl_node_info operatingSystem } -find_cloudbeat_config() { - find_in_cloudbeat_folder $1 "cloudbeat.yml" +find_target_arch() { + _kubectl_node_info architecture } -find_cloudbeat_binary() { - find_in_cloudbeat_folder $1 "cloudbeat$" +is_eks() { + _kubectl_node_info kubeletVersion | grep "eks" } + +_kubectl_node_info() { + kubectl get node -o go-template="{{(index .items 0 ).status.nodeInfo.$1}}" +} \ No newline at end of file diff --git a/scripts/remote_replace_cloudbeat.sh b/scripts/remote_replace_cloudbeat.sh index 4b783b4d7e..2f688632c8 100755 --- a/scripts/remote_replace_cloudbeat.sh +++ b/scripts/remote_replace_cloudbeat.sh @@ -1,22 +1,47 @@ #!/bin/bash +set -euo pipefail + # This script uses the kubectl commands in order to ssh into the cluster defined in the host current-context # The script lets you to remotely replace the cloudbeat binary with a file named cloudbeat, located on the host's running directory source ./scripts/common.sh -LOCAL_BINARY="./cloudbeat" +ARCH=$(find_target_arch) +if [ "$ARCH" = "amd64" ]; then + ARCH="x86_64" +fi +OS=$(find_target_os) +VERSION="8.6.0" # Read from elsewhere +SNAPSHOT="SNAPSHOT" + +LOCAL_DIR=cloudbeat-$VERSION-$SNAPSHOT-$OS-$ARCH +echo "Looking for build distribution: $LOCAL_DIR" +tar -xvf build/distributions/$LOCAL_DIR.tar.gz > /dev/null 2>&1 + +# cloudbeat.spec.yml should be built in the agent +curl -o cloudbeat.spec.yml https://raw.githubusercontent.com/elastic/elastic-agent/feature-arch-v2/specs/cloudbeat.spec.yml > /dev/null 2>&1 -PODS=$(kubectl -n kube-system get pod -l app=elastic-agent -o name) -for P in $PODS; do +for P in $(get_agents); do POD=$(echo $P | cut -d '/' -f 2) - BINARY_FILEPATH="$(find_cloudbeat_binary $POD)" - if [ -z "$BINARY_FILEPATH" ] + out=$(exec_pod $POD "elastic-agent version --yaml --binary-only") + SHA=$(echo $out | cut -d ":" -f4 | awk '{$1=$1};1'| awk '{ print substr($0, 0, 6) }') + echo "Found sha=$SHA in pod=$POD" + + ROOT=/usr/share/elastic-agent/data/elastic-agent-$SHA + DEST=$ROOT/components + cp_to_pod $POD $LOCAL_DIR/cloudbeat $DEST + cp_to_pod $POD $LOCAL_DIR/cloudbeat.yml $DEST/cloudbeat.yml + cp_to_pod $POD cloudbeat.spec.yml $DEST/cloudbeat.spec.yml + + BUNDLE="cis_k8s-default" + if [ ! -z "$(is_eks)" ] then - echo "could not find remote binary file" - exit 1 + BUNDLE="cis_eks-default" fi - - kubectl -n kube-system cp "$LOCAL_BINARY" "$POD":"$BINARY_FILEPATH" - kubectl -n kube-system exec "$POD" -- chown elastic-agent:elastic-agent "$BINARY_FILEPATH" + BUNDLE_DIR=$ROOT/run/cloudbeat/$BUNDLE + exec_pod $POD "mkdir -p $BUNDLE_DIR" + cp_to_pod $POD $LOCAL_DIR/bundle.tar.gz $BUNDLE_DIR/bundle.tar.gz + echo "Copied all the assets, restarting the agent" + exec_pod $POD "elastic-agent restart" done From fc43c2221dbc97921e35edb7966482912578e7a1 Mon Sep 17 00:00:00 2001 From: Oleg Sucharevich Date: Mon, 31 Oct 2022 13:55:05 +0200 Subject: [PATCH 4/7] wip --- deploy/kustomize/base/daemonset.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/deploy/kustomize/base/daemonset.yml b/deploy/kustomize/base/daemonset.yml index 0734c9336e..9fd01de465 100644 --- a/deploy/kustomize/base/daemonset.yml +++ b/deploy/kustomize/base/daemonset.yml @@ -85,9 +85,6 @@ spec: readOnly: true serviceAccountName: cloudbeat terminationGracePeriodSeconds: 30 - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule hostNetwork: true dnsPolicy: ClusterFirstWithHostNet volumes: From 95fdd054bf9909bf747401bfd1964fa680f77d82 Mon Sep 17 00:00:00 2001 From: Oleg Sucharevich Date: Mon, 31 Oct 2022 14:36:12 +0200 Subject: [PATCH 5/7] chore: create config from standalone cloudbeat deployment --- config/config.go | 20 +++++++++++++++++ scripts/common.sh | 5 +++++ scripts/remote_replace_cloudbeat.sh | 34 ++++++++++++++--------------- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/config/config.go b/config/config.go index c5529a89bb..b0695c6bd3 100644 --- a/config/config.go +++ b/config/config.go @@ -77,6 +77,10 @@ var DefaultConfig = Stream{ } func New(cfg *config.C) (Config, error) { + // work with v1 cloudbeat.yml in dev mod + if cfg.HasField("streams") { + return newStandaloneConfig(cfg) + } c := DefaultConfig if err := cfg.Unpack(&c); err != nil { return Config{}, err @@ -99,6 +103,22 @@ func Datastream(namespace string, indexPrefix string) string { return indexPrefix + "-" + namespace } +// stanalone config is used for development flows +// see an example deploy/kustomize/overlays/cloudbeat-vanilla/cloudbeat.yml +func newStandaloneConfig(cfg *config.C) (Config, error) { + c := struct { + Period time.Duration + Streams []Stream + }{4 * time.Hour, []Stream{}} + if err := cfg.Unpack(&c); err != nil { + return Config{}, err + } + return Config{ + Type: InputTypeVanillaK8s, + Stream: c.Streams[0], + }, nil +} + type AwsConfigProvider interface { InitializeAWSConfig(ctx context.Context, cfg aws.ConfigAWS) (awssdk.Config, error) } diff --git a/scripts/common.sh b/scripts/common.sh index b0093a6568..e96c6d620e 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -29,6 +29,11 @@ is_eks() { _kubectl_node_info kubeletVersion | grep "eks" } +get_agent_sha() { + out=$(exec_pod $1 "elastic-agent version --yaml --binary-only") + echo $out | cut -d ":" -f4 | awk '{$1=$1};1'| awk '{ print substr($0, 0, 6) }' +} + _kubectl_node_info() { kubectl get node -o go-template="{{(index .items 0 ).status.nodeInfo.$1}}" } \ No newline at end of file diff --git a/scripts/remote_replace_cloudbeat.sh b/scripts/remote_replace_cloudbeat.sh index 2f688632c8..57f48ad4bd 100755 --- a/scripts/remote_replace_cloudbeat.sh +++ b/scripts/remote_replace_cloudbeat.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -euo pipefail +set -eo pipefail # This script uses the kubectl commands in order to ssh into the cluster defined in the host current-context # The script lets you to remotely replace the cloudbeat binary with a file named cloudbeat, located on the host's running directory @@ -12,36 +12,34 @@ if [ "$ARCH" = "amd64" ]; then ARCH="x86_64" fi OS=$(find_target_os) -VERSION="8.6.0" # Read from elsewhere -SNAPSHOT="SNAPSHOT" +VERSION=$(make get-version) -LOCAL_DIR=cloudbeat-$VERSION-$SNAPSHOT-$OS-$ARCH +LOCAL_DIR=cloudbeat-$VERSION-SNAPSHOT-$OS-$ARCH echo "Looking for build distribution: $LOCAL_DIR" tar -xvf build/distributions/$LOCAL_DIR.tar.gz > /dev/null 2>&1 -# cloudbeat.spec.yml should be built in the agent -curl -o cloudbeat.spec.yml https://raw.githubusercontent.com/elastic/elastic-agent/feature-arch-v2/specs/cloudbeat.spec.yml > /dev/null 2>&1 - for P in $(get_agents); do POD=$(echo $P | cut -d '/' -f 2) - out=$(exec_pod $POD "elastic-agent version --yaml --binary-only") - SHA=$(echo $out | cut -d ":" -f4 | awk '{$1=$1};1'| awk '{ print substr($0, 0, 6) }') + SHA=$(get_agent_sha $POD) echo "Found sha=$SHA in pod=$POD" ROOT=/usr/share/elastic-agent/data/elastic-agent-$SHA DEST=$ROOT/components cp_to_pod $POD $LOCAL_DIR/cloudbeat $DEST - cp_to_pod $POD $LOCAL_DIR/cloudbeat.yml $DEST/cloudbeat.yml - cp_to_pod $POD cloudbeat.spec.yml $DEST/cloudbeat.spec.yml - BUNDLE="cis_k8s-default" - if [ ! -z "$(is_eks)" ] + # Start with COPY_BUNDLE=true to move also the opa bundle to the agent + # the bundle can be found later in in /usr/share/elastic-agent/data/elastic-agent-{SHA}/run/cloudbeat/{BUNDLE_NAME} + if [[ ! -z "$COPY_BUNDLE" ]] then - BUNDLE="cis_eks-default" + BUNDLE="cis_k8s-default" + if [ ! -z "$(is_eks)" ] + then + BUNDLE="cis_eks-default" + fi + BUNDLE_DIR=$ROOT/run/cloudbeat/$BUNDLE + exec_pod $POD "mkdir -p $BUNDLE_DIR" + cp_to_pod $POD $LOCAL_DIR/bundle.tar.gz $BUNDLE_DIR/bundle.tar.gz fi - BUNDLE_DIR=$ROOT/run/cloudbeat/$BUNDLE - exec_pod $POD "mkdir -p $BUNDLE_DIR" - cp_to_pod $POD $LOCAL_DIR/bundle.tar.gz $BUNDLE_DIR/bundle.tar.gz - echo "Copied all the assets, restarting the agent" + echo "Copied all the assets, restarting the agent $POD" exec_pod $POD "elastic-agent restart" done From f2e2e388b6aa9e3f54d9ce4c8c897eb19d5a4448 Mon Sep 17 00:00:00 2001 From: Oleg Sucharevich Date: Wed, 9 Nov 2022 15:22:18 +0200 Subject: [PATCH 6/7] replace restart with kill --- scripts/remote_replace_cloudbeat.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/remote_replace_cloudbeat.sh b/scripts/remote_replace_cloudbeat.sh index 57f48ad4bd..4be8f34777 100755 --- a/scripts/remote_replace_cloudbeat.sh +++ b/scripts/remote_replace_cloudbeat.sh @@ -41,5 +41,7 @@ for P in $(get_agents); do cp_to_pod $POD $LOCAL_DIR/bundle.tar.gz $BUNDLE_DIR/bundle.tar.gz fi echo "Copied all the assets, restarting the agent $POD" - exec_pod $POD "elastic-agent restart" + PID=$(exec_pod $POD "pidof cloudbeat") + exec_pod $POD "kill -9 $PID" + # exec_pod $POD "elastic-agent restart" done From 401dc5a4d91e331726f94227a3969bd6cd750c64 Mon Sep 17 00:00:00 2001 From: Oleg Sucharevich Date: Wed, 9 Nov 2022 17:23:21 +0200 Subject: [PATCH 7/7] tidy --- go.mod | 2 +- go.sum | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 88598b5dad..5d25787b8e 100644 --- a/go.mod +++ b/go.mod @@ -165,7 +165,7 @@ require ( github.com/dop251/goja_nodejs v0.0.0-20220712185256-8cad205bf387 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/eapache/go-resiliency v1.2.0 // indirect - github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 // indirect + github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 github.com/elastic/elastic-agent-system-metrics v0.4.4 // indirect github.com/elastic/go-concert v0.2.0 // indirect github.com/elastic/go-lumber v0.1.0 // indirect diff --git a/go.sum b/go.sum index 1630d84fb2..66f5e34213 100644 --- a/go.sum +++ b/go.sum @@ -531,8 +531,6 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20220726163833-e8a0da132f1f h1:38wPsXXWiK2HH9lI0swZ+H0MZhEbzrpkwVJCxLhU59c= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20220726163833-e8a0da132f1f/go.mod h1:TbnF0mFQuaoeDCZyiCzCd/xLmTtY+NpgRCDv3rouHJM= github.com/elastic/csp-security-policies v1.2.2 h1:lWoYY9KaxMuupVNtGd63y9EvUYYKg6ab2wmdIBV6q+s= github.com/elastic/csp-security-policies v1.2.2/go.mod h1:3fA3X9OiyP7IRNyacOGnWqt1eHSMVZRx7p3bmZm98oE= github.com/elastic/e2e-testing v1.99.2-0.20220117192005-d3365c99b9c4 h1:uYT+Krd8dsvnhnLK9pe/JHZkYtXEGPfbV4Wt1JPPol0= @@ -1548,9 +1546,8 @@ github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.elastic.co/apm v1.10.0/go.mod h1:qoOSi09pnzJDh5fKnfY7bPmQgl8yl2tULdOu03xhui0= +go.elastic.co/apm v1.13.0 h1:SUCDscFfpshuxoxuoSh+r+2E3v1uvd5HrlRK5ZHpyvs= go.elastic.co/apm v1.13.0/go.mod h1:dylGv2HKR0tiCV+wliJz1KHtDyuD8SPe69oV7VyK6WY= -go.elastic.co/apm v1.15.0 h1:uPk2g/whK7c7XiZyz/YCUnAUBNPiyNeE3ARX3G6Gx7Q= -go.elastic.co/apm v1.15.0/go.mod h1:dylGv2HKR0tiCV+wliJz1KHtDyuD8SPe69oV7VyK6WY= go.elastic.co/apm/module/apmelasticsearch v1.10.0/go.mod h1:lwoaGDfZzfb9e6TXd3h8/KNmLAONOas7o5NLVNmv8Xk= go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0 h1:5UaI4agfuGoGRhpFVb6s63Hj/9xtqYxF7kd6T77tNAw= go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0/go.mod h1:GmYz+KDp2LDAa2nd/qJ+OkrjWVEoCRIPAWarbUvyt6Y=