From 675c6cf1767c2f6245a94d172c540e668cdd5a5c Mon Sep 17 00:00:00 2001 From: Oleg Sucharevich Date: Wed, 9 Nov 2022 17:49:01 +0200 Subject: [PATCH] Agent v2 (#458) * Set OPA default log level (#505) * feat: align to elastic-agent v2 api * chore: update relace script * wip * chore: create config from standalone cloudbeat deployment * replace restart with kill * tidy Co-authored-by: Amir Ben Nun <34831306+amirbenun@users.noreply.github.com> --- beater/cloudbeat.go | 2 +- beater/validator.go | 7 +- beater/validator_test.go | 49 ++---- cmd/root.go | 25 +++ config/config.go | 42 +++-- config/config_test.go | 162 ++++++++---------- deploy/kustomize/base/daemonset.yml | 3 - go.mod | 8 +- go.sum | 14 +- launcher/launcher_test.go | 2 +- resources/fetchersManager/factory_aws_test.go | 10 +- resources/fetchersManager/factory_test.go | 43 +++-- scripts/common.sh | 78 ++++----- scripts/remote_replace_cloudbeat.sh | 45 +++-- 14 files changed, 245 insertions(+), 245 deletions(-) diff --git a/beater/cloudbeat.go b/beater/cloudbeat.go index df6ed61484..eb80c50bdf 100644 --- a/beater/cloudbeat.go +++ b/beater/cloudbeat.go @@ -71,7 +71,7 @@ func New(b *beat.Beat, cfg *agentconfig.C) (beat.Beater, error) { return nil, err } - reload.Register.MustRegisterList("inputs", reloader) + reload.RegisterV2.MustRegisterInput(reloader) return s, nil } diff --git a/beater/validator.go b/beater/validator.go index 70c070c717..d2bcf0ed7a 100644 --- a/beater/validator.go +++ b/beater/validator.go @@ -27,14 +27,9 @@ import ( agentconfig "github.com/elastic/elastic-agent-libs/config" ) -type validator struct { -} +type validator struct{} func (v *validator) Validate(cfg *agentconfig.C) error { - if !cfg.HasField("streams") { - return fmt.Errorf("no streams in config") - } - c, err := config.New(cfg) if err != nil { return fmt.Errorf("could not parse reconfiguration %v, skipping with error: %v", cfg.FlattenedKeys(), err) diff --git a/beater/validator_test.go b/beater/validator_test.go index 19f4f7c6c8..4df458801e 100644 --- a/beater/validator_test.go +++ b/beater/validator_test.go @@ -45,39 +45,15 @@ func TestValidatorTestSuite(t *testing.T) { } func (s *ValidatorTestSuite) TestConfig() { - configNoStreams := config.MustNewConfigFrom(` -not_streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d - - e -`) - - configNoRuntimeCfg := config.MustNewConfigFrom(` -streams: - - not_runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d - - e -`) configWithRuntimeCfg := config.MustNewConfigFrom(` -streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d - - e +runtime_cfg: + activated_rules: + cis_k8s: + - a + - b + - c + - d + - e `) testcases := []struct { @@ -87,13 +63,8 @@ streams: { true, config.NewConfig(), - }, { - true, - configNoRuntimeCfg, - }, { - true, - configNoStreams, - }, { + }, + { false, configWithRuntimeCfg, }, diff --git a/cmd/root.go b/cmd/root.go index 3d8ccb594e..fc892900a3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -18,12 +18,18 @@ package cmd import ( + "fmt" + "github.com/elastic/cloudbeat/beater" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" cmd "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" + "github.com/elastic/beats/v7/libbeat/common/reload" _ "github.com/elastic/beats/v7/x-pack/libbeat/include" + "github.com/elastic/beats/v7/x-pack/libbeat/management" ) // Name of this beat @@ -31,3 +37,22 @@ var Name = "cloudbeat" // RootCmd to handle beats cli var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{Name: Name, Version: defaultBeatVersion}) + +func cloudbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + modules, err := management.CreateInputsFromStreams(rawIn, "logs", agentInfo) + if err != nil { + return nil, fmt.Errorf("error creating input list from raw expected config: %w", err) + } + + // format for the reloadable list needed bythe cm.Reload() method + configList, err := management.CreateReloadConfigFromInputs(modules) + if err != nil { + return nil, fmt.Errorf("error creating reloader config: %w", err) + } + + return configList, nil +} + +func init() { + management.ConfigTransform.SetTransform(cloudbeatCfg) +} diff --git a/config/config.go b/config/config.go index 9a28e9a13d..b0695c6bd3 100644 --- a/config/config.go +++ b/config/config.go @@ -44,11 +44,6 @@ type Fetcher struct { Name string `config:"name"` // Name of the fetcher } -type AgentInput struct { - Streams []Stream `config:"streams"` - Type string `config:"type"` -} - type Stream struct { AWSConfig aws.ConfigAWS `config:",inline"` RuntimeCfg *RuntimeConfig `config:"runtime_cfg"` @@ -77,23 +72,26 @@ type Benchmarks struct { CisEks []string `config:"cis_eks,omitempty" yaml:"cis_eks,omitempty" json:"cis_eks,omitempty"` } -var DefaultConfig = AgentInput{ - Type: InputTypeVanillaK8s, - Streams: []Stream{{ - Period: 4 * time.Hour, - }}, +var DefaultConfig = Stream{ + Period: 4 * time.Hour, } func New(cfg *config.C) (Config, error) { + // work with v1 cloudbeat.yml in dev mod + if cfg.HasField("streams") { + return newStandaloneConfig(cfg) + } c := DefaultConfig - if err := cfg.Unpack(&c); err != nil { return Config{}, err } - + inputType := InputTypeVanillaK8s + if c.RuntimeCfg != nil && c.RuntimeCfg.ActivatedRules != nil && len(c.RuntimeCfg.ActivatedRules.CisEks) > 0 { + inputType = InputTypeEks + } return Config{ - Stream: c.Streams[0], - Type: c.Type, + Stream: c, + Type: inputType, }, nil } @@ -105,6 +103,22 @@ func Datastream(namespace string, indexPrefix string) string { return indexPrefix + "-" + namespace } +// stanalone config is used for development flows +// see an example deploy/kustomize/overlays/cloudbeat-vanilla/cloudbeat.yml +func newStandaloneConfig(cfg *config.C) (Config, error) { + c := struct { + Period time.Duration + Streams []Stream + }{4 * time.Hour, []Stream{}} + if err := cfg.Unpack(&c); err != nil { + return Config{}, err + } + return Config{ + Type: InputTypeVanillaK8s, + Stream: c.Streams[0], + }, nil +} + type AwsConfigProvider interface { InitializeAWSConfig(ctx context.Context, cfg aws.ConfigAWS) (awssdk.Config, error) } diff --git a/config/config_test.go b/config/config_test.go index 2b96f30642..dc4b22b217 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -56,51 +56,48 @@ func (s *ConfigTestSuite) TestNew() { }{ { ` - type : cloudbeat/cis_k8s - streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d - - e - fetchers: - - name: a - directory: b - - name: b - directory: b +runtime_cfg: + activated_rules: + cis_k8s: + - a + - b + - c + - d + - e +fetchers: + - name: a + directory: b + - name: b + directory: b `, &Benchmarks{CisK8s: []string{"a", "b", "c", "d", "e"}}, "cloudbeat/cis_k8s", aws.ConfigAWS{}, 2, - }, { + }, + { ` - type : cloudbeat/cis_eks - streams: - - runtime_cfg: - activated_rules: - cis_eks: - - a - - b - - c - - d - - e - access_key_id: key - secret_access_key: secret - session_token: session - shared_credential_file: shared_credential_file - credential_profile_name: credential_profile_name - role_arn: role_arn - fetchers: - - name: a - directory: b - - name: b - directory: b - - name: c - directory: c +runtime_cfg: + activated_rules: + cis_eks: + - a + - b + - c + - d + - e +access_key_id: key +secret_access_key: secret +session_token: session +shared_credential_file: shared_credential_file +credential_profile_name: credential_profile_name +role_arn: role_arn +fetchers: + - name: a + directory: b + - name: b + directory: b + - name: c + directory: c `, &Benchmarks{CisEks: []string{"a", "b", "c", "d", "e"}}, "cloudbeat/cis_eks", @@ -137,23 +134,21 @@ func (s *ConfigTestSuite) TestRuntimeCfgExists() { }{ { ` - streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d - - e +runtime_cfg: + activated_rules: + cis_k8s: + - a + - b + - c + - d + - e `, true, }, { ` - streams: - - not_runtime_cfg: - something: true +not_runtime_cfg: + something: true `, false, }, @@ -177,14 +172,13 @@ func (s *ConfigTestSuite) TestRuntimeConfig() { }{ { ` - streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b - - c - - d +runtime_cfg: + activated_rules: + cis_k8s: + - a + - b + - c + - d `, []string{"a", "b", "c", "d"}, }, } @@ -207,10 +201,10 @@ func (s *ConfigTestSuite) TestRuntimeEvaluatorConfig() { config string expected EvaluatorConfig }{ - {` - streams: - - evaluator: - decision_logs: true + { + ` +evaluator: + decision_logs: true `, EvaluatorConfig{ DecisionLogs: true, @@ -237,19 +231,19 @@ func (s *ConfigTestSuite) TestConfigPeriod() { {"", 4 * time.Hour}, { ` - streams: - - period: 50s -`, 50 * time.Second}, + period: 50s +`, 50 * time.Second, + }, { ` - streams: - - period: 5m -`, 5 * time.Minute}, + period: 5m +`, 5 * time.Minute, + }, { ` - streams: - - period: 2h -`, 2 * time.Hour}, + period: 2h +`, 2 * time.Hour, + }, } for _, test := range tests { @@ -272,13 +266,11 @@ func (s *ConfigTestSuite) TestActivatedRulesFrameWork() { }{ { ` -type: cloudbeat/cis_k8s -streams: - - runtime_cfg: - activated_rules: - cis_k8s: - - a - - b +runtime_cfg: + activated_rules: + cis_k8s: + - a + - b `, []string{"a", "b"}, nil, @@ -286,13 +278,11 @@ streams: }, { ` -type: cloudbeat/cis_eks -streams: - - runtime_cfg: - activated_rules: - cis_eks: - - a - - b +runtime_cfg: + activated_rules: + cis_eks: + - a + - b `, nil, []string{"a", "b"}, diff --git a/deploy/kustomize/base/daemonset.yml b/deploy/kustomize/base/daemonset.yml index 0734c9336e..9fd01de465 100644 --- a/deploy/kustomize/base/daemonset.yml +++ b/deploy/kustomize/base/daemonset.yml @@ -85,9 +85,6 @@ spec: readOnly: true serviceAccountName: cloudbeat terminationGracePeriodSeconds: 30 - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule hostNetwork: true dnsPolicy: ClusterFirstWithHostNet volumes: diff --git a/go.mod b/go.mod index 036eabcfbe..5d25787b8e 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/magefile/mage v1.14.0 github.com/mailru/easyjson v0.7.6 // indirect github.com/mitchellh/gox v1.0.1 - github.com/mitchellh/mapstructure v1.4.3 + github.com/mitchellh/mapstructure v1.5.0 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0 @@ -165,12 +165,12 @@ require ( github.com/dop251/goja_nodejs v0.0.0-20220712185256-8cad205bf387 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/eapache/go-resiliency v1.2.0 // indirect - github.com/elastic/elastic-agent-client/v7 v7.0.0-20220607160924-1a71765a8bbe // indirect + github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 github.com/elastic/elastic-agent-system-metrics v0.4.4 // indirect github.com/elastic/go-concert v0.2.0 // indirect github.com/elastic/go-lumber v0.1.0 // indirect github.com/elastic/go-seccomp-bpf v1.2.0 // indirect - github.com/elastic/go-structform v0.0.9 // indirect + github.com/elastic/go-structform v0.0.10 // indirect github.com/elastic/go-sysinfo v1.8.1 // indirect github.com/elastic/go-ucfg v0.8.6 github.com/elastic/go-windows v1.0.1 // indirect @@ -262,6 +262,8 @@ replace ( github.com/apoydence/eachers => github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 //indirect, see https://github.com/elastic/beats/pull/29780 for details. github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 + //github.com/elastic/beats/v7 => github.com/fearful-symmetry/beats/v7 v1.2.2-0.20220825171527-b0d25807e2ab + github.com/elastic/beats/v7 => github.com/eyalkraft/beats/v7 v7.0.0-testv2 // forked from ^ github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v1.4.8-0.20211018144411-a81f2b630e7c github.com/golang/glog => github.com/elastic/glog v1.0.1-0.20210831205241-7d8b5c89dfc4 github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a diff --git a/go.sum b/go.sum index 51e6608e7d..66f5e34213 100644 --- a/go.sum +++ b/go.sum @@ -531,16 +531,14 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20220726163833-e8a0da132f1f h1:38wPsXXWiK2HH9lI0swZ+H0MZhEbzrpkwVJCxLhU59c= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20220726163833-e8a0da132f1f/go.mod h1:TbnF0mFQuaoeDCZyiCzCd/xLmTtY+NpgRCDv3rouHJM= github.com/elastic/csp-security-policies v1.2.2 h1:lWoYY9KaxMuupVNtGd63y9EvUYYKg6ab2wmdIBV6q+s= github.com/elastic/csp-security-policies v1.2.2/go.mod h1:3fA3X9OiyP7IRNyacOGnWqt1eHSMVZRx7p3bmZm98oE= github.com/elastic/e2e-testing v1.99.2-0.20220117192005-d3365c99b9c4 h1:uYT+Krd8dsvnhnLK9pe/JHZkYtXEGPfbV4Wt1JPPol0= github.com/elastic/e2e-testing v1.99.2-0.20220117192005-d3365c99b9c4/go.mod h1:UcNuf4pX/qDVNQr0zybm1NL2YoWik+jKBaINZqQCA40= github.com/elastic/elastic-agent-autodiscover v0.2.1 h1:Nbeayh3vq2FNm6xaFo34mhUdOu0EVlpj53CqCsbU0E4= github.com/elastic/elastic-agent-autodiscover v0.2.1/go.mod h1:gPnzzfdYNdgznAb+iG9eyyXaQXBbAMHa+Y6Z8hXfcGY= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20220607160924-1a71765a8bbe h1:knHO5fYAT3+sLf64NxcOxdjGysPxsSMkQGB27vMS8TE= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20220607160924-1a71765a8bbe/go.mod h1:fkvyUfFwyAG5OnMF0h+FV9sC0Xn9YLITwQpSuwungQs= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 h1:uJIMfLgCenJvxsVmEjBjYGxt0JddCgw2IxgoNfcIXOk= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484/go.mod h1:fkvyUfFwyAG5OnMF0h+FV9sC0Xn9YLITwQpSuwungQs= github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-libs v0.2.15 h1:hdAbrZZ2mCPcQLRCE3E8xw3mHKl8HFMt36w7jan/XGo= github.com/elastic/elastic-agent-libs v0.2.15/go.mod h1:0J9lzJh+BjttIiVjYDLncKYCEWUUHiiqnuI64y6C6ss= @@ -563,8 +561,9 @@ github.com/elastic/go-lumber v0.1.0 h1:HUjpyg36v2HoKtXlEC53EJ3zDFiDRn65d7B8dBHNi github.com/elastic/go-lumber v0.1.0/go.mod h1:8YvjMIRYypWuPvpxx7WoijBYdbB7XIh/9FqSYQZTtxQ= github.com/elastic/go-seccomp-bpf v1.2.0 h1:K5fToUAMzm0pmdlYORmw0FP0DloRa1SfqRYkum647Yk= github.com/elastic/go-seccomp-bpf v1.2.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE+vxCFzzvQwN7T8= -github.com/elastic/go-structform v0.0.9 h1:HpcS7xljL4kSyUfDJ8cXTJC6rU5ChL1wYb6cx3HLD+o= github.com/elastic/go-structform v0.0.9/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= +github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w= +github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-sysinfo v1.8.1 h1:4Yhj+HdV6WjbCRgGdZpPJ8lZQlXZLKDAeIkmQ/VRvi4= @@ -606,6 +605,8 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/eyalkraft/beats/v7 v7.0.0-testv2 h1:sgLR01TfyP8RSRkUJK5fu7ItQP664odTrZSpk0KZk/A= +github.com/eyalkraft/beats/v7 v7.0.0-testv2/go.mod h1:klCBW4dWSbh8b/YTSLcSWVXSdxObCd+8/b3AqvcD84E= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -1154,8 +1155,9 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= diff --git a/launcher/launcher_test.go b/launcher/launcher_test.go index b415c1272f..b84ae34454 100644 --- a/launcher/launcher_test.go +++ b/launcher/launcher_test.go @@ -174,7 +174,7 @@ func (s *LauncherTestSuite) MockBeatManager(mocks *launcherMocks) { } b, err := instance.NewInitializedBeat(settings) s.NoError(err) - b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.Register, b.Beat.Info.ID) + b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.RegisterV2, b.Beat.Info.ID) s.NoError(err) mocks.beat = &b.Beat } diff --git a/resources/fetchersManager/factory_aws_test.go b/resources/fetchersManager/factory_aws_test.go index 68ee1f7207..2f2c243b6f 100644 --- a/resources/fetchersManager/factory_aws_test.go +++ b/resources/fetchersManager/factory_aws_test.go @@ -19,6 +19,7 @@ package fetchersManager import ( "context" + "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/cloudbeat/config" "github.com/elastic/cloudbeat/resources/fetching" @@ -78,8 +79,6 @@ func (n *awsTestFactory) Create(log *logp.Logger, c *agentconfig.C, ch chan fetc func awsMockedFetcherConfig(s *FactoriesTestSuite, awsConfig aws.ConfigAWS) *agentconfig.C { c := agentconfig.NewConfig() - conf := config.DefaultConfig - conf.Type = config.InputTypeEks err := c.Merge(awsConfig) s.NoError(err) @@ -87,11 +86,12 @@ func awsMockedFetcherConfig(s *FactoriesTestSuite, awsConfig aws.ConfigAWS) *age } func (s *FactoriesTestSuite) TestCreateFetcherWithAwsCredentials() { - var tests = []struct { + tests := []struct { fetcherName string awsConfig aws.ConfigAWS }{ - {"some_fetcher", + { + "some_fetcher", aws.ConfigAWS{ AccessKeyID: "key", SecretAccessKey: "secret", @@ -120,7 +120,7 @@ func (s *FactoriesTestSuite) TestCreateFetcherWithAwsCredentials() { } func (s *FactoriesTestSuite) TestRegisterFetchersWithAwsCredentials() { - var tests = []struct { + tests := []struct { fetcherName string awsConfig aws.ConfigAWS }{ diff --git a/resources/fetchersManager/factory_test.go b/resources/fetchersManager/factory_test.go index b5719fa5fe..3c0da8b184 100644 --- a/resources/fetchersManager/factory_test.go +++ b/resources/fetchersManager/factory_test.go @@ -19,9 +19,10 @@ package fetchersManager import ( "context" + "testing" + "github.com/elastic/cloudbeat/resources/utils/testhelper" "github.com/elastic/elastic-agent-libs/logp" - "testing" "github.com/elastic/cloudbeat/config" "github.com/elastic/cloudbeat/resources/fetching" @@ -98,7 +99,7 @@ func (s *FactoriesTestSuite) TearDownTest() { } func (s *FactoriesTestSuite) TestListFetcher() { - var tests = []struct { + tests := []struct { key string }{ {"process"}, @@ -114,7 +115,7 @@ func (s *FactoriesTestSuite) TestListFetcher() { } func (s *FactoriesTestSuite) TestCreateFetcher() { - var tests = []struct { + tests := []struct { key string value int }{ @@ -138,7 +139,7 @@ func (s *FactoriesTestSuite) TestCreateFetcher() { } func (s *FactoriesTestSuite) TestCreateFetcherCollision() { - var tests = []struct { + tests := []struct { key string }{ {"process"}, @@ -153,7 +154,7 @@ func (s *FactoriesTestSuite) TestCreateFetcherCollision() { } func (s *FactoriesTestSuite) TestRegisterFetchers() { - var tests = []struct { + tests := []struct { key string value int integrationType string @@ -192,7 +193,7 @@ func (s *FactoriesTestSuite) TestRegisterFetchers() { } func (s *FactoriesTestSuite) TestRegisterNotFoundFetchers() { - var tests = []struct { + tests := []struct { key string value int }{ @@ -213,31 +214,27 @@ func (s *FactoriesTestSuite) TestRegisterNotFoundFetchers() { } func (s *FactoriesTestSuite) TestRegisterFromFullConfig() { - var tests = []struct { + tests := []struct { config string }{ { ` -type: cloudbeat/cis_k8s -streams: - - not_data_yaml: - activated_rules: - cis_k8s: - - a - fetchers: - - name: process +not_data_yaml: + activated_rules: + cis_k8s: + - a +fetchers: + - name: process `, }, { ` -type: cloudbeat/cis_eks -streams: - - not_data_yaml: - activated_rules: - cis_k8s: - - a - fetchers: - - name: aws-eks +not_data_yaml: + activated_rules: + cis_k8s: + - a +fetchers: + - name: aws-eks `, }, } diff --git a/scripts/common.sh b/scripts/common.sh index 6cde4408c5..e96c6d620e 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -1,57 +1,39 @@ #!/bin/bash -# Based on the kubeconfig on the running host, -# the function remotely searches for a file with a given pattern on a given folder -# $1 is the pod on which it should run -# $2 is the folder on which it should search -# $3 is a regex pattern of the file which you want to find -# The function returns the file name -find_in_folder() { - POD=$1 - FOLDER=$2 - PATTERN=$3 - RES=$(kubectl -n kube-system exec $POD -- ls -1 $FOLDER) - if [ -z "$RES" ] - then - return - fi - - RES=$(echo "$RES" | grep $PATTERN) - if [ -z "$RES" ] - then - return - fi - - echo "${RES}" +exec_pod() { + pod=$1 + cmd=$2 + kubectl -n kube-system exec $pod -- $cmd } -# Based on the kubeconfig on the running host, -# the function remotely searches for a file with a given pattern on cloudbeat installation folder -# /usr/share/elastic-agent/data/elastic-agent-*/install/cloudbeat-*/ -# $1 is the pod on which it should run -# $2 is a regex pattern of the file which you want to find -# The function returns the file full path -find_in_cloudbeat_folder() { - POD=$1 - - PREFIX="/usr/share/elastic-agent/data" - PATH_PARTS=("elastic-agent-" "install" "cloudbeat-" "$2") - for NEXT in ${PATH_PARTS[@]}; do - FOUND=$(find_in_folder $POD $PREFIX $NEXT) - if [ -z "$FOUND" ] - then - return - fi - PREFIX="${PREFIX}/${FOUND}" - done - - echo $PREFIX +cp_to_pod() { + pod=$1 + source=$2 + dest=$3 + kubectl cp $2 kube-system/$1:$dest } -find_cloudbeat_config() { - find_in_cloudbeat_folder $1 "cloudbeat.yml" +get_agents() { + kubectl -n kube-system get pod -l app=elastic-agent -o name } -find_cloudbeat_binary() { - find_in_cloudbeat_folder $1 "cloudbeat$" +find_target_os() { + _kubectl_node_info operatingSystem } + +find_target_arch() { + _kubectl_node_info architecture +} + +is_eks() { + _kubectl_node_info kubeletVersion | grep "eks" +} + +get_agent_sha() { + out=$(exec_pod $1 "elastic-agent version --yaml --binary-only") + echo $out | cut -d ":" -f4 | awk '{$1=$1};1'| awk '{ print substr($0, 0, 6) }' +} + +_kubectl_node_info() { + kubectl get node -o go-template="{{(index .items 0 ).status.nodeInfo.$1}}" +} \ No newline at end of file diff --git a/scripts/remote_replace_cloudbeat.sh b/scripts/remote_replace_cloudbeat.sh index 4b783b4d7e..4be8f34777 100755 --- a/scripts/remote_replace_cloudbeat.sh +++ b/scripts/remote_replace_cloudbeat.sh @@ -1,22 +1,47 @@ #!/bin/bash +set -eo pipefail + # This script uses the kubectl commands in order to ssh into the cluster defined in the host current-context # The script lets you to remotely replace the cloudbeat binary with a file named cloudbeat, located on the host's running directory source ./scripts/common.sh -LOCAL_BINARY="./cloudbeat" +ARCH=$(find_target_arch) +if [ "$ARCH" = "amd64" ]; then + ARCH="x86_64" +fi +OS=$(find_target_os) +VERSION=$(make get-version) + +LOCAL_DIR=cloudbeat-$VERSION-SNAPSHOT-$OS-$ARCH +echo "Looking for build distribution: $LOCAL_DIR" +tar -xvf build/distributions/$LOCAL_DIR.tar.gz > /dev/null 2>&1 -PODS=$(kubectl -n kube-system get pod -l app=elastic-agent -o name) -for P in $PODS; do +for P in $(get_agents); do POD=$(echo $P | cut -d '/' -f 2) - BINARY_FILEPATH="$(find_cloudbeat_binary $POD)" - if [ -z "$BINARY_FILEPATH" ] + SHA=$(get_agent_sha $POD) + echo "Found sha=$SHA in pod=$POD" + + ROOT=/usr/share/elastic-agent/data/elastic-agent-$SHA + DEST=$ROOT/components + cp_to_pod $POD $LOCAL_DIR/cloudbeat $DEST + + # Start with COPY_BUNDLE=true to move also the opa bundle to the agent + # the bundle can be found later in in /usr/share/elastic-agent/data/elastic-agent-{SHA}/run/cloudbeat/{BUNDLE_NAME} + if [[ ! -z "$COPY_BUNDLE" ]] then - echo "could not find remote binary file" - exit 1 + BUNDLE="cis_k8s-default" + if [ ! -z "$(is_eks)" ] + then + BUNDLE="cis_eks-default" + fi + BUNDLE_DIR=$ROOT/run/cloudbeat/$BUNDLE + exec_pod $POD "mkdir -p $BUNDLE_DIR" + cp_to_pod $POD $LOCAL_DIR/bundle.tar.gz $BUNDLE_DIR/bundle.tar.gz fi - - kubectl -n kube-system cp "$LOCAL_BINARY" "$POD":"$BINARY_FILEPATH" - kubectl -n kube-system exec "$POD" -- chown elastic-agent:elastic-agent "$BINARY_FILEPATH" + echo "Copied all the assets, restarting the agent $POD" + PID=$(exec_pod $POD "pidof cloudbeat") + exec_pod $POD "kill -9 $PID" + # exec_pod $POD "elastic-agent restart" done