From 97b11b91a4b9ee4bd5d0135d0d3c0f259abaf1c0 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Tue, 11 May 2021 11:34:55 -0700 Subject: [PATCH 1/8] Remove beats central management feature. Remove the Kibana beats central management feature as it has been replaced by Fleet. --- x-pack/auditbeat/cmd/root.go | 3 +- x-pack/filebeat/cmd/root.go | 3 +- x-pack/heartbeat/cmd/root.go | 4 +- x-pack/journalbeat/cmd/root.go | 4 +- x-pack/libbeat/cmd/enroll.go | 131 ----- .../{cmd/inject.go => include/include.go} | 15 +- x-pack/libbeat/libbeat.go | 3 +- x-pack/libbeat/management/api/auth_client.go | 110 ---- .../management/api/auth_client_test.go | 198 -------- x-pack/libbeat/management/api/client.go | 105 ---- x-pack/libbeat/management/api/client_test.go | 37 -- .../libbeat/management/api/configuration.go | 76 --- .../management/api/configuration_test.go | 71 --- x-pack/libbeat/management/api/convert.go | 79 --- x-pack/libbeat/management/api/convert_test.go | 111 ---- x-pack/libbeat/management/api/enroll.go | 57 --- x-pack/libbeat/management/api/enroll_test.go | 108 ---- .../management/api/enrollment_token.go | 32 -- .../management/api/enrollment_token_test.go | 29 -- .../libbeat/management/api/event_reporter.go | 125 ----- .../management/api/event_reporter_test.go | 74 --- x-pack/libbeat/management/cache.go | 66 --- x-pack/libbeat/management/cache_test.go | 38 -- x-pack/libbeat/management/config.go | 137 +---- x-pack/libbeat/management/config_test.go | 18 - x-pack/libbeat/management/enroll.go | 109 ---- x-pack/libbeat/management/error.go | 10 - x-pack/libbeat/management/error_test.go | 25 +- x-pack/libbeat/management/fleet/config.go | 2 +- x-pack/libbeat/management/manager.go | 345 ------------- x-pack/libbeat/management/manager_test.go | 472 ------------------ x-pack/libbeat/management/plugin.go | 31 -- x-pack/libbeat/management/state.go | 82 --- x-pack/libbeat/management/state_test.go | 58 --- x-pack/metricbeat/cmd/root.go | 3 +- x-pack/osquerybeat/cmd/root.go | 4 +- x-pack/packetbeat/cmd/root.go | 4 +- x-pack/winlogbeat/cmd/root.go | 3 +- 38 files changed, 40 insertions(+), 2742 deletions(-) delete mode 100644 x-pack/libbeat/cmd/enroll.go rename x-pack/libbeat/{cmd/inject.go => include/include.go} (70%) delete mode 100644 x-pack/libbeat/management/api/auth_client.go delete mode 100644 x-pack/libbeat/management/api/auth_client_test.go delete mode 100644 x-pack/libbeat/management/api/client.go delete mode 100644 x-pack/libbeat/management/api/client_test.go delete mode 100644 x-pack/libbeat/management/api/convert.go delete mode 100644 x-pack/libbeat/management/api/convert_test.go delete mode 100644 x-pack/libbeat/management/api/enroll.go delete mode 100644 x-pack/libbeat/management/api/enroll_test.go delete mode 100644 x-pack/libbeat/management/api/enrollment_token.go delete mode 100644 x-pack/libbeat/management/api/enrollment_token_test.go delete mode 100644 x-pack/libbeat/management/api/event_reporter.go delete mode 100644 x-pack/libbeat/management/api/event_reporter_test.go delete mode 100644 x-pack/libbeat/management/cache.go delete mode 100644 x-pack/libbeat/management/cache_test.go delete mode 100644 x-pack/libbeat/management/config_test.go delete mode 100644 x-pack/libbeat/management/enroll.go delete mode 100644 x-pack/libbeat/management/manager.go delete mode 100644 x-pack/libbeat/management/manager_test.go delete mode 100644 x-pack/libbeat/management/plugin.go delete mode 100644 x-pack/libbeat/management/state.go delete mode 100644 x-pack/libbeat/management/state_test.go diff --git a/x-pack/auditbeat/cmd/root.go b/x-pack/auditbeat/cmd/root.go index 7e805a92e7d..213a0c3c68a 100644 --- a/x-pack/auditbeat/cmd/root.go +++ b/x-pack/auditbeat/cmd/root.go @@ -7,10 +7,10 @@ package cmd import ( auditbeatcmd "github.com/elastic/beats/v7/auditbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" // Register Auditbeat x-pack modules. _ "github.com/elastic/beats/v7/x-pack/auditbeat/include" + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // Name of the beat @@ -23,5 +23,4 @@ func init() { settings := auditbeatcmd.AuditbeatSettings() settings.ElasticLicensed = true RootCmd = auditbeatcmd.Initialize(settings) - xpackcmd.AddXPack(RootCmd, auditbeatcmd.Name) } diff --git a/x-pack/filebeat/cmd/root.go b/x-pack/filebeat/cmd/root.go index b95a3cb9e96..18bdc321b30 100644 --- a/x-pack/filebeat/cmd/root.go +++ b/x-pack/filebeat/cmd/root.go @@ -7,11 +7,11 @@ package cmd import ( fbcmd "github.com/elastic/beats/v7/filebeat/cmd" cmd "github.com/elastic/beats/v7/libbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" // Register the includes. _ "github.com/elastic/beats/v7/x-pack/filebeat/include" inputs "github.com/elastic/beats/v7/x-pack/filebeat/input/default-inputs" + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) const Name = fbcmd.Name @@ -21,6 +21,5 @@ func Filebeat() *cmd.BeatsRootCmd { settings := fbcmd.FilebeatSettings() settings.ElasticLicensed = true command := fbcmd.Filebeat(inputs.Init, settings) - xpackcmd.AddXPack(command, Name) return command } diff --git a/x-pack/heartbeat/cmd/root.go b/x-pack/heartbeat/cmd/root.go index 8e25993bd87..d777f631e48 100644 --- a/x-pack/heartbeat/cmd/root.go +++ b/x-pack/heartbeat/cmd/root.go @@ -7,7 +7,8 @@ package cmd import ( heartbeatCmd "github.com/elastic/beats/v7/heartbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" + + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // RootCmd to handle beats cli @@ -17,5 +18,4 @@ func init() { settings := heartbeatCmd.HeartbeatSettings() settings.ElasticLicensed = true RootCmd = heartbeatCmd.Initialize(settings) - xpackcmd.AddXPack(RootCmd, heartbeatCmd.Name) } diff --git a/x-pack/journalbeat/cmd/root.go b/x-pack/journalbeat/cmd/root.go index 7d27c7e8fef..d549011f73a 100644 --- a/x-pack/journalbeat/cmd/root.go +++ b/x-pack/journalbeat/cmd/root.go @@ -7,7 +7,8 @@ package cmd import ( journalbeatCmd "github.com/elastic/beats/v7/journalbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" + + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // RootCmd to handle beats cli @@ -17,5 +18,4 @@ func init() { settings := journalbeatCmd.JournalbeatSettings() settings.ElasticLicensed = true RootCmd = journalbeatCmd.Initialize(settings) - xpackcmd.AddXPack(RootCmd, journalbeatCmd.Name) } diff --git a/x-pack/libbeat/cmd/enroll.go b/x-pack/libbeat/cmd/enroll.go deleted file mode 100644 index 3f44647bd5b..00000000000 --- a/x-pack/libbeat/cmd/enroll.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package cmd - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - - "github.com/elastic/beats/v7/libbeat/cmd/instance" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/cli" - "github.com/elastic/beats/v7/x-pack/libbeat/management" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" -) - -func getBeat(name, version string) (*instance.Beat, error) { - b, err := instance.NewInitializedBeat(instance.Settings{Name: name, Version: version}) - if err != nil { - return nil, fmt.Errorf("error creating beat: %s", err) - } - return b, nil -} - -func genEnrollCmd(name, version string) *cobra.Command { - var username, password string - var force bool - - enrollCmd := cobra.Command{ - Use: "enroll []", - Short: "Enroll in Kibana for Central Management", - Long: `This will enroll in Kibana Beats Central Management. If you pass an enrollment token - it will be used. You can also enroll using a username and password combination.`, - Args: cobra.RangeArgs(1, 2), - Run: cli.RunWith( - func(cmd *cobra.Command, args []string) error { - beat, err := getBeat(name, version) - if err != nil { - return err - } - - kibanaURL := args[0] - - if username == "" && len(args) == 1 { - return errors.New("You should pass either an enrollment token or use --username flag") - } - - // Retrieve any available configuration avaible for Kibana, either - // from the configuration file or using `-E`. - kibanaRaw, err := kibanaConfig(beat.Config.Management) - if err != nil { - return err - } - - // retrieve an enrollment token using username/password - config, err := api.ConfigFromURL(kibanaURL, kibanaRaw) - if err != nil { - return err - } - - confirm, err := confirmConfigOverwrite(force) - if err != nil { - return err - } - - if !confirm { - fmt.Println("Enrollment was canceled by the user") - return nil - } - - var enrollmentToken string - if len(args) == 2 { - // use given enrollment token - enrollmentToken = args[1] - } else { - // pass username/password - config.IgnoreVersion = true - config.Username = username - config.Password, err = cli.ReadPassword(password) - if err != nil { - return err - } - - client, err := api.NewClient(config) - if err != nil { - return err - } - enrollmentToken, err = client.CreateEnrollmentToken() - if err != nil { - return errors.Wrap(err, "Error creating a new enrollment token") - } - } - - err = management.Enroll(beat, config, enrollmentToken) - if err != nil { - return errors.Wrap(err, "Error while enrolling") - } - - fmt.Println("Enrolled and ready to retrieve settings from Kibana") - return nil - }), - } - - enrollCmd.Flags().StringVar(&username, "username", "elastic", "Username to use when enrolling without token") - enrollCmd.Flags().StringVar(&password, "password", "stdin", "Method to read the password to use when enrolling without token (stdin or env:VAR_NAME)") - enrollCmd.Flags().BoolVar(&force, "force", false, "Force overwrite of current configuraiton, do not prompt for confirmation") - - return &enrollCmd -} - -func kibanaConfig(config *common.Config) (*common.Config, error) { - if config != nil && config.HasField("kibana") { - sub, err := config.Child("kibana", -1) - if err != nil { - return nil, err - } - return sub, nil - } - return common.NewConfig(), nil -} - -func confirmConfigOverwrite(force bool) (bool, error) { - if force { - return true, nil - } - - return cli.Confirm("This will replace your current settings. Do you want to continue?", true) -} diff --git a/x-pack/libbeat/cmd/inject.go b/x-pack/libbeat/include/include.go similarity index 70% rename from x-pack/libbeat/cmd/inject.go rename to x-pack/libbeat/include/include.go index 6879e0358e4..c82fc7b07af 100644 --- a/x-pack/libbeat/cmd/inject.go +++ b/x-pack/libbeat/include/include.go @@ -2,16 +2,12 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package cmd +package include import ( - "github.com/elastic/beats/v7/libbeat/cmd" - - // register central management - _ "github.com/elastic/beats/v7/x-pack/libbeat/management" - - // Register fleet + // Register Fleet _ "github.com/elastic/beats/v7/x-pack/libbeat/management/fleet" + // register processors _ "github.com/elastic/beats/v7/x-pack/libbeat/processors/add_cloudfoundry_metadata" _ "github.com/elastic/beats/v7/x-pack/libbeat/processors/add_nomad_metadata" @@ -21,8 +17,3 @@ import ( _ "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws/elb" _ "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/nomad" ) - -// AddXPack extends the given root folder with XPack features -func AddXPack(root *cmd.BeatsRootCmd, name string) { - root.AddCommand(genEnrollCmd(name, "")) -} diff --git a/x-pack/libbeat/libbeat.go b/x-pack/libbeat/libbeat.go index eeaf82cccbd..501787b3613 100644 --- a/x-pack/libbeat/libbeat.go +++ b/x-pack/libbeat/libbeat.go @@ -9,14 +9,13 @@ import ( "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/mock" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // RootCmd to test libbeat var RootCmd = cmd.GenRootCmdWithSettings(mock.New, mock.Settings) func main() { - xpackcmd.AddXPack(RootCmd, mock.Name) if err := RootCmd.Execute(); err != nil { os.Exit(1) } diff --git a/x-pack/libbeat/management/api/auth_client.go b/x-pack/libbeat/management/api/auth_client.go deleted file mode 100644 index cb9f5113944..00000000000 --- a/x-pack/libbeat/management/api/auth_client.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "encoding/json" - "fmt" - "net/http" - "sort" - "time" - - "github.com/gofrs/uuid" - "github.com/joeshaw/multierror" -) - -// EventType is the type of event that the events endpoint can understand. -type EventType string - -// Event is the interface for the events to be send to the event endpoint. -type Event interface { - json.Marshaler - EventType() EventType -} - -// EventRequest is the data send to the CM event endpoint. -type EventRequest struct { - Timestamp time.Time `json:"timestamp"` - EventType EventType `json:"type"` - Event Event `json:"event"` -} - -// EventAPIResponse is the top level response for the events endpoints. -type EventAPIResponse struct { - BaseResponse - Response []EventResponse `json:"results"` -} - -// EventResponse is the indiviual response for each event request. -type EventResponse struct { - BaseResponse -} - -// AuthClienter is the interface exposed by the auth client and is useful for testing without calling -// a remote endpoint. -type AuthClienter interface { - // SendEvents takes a slices of event request and send them to the endpoint. - SendEvents([]EventRequest) error - - // Configuration retrieves the list of configuration blocks from Kibana - Configuration() (ConfigBlocks, error) -} - -// AuthClient is a authenticated client to the CM endpoint and exposes the calls that require -// the clients to pass credentials (UUID and AccessToken). -type AuthClient struct { - Client *Client - BeatUUID uuid.UUID - AccessToken string -} - -func (c AuthClient) headers() http.Header { - headers := http.Header{} - headers.Set("kbn-beats-access-token", c.AccessToken) - return headers -} - -// SendEvents send a list of events to Kibana. -func (c *AuthClient) SendEvents(requests []EventRequest) error { - sort.SliceStable(requests, func(i, j int) bool { - return requests[i].Timestamp.Before(requests[j].Timestamp) - }) - - resp := EventAPIResponse{} - url := fmt.Sprintf("/api/beats/%s/events", c.BeatUUID) - statusCode, err := c.Client.request("POST", url, requests, c.headers(), &resp) - if err != nil { - return err - } - - if statusCode != http.StatusOK { - return fmt.Errorf( - "invalid response code while sending events, expected 200 and received %d", - statusCode, - ) - } - - if len(resp.Response) != len(requests) { - return fmt.Errorf( - "number of response and the request do not match, expecting %d and received %d", - len(requests), - len(resp.Response), - ) - } - - // Loop through the responses and see if all items are marked as `success` we assume the response - // are in the same order as the sending order. - // - // We could add logic later to retry them, currently if sending error fails it's probably because - // Kibana is not answering and the next fetch will probably fails. - var errors multierror.Errors - for _, response := range resp.Response { - if !response.Success { - errors = append(errors, fmt.Errorf("error sending event, reason: %+v", response.Error.Message)) - } - } - - return errors.Err() -} diff --git a/x-pack/libbeat/management/api/auth_client_test.go b/x-pack/libbeat/management/api/auth_client_test.go deleted file mode 100644 index 340a98847e7..00000000000 --- a/x-pack/libbeat/management/api/auth_client_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "encoding/json" - "fmt" - "net/http" - "testing" - "time" - - "github.com/gofrs/uuid" - "github.com/stretchr/testify/assert" -) - -var testEventType = EventType("TEST_EVENT") - -// Create a custom Event type for testing. -type testEvent struct { - Message string `json:"message"` - Type EventType `json:"event_type"` -} - -func (er *EventRequest) UnmarshalJSON(b []byte) error { - resp := struct { - EventType EventType `json:"type"` - Event json.RawMessage `json:"event"` - }{} - - if err := json.Unmarshal(b, &resp); err != nil { - return err - } - - switch resp.EventType { - case testEventType: - event := &testEvent{} - if err := json.Unmarshal(resp.Event, event); err != nil { - return err - } - *er = EventRequest{EventType: resp.EventType, Event: event} - return nil - } - return fmt.Errorf("unknown event type of '%s'", resp.EventType) -} - -func (t *testEvent) EventType() EventType { - return t.Type -} -func (t *testEvent) MarshalJSON() ([]byte, error) { - return json.Marshal(*t) -} - -func (t *testEvent) UnmarshalJSON(b []byte) error { - resp := struct { - Message string `json:"message"` - Type EventType `json:"type"` - }{} - if err := json.Unmarshal(b, &resp); err != nil { - return err - } - *t = testEvent{Message: resp.Message} - return nil -} - -func TestReportEvents(t *testing.T) { - beatUUID, err := uuid.NewV4() - if !assert.NoError(t, err) { - return - } - - accessToken := "my-enroll-token" - - t.Run("successfully send events", func(t *testing.T) { - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Check correct path is used - assert.Equal(t, "/api/beats/"+beatUUID.String()+"/events", r.URL.Path) - - // Check enrollment token is correct - assert.Equal(t, accessToken, r.Header.Get("kbn-beats-access-token")) - - var response []EventRequest - - decoder := json.NewDecoder(r.Body) - err := decoder.Decode(&response) - if !assert.NoError(t, err) { - return - } - - if !assert.Equal(t, 1, len(response)) { - return - } - - expected := &testEvent{Message: "OK"} - received := response[0].Event.(*testEvent) - - if !assert.Equal(t, expected.Message, received.Message) { - return - } - - apiResponse := EventAPIResponse{ - Response: []EventResponse{EventResponse{BaseResponse: BaseResponse{Success: true}}}, - } - - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(apiResponse) - })) - defer server.Close() - auth := &AuthClient{Client: client, AccessToken: accessToken, BeatUUID: beatUUID} - - events := []*testEvent{&testEvent{Message: "OK"}} - - err = reportEvents(auth, events) - assert.NoError(t, err) - }) - - t.Run("bubble up any errors", func(t *testing.T) { - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusBadRequest) - response := BaseResponse{ - Success: false, - Error: ErrorResponse{ - Message: "bad request", - Code: http.StatusBadRequest, - }, - } - json.NewEncoder(w).Encode(response) - })) - defer server.Close() - - auth := &AuthClient{Client: client, AccessToken: accessToken, BeatUUID: beatUUID} - - events := []*testEvent{&testEvent{Message: "OK"}} - - err = reportEvents(auth, events) - assert.Error(t, err) - }) - - t.Run("assert the response", func(t *testing.T) { - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - apiResponse := EventAPIResponse{ - Response: []EventResponse{ - EventResponse{BaseResponse: BaseResponse{Success: true}}, - EventResponse{BaseResponse: BaseResponse{Success: false}}, - }, - } - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(apiResponse) - })) - defer server.Close() - - auth := &AuthClient{Client: client, AccessToken: accessToken, BeatUUID: beatUUID} - - events := []*testEvent{ - &testEvent{Message: "testing-1"}, - &testEvent{Message: "testing-2"}, - } - - err = reportEvents(auth, events) - assert.Error(t, err) - }) - - t.Run("enforce the match of response/request", func(t *testing.T) { - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - apiResponse := EventAPIResponse{ - Response: []EventResponse{ - EventResponse{BaseResponse: BaseResponse{Success: true}}, - }, - } - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(apiResponse) - })) - defer server.Close() - - auth := &AuthClient{Client: client, AccessToken: accessToken, BeatUUID: beatUUID} - - events := []*testEvent{ - &testEvent{Message: "testing-1"}, - &testEvent{Message: "testing-2"}, - } - - err = reportEvents(auth, events) - assert.Error(t, err) - }) -} - -func reportEvents(client AuthClienter, events []*testEvent) error { - requests := make([]EventRequest, len(events)) - for idx, err := range events { - requests[idx] = EventRequest{ - Timestamp: time.Now(), - EventType: testEventType, - Event: err, - } - } - return client.SendEvents(requests) -} diff --git a/x-pack/libbeat/management/api/client.go b/x-pack/libbeat/management/api/client.go deleted file mode 100644 index 6212a69379f..00000000000 --- a/x-pack/libbeat/management/api/client.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "bytes" - "encoding/json" - "net/http" - "net/url" - "time" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/kibana" -) - -const defaultTimeout = 10 * time.Second - -// Client to Central Management API -type Client struct { - client *kibana.Client -} - -// ConfigFromURL generates a full kibana client config from an URL -func ConfigFromURL(kibanaURL string, config *common.Config) (*kibana.ClientConfig, error) { - data, err := url.Parse(kibanaURL) - if err != nil { - return nil, err - } - - var username, password string - if data.User != nil { - username = data.User.Username() - password, _ = data.User.Password() - } - - // Lets pick up any configuration from either the YAML or from the -E flags. - // and merge it with the provided URL. - kibana := kibana.ClientConfig{} - if err := config.Unpack(&kibana); err != nil { - return nil, err - } - - kibana.Protocol = data.Scheme - kibana.Host = data.Host - kibana.Path = data.Path - kibana.Username = username - kibana.Password = password - kibana.Timeout = defaultTimeout - - return &kibana, nil -} - -// NewClient creates and returns a kibana client -func NewClient(cfg *kibana.ClientConfig) (*Client, error) { - client, err := kibana.NewClientWithConfig(cfg) - if err != nil { - return nil, err - } - return &Client{ - client: client, - }, nil -} - -// do a request to the API and unmarshall the message, error if anything fails -func (c *Client) request(method, extraPath string, - body interface{}, headers http.Header, resp interface{}) (int, error) { - - bodyJSON, err := json.Marshal(body) - if err != nil { - return 400, err - } - - statusCode, result, err := c.client.Request( - method, - extraPath, - nil, - headers, - bytes.NewBuffer(bodyJSON), - ) - if err != nil { - return statusCode, err - } - - if statusCode >= 300 { - err = extractError(result) - } else { - if err = json.Unmarshal(result, resp); err != nil { - return statusCode, errors.Wrap(err, "error unmarshaling Kibana response") - } - } - - return statusCode, err -} - -func extractError(b []byte) error { - var result BaseResponse - if err := json.Unmarshal(b, &result); err != nil { - return errors.Wrap(err, "error while parsing Kibana response") - } - return errors.New(result.Error.Message) -} diff --git a/x-pack/libbeat/management/api/client_test.go b/x-pack/libbeat/management/api/client_test.go deleted file mode 100644 index 0bdd8d2de59..00000000000 --- a/x-pack/libbeat/management/api/client_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "net/http" - "net/http/httptest" - "testing" - - "github.com/elastic/beats/v7/libbeat/common" -) - -func newServerClientPair(t *testing.T, handler http.HandlerFunc) (*httptest.Server, *Client) { - mux := http.NewServeMux() - mux.Handle("/api/status", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "Unauthorized", 401) - })) - mux.Handle("/", handler) - - server := httptest.NewServer(mux) - - config, err := ConfigFromURL(server.URL, common.NewConfig()) - if err != nil { - t.Fatal(err) - } - - config.IgnoreVersion = true - - client, err := NewClient(config) - if err != nil { - t.Fatal(err) - } - - return server, client -} diff --git a/x-pack/libbeat/management/api/configuration.go b/x-pack/libbeat/management/api/configuration.go index 04bf525eab4..3ae1ef913d0 100644 --- a/x-pack/libbeat/management/api/configuration.go +++ b/x-pack/libbeat/management/api/configuration.go @@ -5,11 +5,6 @@ package api import ( - "encoding/json" - "fmt" - "net/http" - "sort" - "github.com/mitchellh/hashstructure" "github.com/pkg/errors" @@ -18,8 +13,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) -var errConfigurationNotFound = errors.New("no configuration found, you need to enroll your Beat") - // ConfigBlock stores a piece of config from central management type ConfigBlock struct { Raw map[string]interface{} @@ -50,70 +43,6 @@ func (c *ConfigBlock) ConfigWithMeta() (*reload.ConfigWithMeta, error) { }, nil } -type configResponse struct { - Type string - Raw map[string]interface{} -} - -func (c *configResponse) UnmarshalJSON(b []byte) error { - var resp = struct { - Type string `json:"type"` - Raw map[string]interface{} `json:"config"` - }{} - - if err := json.Unmarshal(b, &resp); err != nil { - return err - } - - converter := selectConverter(resp.Type) - newMap, err := converter(resp.Raw) - if err != nil { - return err - } - *c = configResponse{ - Type: resp.Type, - Raw: newMap, - } - return nil -} - -// Configuration retrieves the list of configuration blocks from Kibana -func (c *AuthClient) Configuration() (ConfigBlocks, error) { - resp := struct { - BaseResponse - ConfigBlocks []*configResponse `json:"list"` - }{} - url := fmt.Sprintf("/api/beats/agent/%s/configuration", c.BeatUUID) - statusCode, err := c.Client.request("GET", url, nil, c.headers(), &resp) - if statusCode == http.StatusNotFound { - return nil, errConfigurationNotFound - } - - if err != nil { - return nil, err - } - - blocks := map[string][]*ConfigBlock{} - for _, block := range resp.ConfigBlocks { - blocks[block.Type] = append(blocks[block.Type], &ConfigBlock{Raw: block.Raw}) - } - - // keep the ordering consistent while grouping the items. - keys := make([]string, 0, len(blocks)) - for k := range blocks { - keys = append(keys, k) - } - sort.Strings(keys) - - res := ConfigBlocks{} - for _, t := range keys { - b := blocks[t] - res = append(res, ConfigBlocksWithType{Type: t, Blocks: b}) - } - - return res, nil -} - // ConfigBlocksEqual returns true if the given config blocks are equal, false if not func ConfigBlocksEqual(a, b ConfigBlocks) (bool, error) { // If there is an errors when hashing the config blocks its because the format changed. @@ -129,8 +58,3 @@ func ConfigBlocksEqual(a, b ConfigBlocks) (bool, error) { return aHash == bHash, nil } - -// IsConfigurationNotFound returns true if the configuration was not found. -func IsConfigurationNotFound(err error) bool { - return err == errConfigurationNotFound -} diff --git a/x-pack/libbeat/management/api/configuration_test.go b/x-pack/libbeat/management/api/configuration_test.go index 94fed11f17f..aaee9f80955 100644 --- a/x-pack/libbeat/management/api/configuration_test.go +++ b/x-pack/libbeat/management/api/configuration_test.go @@ -5,60 +5,11 @@ package api import ( - "fmt" - "net/http" "testing" - "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" ) -// {"list":[{"id":"6c385a04-f315-489e-9208-c87f41911782","type":"filebeat.inputs","config":{"paths":["/tmp/hello.log"]},"tag":"89be4cfd-6249-4ac2-abe2-8f82520ba435"},{"id":"315ff7e9-ae24-4c99-a9d0-ed4314bc8e60","type":"output","config":{"_sub_type":"elasticsearch","username":"elastic","password":"changeme"},"tag":"89be4cfd-6249-4ac2-abe2-8f82520ba435"}],"success":true} -func TestConfiguration(t *testing.T) { - beatUUID, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating Beat ID: %v", err) - } - - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Check correct path is used - assert.Equal(t, "/api/beats/agent/"+beatUUID.String()+"/configuration", r.URL.Path) - - // Check enrollment token is correct - assert.Equal(t, "thisismyenrollmenttoken", r.Header.Get("kbn-beats-access-token")) - - fmt.Fprintf(w, `{"success": true, "list":[{"type":"filebeat.modules","config":{"_sub_type":"apache2"}},{"type":"metricbeat.modules","config":{"_sub_type":"system","period":"10s"}}]}`) - })) - defer server.Close() - - auth := AuthClient{Client: client, AccessToken: "thisismyenrollmenttoken", BeatUUID: beatUUID} - - configs, err := auth.Configuration() - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, 2, len(configs)) - checked := 0 - for _, config := range configs { - if config.Type == "metricbeat.modules" { - assert.Equal(t, &ConfigBlock{Raw: map[string]interface{}{ - "module": "system", - "period": "10s", - }}, config.Blocks[0]) - checked++ - - } else if config.Type == "filebeat.modules" { - assert.Equal(t, &ConfigBlock{Raw: map[string]interface{}{ - "module": "apache2", - }}, config.Blocks[0]) - checked++ - } - } - - assert.Equal(t, 2, checked) -} - func TestConfigBlocksEqual(t *testing.T) { tests := []struct { name string @@ -200,25 +151,3 @@ func TestConfigBlocksEqual(t *testing.T) { }) } } - -func TestUnEnroll(t *testing.T) { - beatUUID, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating Beat UUID: %v", err) - } - - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Check correct path is used - assert.Equal(t, "/api/beats/agent/"+beatUUID.String()+"/configuration", r.URL.Path) - - // Check enrollment token is correct - assert.Equal(t, "thisismyenrollmenttoken", r.Header.Get("kbn-beats-access-token")) - - http.NotFound(w, r) - })) - defer server.Close() - - auth := AuthClient{Client: client, AccessToken: "thisismyenrollmenttoken", BeatUUID: beatUUID} - _, err = auth.Configuration() - assert.True(t, IsConfigurationNotFound(err)) -} diff --git a/x-pack/libbeat/management/api/convert.go b/x-pack/libbeat/management/api/convert.go deleted file mode 100644 index af7da53be46..00000000000 --- a/x-pack/libbeat/management/api/convert.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "fmt" - "strings" -) - -type converter func(map[string]interface{}) (map[string]interface{}, error) - -var mapper = map[string]converter{ - ".inputs": noopConvert, - ".modules": convertMultiple, - "output": convertSingle, -} - -var errSubTypeNotFound = fmt.Errorf("'%s' key not found", subTypeKey) - -var ( - subTypeKey = "_sub_type" - moduleKey = "module" -) - -func selectConverter(t string) converter { - for k, v := range mapper { - if strings.Index(t, k) > -1 { - return v - } - } - return noopConvert -} - -func convertSingle(m map[string]interface{}) (map[string]interface{}, error) { - subType, err := extractSubType(m) - if err != nil { - return nil, err - } - - delete(m, subTypeKey) - newMap := map[string]interface{}{subType: m} - return newMap, nil -} - -func convertMultiple(m map[string]interface{}) (map[string]interface{}, error) { - subType, err := extractSubType(m) - if err != nil { - return nil, err - } - - v, ok := m[moduleKey] - - if ok && v != subType { - return nil, fmt.Errorf("module key already exist in the raw document and doesn't match the 'sub_type', expecting '%s' and received '%s", subType, v) - } - - m[moduleKey] = subType - delete(m, subTypeKey) - return m, nil -} - -func noopConvert(m map[string]interface{}) (map[string]interface{}, error) { - return m, nil -} - -func extractSubType(m map[string]interface{}) (string, error) { - subType, ok := m[subTypeKey] - if !ok { - return "", errSubTypeNotFound - } - - k, ok := subType.(string) - if !ok { - return "", fmt.Errorf("invalid type for `sub_type`, expecting a string received %T", subType) - } - return k, nil -} diff --git a/x-pack/libbeat/management/api/convert_test.go b/x-pack/libbeat/management/api/convert_test.go deleted file mode 100644 index e8f32e90494..00000000000 --- a/x-pack/libbeat/management/api/convert_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "reflect" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestConvertAPI(t *testing.T) { - tests := map[string]struct { - t string - config map[string]interface{} - expected map[string]interface{} - err bool - }{ - "output": { - t: "output", - config: map[string]interface{}{ - "_sub_type": "elasticsearch", - "username": "foobar", - }, - expected: map[string]interface{}{ - "elasticsearch": map[string]interface{}{ - "username": "foobar", - }, - }, - }, - "filebeat inputs": { - t: "filebeat.inputs", - config: map[string]interface{}{ - "type": "log", - "paths": []string{ - "/var/log/message.log", - "/var/log/system.log", - }, - }, - expected: map[string]interface{}{ - "type": "log", - "paths": []string{ - "/var/log/message.log", - "/var/log/system.log", - }, - }, - }, - "filebeat modules": { - t: "filebeat.modules", - config: map[string]interface{}{ - "_sub_type": "system", - }, - expected: map[string]interface{}{ - "module": "system", - }, - }, - "metricbeat modules": { - t: "metricbeat.modules", - config: map[string]interface{}{ - "_sub_type": "logstash", - }, - expected: map[string]interface{}{ - "module": "logstash", - }, - }, - "badly formed output": { - err: true, - t: "output", - config: map[string]interface{}{ - "nosubtype": "logstash", - }, - }, - "badly formed filebeat module": { - err: true, - t: "filebeat.modules", - config: map[string]interface{}{ - "nosubtype": "logstash", - }, - }, - "badly formed metricbeat module": { - err: true, - t: "metricbeat.modules", - config: map[string]interface{}{ - "nosubtype": "logstash", - }, - }, - "unknown type is passthrough": { - t: "unkown", - config: map[string]interface{}{ - "nosubtype": "logstash", - }, - expected: map[string]interface{}{ - "nosubtype": "logstash", - }, - }, - } - - for name, test := range tests { - test := test - t.Run(name, func(t *testing.T) { - converter := selectConverter(test.t) - newMap, err := converter(test.config) - if !assert.Equal(t, test.err, err != nil) { - return - } - assert.True(t, reflect.DeepEqual(newMap, test.expected)) - }) - } -} diff --git a/x-pack/libbeat/management/api/enroll.go b/x-pack/libbeat/management/api/enroll.go deleted file mode 100644 index 354d263d076..00000000000 --- a/x-pack/libbeat/management/api/enroll.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "net/http" - - "github.com/gofrs/uuid" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common" -) - -type enrollResponse struct { - BaseResponse - AccessToken string `json:"item"` -} - -func (e *enrollResponse) Validate() error { - if !e.Success || len(e.AccessToken) == 0 { - return errors.New("empty access_token") - } - return nil -} - -// Enroll a beat in central management, this call returns a valid access token to retrieve -// configurations -func (c *Client) Enroll( - beatType, beatName, beatVersion, hostname string, - beatUUID uuid.UUID, - enrollmentToken string, -) (string, error) { - params := common.MapStr{ - "type": beatType, - "name": beatName, - "version": beatVersion, - "host_name": hostname, - } - - resp := enrollResponse{} - - headers := http.Header{} - headers.Set("kbn-beats-enrollment-token", enrollmentToken) - - _, err := c.request("POST", "/api/beats/agent/"+beatUUID.String(), params, headers, &resp) - if err != nil { - return "", err - } - - if err := resp.Validate(); err != nil { - return "", err - } - - return resp.AccessToken, err -} diff --git a/x-pack/libbeat/management/api/enroll_test.go b/x-pack/libbeat/management/api/enroll_test.go deleted file mode 100644 index a120fb8a2e8..00000000000 --- a/x-pack/libbeat/management/api/enroll_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "testing" - - "github.com/gofrs/uuid" - "github.com/stretchr/testify/assert" -) - -func TestEnrollValid(t *testing.T) { - beatUUID, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating Beat ID: %v", err) - } - - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - body, err := ioutil.ReadAll(r.Body) - if err != nil { - t.Fatal(err) - } - - // Check correct path is used - assert.Equal(t, "/api/beats/agent/"+beatUUID.String(), r.URL.Path) - - // Check enrollment token is correct - assert.Equal(t, "thisismyenrollmenttoken", r.Header.Get("kbn-beats-enrollment-token")) - - request := struct { - Hostname string `json:"host_name"` - Type string `json:"type"` - Version string `json:"version"` - Name string `json:"name"` - }{} - if err := json.Unmarshal(body, &request); err != nil { - t.Fatal(err) - } - - assert.Equal(t, "myhostname.lan", request.Hostname) - assert.Equal(t, "metricbeat", request.Type) - assert.Equal(t, "6.3.0", request.Version) - assert.Equal(t, "beatname", request.Name) - - fmt.Fprintf(w, `{"success": true, "item": "fooo"}`) - })) - defer server.Close() - - accessToken, err := client.Enroll("metricbeat", "beatname", "6.3.0", "myhostname.lan", beatUUID, "thisismyenrollmenttoken") - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, "fooo", accessToken) -} - -func TestEnrollError(t *testing.T) { - tests := map[string]struct { - body string - responseCode int - }{ - "invalid enrollment token": { - body: `{"success": false, "error": { "message": "Invalid enrollment token", "code": 400 }}`, - responseCode: 400, - }, - //NOTE(ph): I believe this is now fixed in the API. - "invalid token response": { - body: `{"success": true, "item": ""}`, - responseCode: 200, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - beatUUID, err := uuid.NewV4() - if err != nil { - t.Fatal(err) - } - - server, client := newServerClientPair(t, http.HandlerFunc(func( - w http.ResponseWriter, - r *http.Request, - ) { - http.Error(w, test.body, test.responseCode) - })) - defer server.Close() - - accessToken, err := client.Enroll( - "metricbeat", - "beatname", - "6.3.0", - "myhostname.lan", - beatUUID, - "thisismyenrollmenttoken", - ) - - assert.Error(t, err) - assert.Equal(t, "", accessToken) - }) - } -} diff --git a/x-pack/libbeat/management/api/enrollment_token.go b/x-pack/libbeat/management/api/enrollment_token.go deleted file mode 100644 index cf20a075a73..00000000000 --- a/x-pack/libbeat/management/api/enrollment_token.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "fmt" - "net/http" -) - -// CreateEnrollmentToken talks to Kibana API and generates an enrollment token -func (c *Client) CreateEnrollmentToken() (string, error) { - headers := http.Header{} - - resp := struct { - Results []struct { - Token string `json:"item"` - } `json:"results"` - }{} - - _, err := c.request("POST", "/api/beats/enrollment_tokens", nil, headers, &resp) - if err != nil { - return "", err - } - - if tokensCount := len(resp.Results); tokensCount != 1 { - return "", fmt.Errorf("Unexpected number of tokens, got %d, only one expected", tokensCount) - } - - return resp.Results[0].Token, nil -} diff --git a/x-pack/libbeat/management/api/enrollment_token_test.go b/x-pack/libbeat/management/api/enrollment_token_test.go deleted file mode 100644 index 2b5241e8f31..00000000000 --- a/x-pack/libbeat/management/api/enrollment_token_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "fmt" - "net/http" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestEnrollmentToken(t *testing.T) { - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Check correct path is used - assert.Equal(t, "/api/beats/enrollment_tokens", r.URL.Path) - fmt.Fprintf(w, `{"results": [{"item":"65074ff8639a4661ba7e1bd5ccc209ed"}]}`) - })) - defer server.Close() - - token, err := client.CreateEnrollmentToken() - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, "65074ff8639a4661ba7e1bd5ccc209ed", token) -} diff --git a/x-pack/libbeat/management/api/event_reporter.go b/x-pack/libbeat/management/api/event_reporter.go deleted file mode 100644 index 8ed492c7e6b..00000000000 --- a/x-pack/libbeat/management/api/event_reporter.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "sync" - "time" - - "github.com/joeshaw/multierror" - - "github.com/elastic/beats/v7/libbeat/logp" -) - -const debugK = "event_reporter" - -// EventReporter is an object that will periodically send asyncronously events to the -// CM events endpoints. -type EventReporter struct { - logger *logp.Logger - client AuthClienter - period time.Duration - maxBatchSize int - done chan struct{} - buffer []Event - mu sync.Mutex - wg sync.WaitGroup -} - -// NewEventReporter returns a new event reporter -func NewEventReporter( - logger *logp.Logger, - client AuthClienter, - period time.Duration, - maxBatchSize int, -) *EventReporter { - log := logger.Named(debugK) - return &EventReporter{ - logger: log, - client: client, - period: period, - maxBatchSize: maxBatchSize, - done: make(chan struct{}), - } -} - -// Start starts the event reported and wait for new events. -func (e *EventReporter) Start() { - e.wg.Add(1) - go e.worker() - e.logger.Info("Starting event reporter service") -} - -// Stop stops the reporting events to the endpoint. -func (e *EventReporter) Stop() { - e.logger.Info("Stopping event reporter service") - close(e.done) - e.wg.Wait() -} - -func (e *EventReporter) worker() { - defer e.wg.Done() - ticker := time.NewTicker(e.period) - defer ticker.Stop() - - var done bool - for !done { - select { - case <-e.done: - done = true - case <-ticker.C: - } - - var buf []Event - e.mu.Lock() - buf, e.buffer = e.buffer, nil - e.mu.Unlock() - - e.reportEvents(buf) - } -} - -func (e *EventReporter) reportEvents(events []Event) { - if len(events) == 0 { - return - } - e.logger.Debugf("Reporting %d events to Kibana", len(events)) - if err := e.sendBatchEvents(events); err != nil { - e.logger.Errorf("could not send events, error: %+v", err) - } -} - -func (e *EventReporter) sendBatchEvents(events []Event) error { - var errors multierror.Errors - for pos := 0; pos < len(events); pos += e.maxBatchSize { - j := pos + e.maxBatchSize - if j > len(events) { - j = len(events) - } - if err := e.sendEvents(events[pos:j]); err != nil { - errors = append(errors, err) - } - } - return errors.Err() -} - -func (e *EventReporter) sendEvents(events []Event) error { - requests := make([]EventRequest, len(events)) - for i, event := range events { - requests[i] = EventRequest{ - Timestamp: time.Now(), - EventType: event.EventType(), - Event: event, - } - } - return e.client.SendEvents(requests) -} - -// AddEvent adds an event to be send on the next tick. -func (e *EventReporter) AddEvent(event Event) { - e.mu.Lock() - defer e.mu.Unlock() - e.buffer = append(e.buffer, event) -} diff --git a/x-pack/libbeat/management/api/event_reporter_test.go b/x-pack/libbeat/management/api/event_reporter_test.go deleted file mode 100644 index e1576bbd093..00000000000 --- a/x-pack/libbeat/management/api/event_reporter_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "math" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/libbeat/logp" -) - -type memoryAuthClient struct { - Requests chan []EventRequest - Err error -} - -func (m *memoryAuthClient) SendEvents(requests []EventRequest) error { - if m.Err != nil { - return m.Err - } - - m.Requests <- requests - return nil -} - -func (m *memoryAuthClient) Close() { - close(m.Requests) -} - -func (m *memoryAuthClient) Configuration() (ConfigBlocks, error) { - return ConfigBlocks{}, nil -} - -func newMemoryAuthClient() *memoryAuthClient { - return &memoryAuthClient{Requests: make(chan []EventRequest)} -} - -func TestReporterReportEvents(t *testing.T) { - t.Run("single request", testBatch(1, 100)) - t.Run("receive all events when the requests size is exactly the batch size", testBatch(100, 100)) - t.Run("receive all events when events are send in multiple batch", testBatch(1234, 25)) -} - -func testBatch(numberOfEvents, maxBatchSize int) func(*testing.T) { - return func(t *testing.T) { - event := &testEvent{Message: "OK"} - client := newMemoryAuthClient() - defer client.Close() - reporter := NewEventReporter(logp.NewLogger(""), client, 1*time.Second, maxBatchSize) - reporter.Start() - defer reporter.Stop() - - go func() { - for i := 0; i < numberOfEvents; i++ { - reporter.AddEvent(event) - } - }() - - var receivedEvents int - expectedbatch := int(math.Ceil(float64(numberOfEvents) / float64(maxBatchSize))) - - for receivedBatchs := 0; receivedBatchs < expectedbatch; receivedBatchs++ { - requests := <-client.Requests - receivedEvents += len(requests) - } - - assert.Equal(t, numberOfEvents, receivedEvents) - } -} diff --git a/x-pack/libbeat/management/cache.go b/x-pack/libbeat/management/cache.go deleted file mode 100644 index 58146a3cbe4..00000000000 --- a/x-pack/libbeat/management/cache.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "io/ioutil" - "os" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/file" - "github.com/elastic/beats/v7/libbeat/paths" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" - - "github.com/pkg/errors" - "gopkg.in/yaml.v2" -) - -// Cache keeps a copy of configs provided by Kibana, it's used when Kibana is down -type Cache struct { - Configs api.ConfigBlocks -} - -// Load settings from its source file -func (c *Cache) Load() error { - path := paths.Resolve(paths.Data, "management.yml") - config, err := common.LoadFile(path) - if err != nil { - if os.IsNotExist(err) { - // File is not present, beat is not enrolled - return nil - } - return err - } - - if err = config.Unpack(&c); err != nil { - return err - } - - return nil -} - -// Save settings to management.yml file -func (c *Cache) Save() error { - path := paths.Resolve(paths.Data, "management.yml") - - data, err := yaml.Marshal(c) - if err != nil { - return err - } - - // write temporary file first - tempFile := path + ".new" - if err := ioutil.WriteFile(tempFile, data, 0600); err != nil { - return errors.Wrap(err, "failed to store central management settings") - } - - // move temporary file into final location - return file.SafeFileRotate(path, tempFile) -} - -// HasConfig returns true if configs are cached. -func (c *Cache) HasConfig() bool { - return len(c.Configs) > 0 -} diff --git a/x-pack/libbeat/management/cache_test.go b/x-pack/libbeat/management/cache_test.go deleted file mode 100644 index 1acbe76d60d..00000000000 --- a/x-pack/libbeat/management/cache_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" -) - -func TestHasConfig(t *testing.T) { - tests := map[string]struct { - configs api.ConfigBlocks - expected bool - }{ - "with config": { - configs: api.ConfigBlocks{ - api.ConfigBlocksWithType{Type: "metricbeat "}, - }, - expected: true, - }, - "without config": { - configs: api.ConfigBlocks{}, - expected: false, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - cache := Cache{Configs: test.configs} - assert.Equal(t, test.expected, cache.HasConfig()) - }) - } -} diff --git a/x-pack/libbeat/management/config.go b/x-pack/libbeat/management/config.go index 61cb0c5cda4..740f28165fb 100644 --- a/x-pack/libbeat/management/config.go +++ b/x-pack/libbeat/management/config.go @@ -4,145 +4,10 @@ package management -import ( - "io" - "text/template" - "time" - - "gopkg.in/yaml.v2" - - "github.com/elastic/beats/v7/libbeat/kibana" -) - -// ManagedConfigTemplate is used to overwrite settings file during enrollment -const ManagedConfigTemplate = ` - -#========================= Central Management ================================= - -# Beats is configured under central management, you can define most settings -# from the Kibana UI. You can update this file to configure the settings that -# are not supported by Kibana Beats management. - -{{.CentralManagementSettings}} -#================================ General ===================================== - -# The name of the shipper that publishes the network data. It can be used to group -# all the transactions sent by a single shipper in the web interface. -#name: - -# The tags of the shipper are included in their own field with each -# transaction published. -#tags: ["service-X", "web-tier"] - -# Optional fields that you can specify to add additional information to the -# output. -#fields: -# env: staging - -#================================ Logging ===================================== - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -#logging.level: debug - -# At debug level, you can selectively enable logging only for some components. -# To enable all selectors use ["*"]. Examples of other selectors are "beat", -# "publisher", "service". -#logging.selectors: ["*"] - -#============================== X-Pack Monitoring =============================== -# {{.BeatName}} can export internal metrics to a central Elasticsearch monitoring -# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The -# reporting is disabled by default. - -# Set to true to enable the monitoring reporter. -#monitoring.enabled: false - -# Uncomment to send the metrics to Elasticsearch. Most settings from the -# Elasticsearch output are accepted here as well. -# Note that the settings should point to your Elasticsearch *monitoring* cluster. -# Any setting that is not set is automatically inherited from the Elasticsearch -# output configuration, so if you have the Elasticsearch output configured such -# that it is pointing to your Elasticsearch monitoring cluster, you can simply -# uncomment the following line. -#monitoring.elasticsearch: -` - const ( // ModeCentralManagement is a default CM mode, using existing processes. - ModeCentralManagement = "x-pack-cm" + ModeCentralManagement = "x-pack-cm" // TODO remove? // ModeFleet is a management mode where fleet is used to retrieve configurations. ModeFleet = "x-pack-fleet" ) - -// Config for central management. -type Config struct { - // true when enrolled - Enabled bool `config:"enabled" yaml:"enabled"` - - // Mode specifies whether beat uses Central Management or Fleet. - // Options: [cm, fleet] - Mode string `config:"mode" yaml:"mode"` - - // Poll configs period - Period time.Duration `config:"period" yaml:"period"` - - EventsReporter EventReporterConfig `config:"events_reporter" yaml:"events_reporter"` - - AccessToken string `config:"access_token" yaml:"access_token"` - - Kibana *kibana.ClientConfig `config:"kibana" yaml:"kibana"` - - Blacklist ConfigBlacklistSettings `config:"blacklist" yaml:"blacklist"` -} - -// EventReporterConfig configuration of the events reporter. -type EventReporterConfig struct { - Period time.Duration `config:"period" yaml:"period"` - MaxBatchSize int `config:"max_batch_size" yaml:"max_batch_size" validate:"nonzero,positive"` -} - -func defaultConfig() *Config { - return &Config{ - Mode: ModeCentralManagement, - Period: 60 * time.Second, - EventsReporter: EventReporterConfig{ - Period: 30 * time.Second, - MaxBatchSize: 1000, - }, - Blacklist: ConfigBlacklistSettings{ - Patterns: map[string]string{ - "output": "console|file", - }, - }, - } -} - -type templateParams struct { - CentralManagementSettings string - BeatName string -} - -// OverwriteConfigFile will overwrite beat settings file with the enrolled template. -func (c *Config) OverwriteConfigFile(wr io.Writer, beatName string) error { - t := template.Must(template.New("beat.management.yml").Parse(ManagedConfigTemplate)) - - tmp := struct { - Management *Config `yaml:"management"` - }{ - Management: c, - } - - data, err := yaml.Marshal(tmp) - if err != nil { - return err - } - - params := templateParams{ - CentralManagementSettings: string(data), - BeatName: beatName, - } - - return t.Execute(wr, params) -} diff --git a/x-pack/libbeat/management/config_test.go b/x-pack/libbeat/management/config_test.go deleted file mode 100644 index 25ece2e216c..00000000000 --- a/x-pack/libbeat/management/config_test.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func EnsureBlacklistItems(t *testing.T) { - // NOTE: We do not permit to configure the console or the file output with CM for security reason. - c := defaultConfig() - v, _ := c.Blacklist.Patterns["output"] - assert.Equal(t, "console|file", v) -} diff --git a/x-pack/libbeat/management/enroll.go b/x-pack/libbeat/management/enroll.go deleted file mode 100644 index 407808ec7ae..00000000000 --- a/x-pack/libbeat/management/enroll.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "fmt" - "os" - "time" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/cfgfile" - "github.com/elastic/beats/v7/libbeat/cmd/instance" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/common/file" - "github.com/elastic/beats/v7/libbeat/keystore" - "github.com/elastic/beats/v7/libbeat/kibana" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" -) - -const accessTokenKey = "management.accesstoken" - -// Enroll this beat to the given kibana -// This will use Central Management API to enroll and retrieve an access key for config retrieval -func Enroll( - beat *instance.Beat, - kibanaConfig *kibana.ClientConfig, - enrollmentToken string, -) error { - // Ignore kibana version to avoid permission errors - kibanaConfig.IgnoreVersion = true - - client, err := api.NewClient(kibanaConfig) - if err != nil { - return err - } - - cfgwarn.Deprecate("8.0.0", "Central Management is no longer under development and has been deprecated. We are working hard to deliver a new and more comprehensive solution and look forward to sharing it with you") - - accessToken, err := client.Enroll(beat.Info.Beat, beat.Info.Name, beat.Info.Version, beat.Info.Hostname, beat.Info.ID, enrollmentToken) - if err != nil { - return err - } - - // Store access token in keystore - if err := storeAccessToken(beat, accessToken); err != nil { - return err - } - - // Enrolled, persist state - config := defaultConfig() - config.Enabled = true - config.AccessToken = "${" + accessTokenKey + "}" - config.Kibana = kibanaConfig - - configFile := cfgfile.GetDefaultCfgfile() - - ts := time.Now() - - // This timestamp format is a variation of RFC3339 replacing colons with - // slashes so that it can be used as part of a filename in all OSes. - // (Colon is not a valid character for filenames in Windows). - // Also removed the TZ-offset as that can cause a plus sign to be output. - const fsSafeTimestamp = "2006-01-02T15-04-05" - - // backup current settings: - backConfigFile := configFile + "." + ts.Format(fsSafeTimestamp) + ".bak" - fmt.Println("Saving a copy of current settings to " + backConfigFile) - err = file.SafeFileRotate(backConfigFile, configFile) - if err != nil { - return errors.Wrap(err, "creating a backup copy of current settings") - } - - // create the new ones: - f, err := os.OpenFile(configFile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - return errors.Wrap(err, "opening settings file") - } - defer f.Close() - - if err := config.OverwriteConfigFile(f, beat.Beat.Info.Beat); err != nil { - return errors.Wrap(err, "overriding settings file") - } - - return nil -} - -func storeAccessToken(beat *instance.Beat, accessToken string) error { - keyStore := beat.Keystore() - - wKeystore, err := keystore.AsWritableKeystore(keyStore) - if err != nil { - return err - } - - if !keyStore.IsPersisted() { - - if err := wKeystore.Create(false); err != nil { - return errors.Wrap(err, "error creating keystore") - } - } - if err := wKeystore.Store(accessTokenKey, []byte(accessToken)); err != nil { - return errors.Wrap(err, "error storing the access token") - } - - return wKeystore.Save() -} diff --git a/x-pack/libbeat/management/error.go b/x-pack/libbeat/management/error.go index 31fd8df3d08..b70aa5387b3 100644 --- a/x-pack/libbeat/management/error.go +++ b/x-pack/libbeat/management/error.go @@ -11,8 +11,6 @@ import ( "github.com/gofrs/uuid" "github.com/pkg/errors" - - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" ) // ErrorType is type of error that the events endpoint understand. @@ -21,9 +19,6 @@ type ErrorType string // ConfigError is the type of error send when an unpack or a blacklist happen. var ConfigError = ErrorType("CONFIG") -// ErrorEvent is the event type when an error happen. -var ErrorEvent = api.EventType("ERROR") - // Error is a config error to be reported to kibana. type Error struct { Type ErrorType @@ -31,11 +26,6 @@ type Error struct { Err error } -// EventType returns a ErrorEvent. -func (e *Error) EventType() api.EventType { - return ErrorEvent -} - // MarshalJSON transform an error into a JSON document. func (e *Error) MarshalJSON() ([]byte, error) { return json.Marshal(&struct { diff --git a/x-pack/libbeat/management/error_test.go b/x-pack/libbeat/management/error_test.go index 4d29d3aacad..4c0d6beb141 100644 --- a/x-pack/libbeat/management/error_test.go +++ b/x-pack/libbeat/management/error_test.go @@ -11,10 +11,29 @@ import ( "github.com/gofrs/uuid" "github.com/pkg/errors" "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" ) +// Ensure that all events have a Message key that can by used by the GUI. +func ensureJSONhasGeneralfield(t *testing.T, obj json.Marshaler) func(*testing.T) { + return func(t *testing.T) { + serialized, err := json.Marshal(obj) + if !assert.NoError(t, err) { + return + } + + message := struct { + Message string `json:"message"` + }{} + + err = json.Unmarshal(serialized, &message) + + if !assert.NoError(t, err) { + return + } + assert.NotEmpty(t, message) + } +} + func TestErrorSerialization(t *testing.T) { id, _ := uuid.NewV4() t.Run("serialize ok", func(t *testing.T) { @@ -42,7 +61,7 @@ func TestErrorSerialization(t *testing.T) { assert.Equal(t, e.UUID.String(), resp.UUID) assert.Equal(t, e.Err.Error(), resp.Message) - assert.Equal(t, e.Type, api.EventType(resp.Type)) + assert.Equal(t, e.Type, resp.Type) }) t.Run("ensure that json general fields are present", ensureJSONhasGeneralfield(t, &Error{ diff --git a/x-pack/libbeat/management/fleet/config.go b/x-pack/libbeat/management/fleet/config.go index 9572d30acc9..3ab745ef069 100644 --- a/x-pack/libbeat/management/fleet/config.go +++ b/x-pack/libbeat/management/fleet/config.go @@ -17,7 +17,7 @@ type Config struct { func defaultConfig() *Config { return &Config{ - Mode: xmanagement.ModeCentralManagement, + Mode: xmanagement.ModeCentralManagement, // TODO Fleet mode? Blacklist: xmanagement.ConfigBlacklistSettings{ Patterns: map[string]string{ "output": "console|file", diff --git a/x-pack/libbeat/management/manager.go b/x-pack/libbeat/management/manager.go deleted file mode 100644 index cfbf72f234c..00000000000 --- a/x-pack/libbeat/management/manager.go +++ /dev/null @@ -1,345 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "fmt" - "sync" - "time" - - "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/elastic/elastic-agent-client/v7/pkg/client" - - "github.com/gofrs/uuid" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/management" -) - -var errEmptyAccessToken = errors.New("access_token is empty, you must reenroll your Beat") - -// ConfigManager handles internal config updates. By retrieving -// new configs from Kibana and applying them to the Beat -type ConfigManager struct { - config *Config - cache *Cache - logger *logp.Logger - client api.AuthClienter - beatUUID uuid.UUID - done chan struct{} - registry *reload.Registry - wg sync.WaitGroup - blacklist *ConfigBlacklist - reporter *api.EventReporter - state *State - mux sync.RWMutex -} - -// NewConfigManager returns a X-Pack Beats Central Management manager -func NewConfigManager(config *common.Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { - c := defaultConfig() - if config.Enabled() { - if err := config.Unpack(&c); err != nil { - return nil, errors.Wrap(err, "parsing central management settings") - } - } - return NewConfigManagerWithConfig(c, registry, beatUUID) -} - -// NewConfigManagerWithConfig returns a X-Pack Beats Central Management manager -func NewConfigManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { - var client *api.Client - var cache *Cache - var blacklist *ConfigBlacklist - - if c.Enabled { - cfgwarn.Deprecate("8.0.0", "Central Management is no longer under development and has been deprecated. We are working hard to deliver a new and more comprehensive solution and look forward to sharing it with you") - - var err error - - if err = validateConfig(c); err != nil { - return nil, errors.Wrap(err, "wrong settings for configurations") - } - - // Initialize configs blacklist - blacklist, err = NewConfigBlacklist(c.Blacklist) - if err != nil { - return nil, errors.Wrap(err, "wrong settings for configurations blacklist") - } - - // Initialize central management settings cache - cache = &Cache{} - if err := cache.Load(); err != nil { - return nil, errors.Wrap(err, "reading central management internal cache") - } - - // Ignore kibana version to avoid permission errors - c.Kibana.IgnoreVersion = true - - client, err = api.NewClient(c.Kibana) - if err != nil { - return nil, errors.Wrap(err, "initializing kibana client") - } - } - - authClient := &api.AuthClient{Client: client, AccessToken: c.AccessToken, BeatUUID: beatUUID} - log := logp.NewLogger(management.DebugK) - - return &ConfigManager{ - config: c, - cache: cache, - blacklist: blacklist, - logger: log, - client: authClient, - done: make(chan struct{}), - beatUUID: beatUUID, - registry: registry, - reporter: api.NewEventReporter( - log, - authClient, - c.EventsReporter.Period, - c.EventsReporter.MaxBatchSize, - ), - }, nil -} - -// Enabled returns true if config management is enabled -func (cm *ConfigManager) Enabled() bool { - return cm.config.Enabled -} - -func (cm *ConfigManager) RegisterAction(action client.Action) {} - -func (cm *ConfigManager) UnregisterAction(action client.Action) {} - -func (cm *ConfigManager) SetPayload(map[string]interface{}) {} - -// Start the config manager -func (cm *ConfigManager) Start(_ func()) { - if !cm.Enabled() { - return - } - cfgwarn.Beta("Central management is enabled") - cm.logger.Info("Starting central management service") - - cm.reporter.Start() - cm.wg.Add(1) - go cm.worker() -} - -// Stop the config manager -func (cm *ConfigManager) Stop() { - if !cm.Enabled() { - return - } - - // stop collecting configuration - cm.logger.Info("Stopping central management service") - close(cm.done) - cm.wg.Wait() - - // report last state and stop reporting. - cm.updateState(Stopped) - cm.reporter.Stop() -} - -// CheckRawConfig check settings are correct to start the beat. This method -// checks there are no collision between the existing configuration and what -// central management can configure. -func (cm *ConfigManager) CheckRawConfig(cfg *common.Config) error { - // TODO implement this method - return nil -} - -// UpdateStatus updates the manager with the current status for the beat. -func (cm *ConfigManager) UpdateStatus(_ management.Status, _ string) { - // do nothing; no longer under development and has been deprecated -} - -func (cm *ConfigManager) worker() { - defer cm.wg.Done() - - // Initial fetch && apply (even if errors happen while fetching) - firstRun := true - period := 0 * time.Second - - cm.updateState(Starting) - - // Start worker loop: fetch + apply + cache new settings - for { - select { - case <-cm.done: - return - case <-time.After(period): - } - - changed := cm.fetch() - if changed || firstRun { - cm.updateState(InProgress) - // configs changed, apply changes - // TODO only reload the blocks that changed - if errs := cm.apply(); !errs.IsEmpty() { - cm.reportErrors(errs) - cm.updateState(Failed) - cm.logger.Errorf("Could not apply the configuration, error: %+v", errs) - } else { - cm.updateState(Running) - } - } - - if changed { - // store new configs (already applied) - cm.logger.Info("Storing new state") - if err := cm.cache.Save(); err != nil { - cm.logger.Errorf("error storing central management state: %s", err) - } - } - - if firstRun { - period = cm.config.Period - firstRun = false - } - } -} - -func (cm *ConfigManager) reportErrors(errs Errors) { - for _, err := range errs { - cm.reporter.AddEvent(err) - } -} - -// fetch configurations from kibana, return true if they changed -func (cm *ConfigManager) fetch() bool { - cm.logger.Debug("Retrieving new configurations from Kibana") - configs, err := cm.client.Configuration() - - if api.IsConfigurationNotFound(err) { - if cm.cache.HasConfig() { - cm.logger.Error("Disabling all running configuration because no configurations were found for this Beat, the endpoint returned a 404 or the beat is not enrolled with central management") - cm.cache.Configs = api.ConfigBlocks{} - } - return true - } - - if err != nil { - cm.logger.Errorf("error retrieving new configurations, will use cached ones: %s", err) - return false - } - - equal, err := api.ConfigBlocksEqual(configs, cm.cache.Configs) - if err != nil { - cm.logger.Errorf("error comparing the configurations, will use cached ones: %s", err) - return false - } - - if equal { - cm.logger.Debug("configuration didn't change, sleeping") - return false - } - - cm.logger.Info("New configurations retrieved") - cm.cache.Configs = configs - - return true -} - -func (cm *ConfigManager) apply() Errors { - var errors Errors - missing := map[string]bool{} - for _, name := range cm.registry.GetRegisteredNames() { - missing[name] = true - } - - // Detect unwanted configs from the list - if errs := cm.blacklist.Detect(cm.cache.Configs); !errs.IsEmpty() { - errors = append(errors, errs...) - return errors - } - - // Reload configs - for _, b := range cm.cache.Configs { - if err := cm.reload(b.Type, b.Blocks); err != nil { - errors = append(errors, err) - } - missing[b.Type] = false - } - - // Unset missing configs - for name := range missing { - if missing[name] { - if err := cm.reload(name, []*api.ConfigBlock{}); err != nil { - errors = append(errors, err) - } - } - } - - return errors -} - -func (cm *ConfigManager) reload(t string, blocks []*api.ConfigBlock) *Error { - cm.logger.Infof("Applying settings for %s", t) - if obj := cm.registry.GetReloadable(t); obj != nil { - // Single object - if len(blocks) > 1 { - err := fmt.Errorf("got an invalid number of configs for %s: %d, expected: 1", t, len(blocks)) - cm.logger.Error(err) - return NewConfigError(err) - } - - var config *reload.ConfigWithMeta - var err error - if len(blocks) == 1 { - config, err = blocks[0].ConfigWithMeta() - if err != nil { - cm.logger.Error(err) - return NewConfigError(err) - } - } - - if err := obj.Reload(config); err != nil { - cm.logger.Error(err) - return NewConfigError(err) - } - } else if obj := cm.registry.GetReloadableList(t); obj != nil { - // List - var configs []*reload.ConfigWithMeta - for _, block := range blocks { - config, err := block.ConfigWithMeta() - if err != nil { - cm.logger.Error(err) - return NewConfigError(err) - } - configs = append(configs, config) - } - - if err := obj.Reload(configs); err != nil { - cm.logger.Error(err) - return NewConfigError(err) - } - } - - return nil -} - -func (cm *ConfigManager) updateState(state State) { - cm.mux.Lock() - defer cm.mux.Unlock() - cm.state = &state - cm.reporter.AddEvent(&state) - cm.logger.Infof("Updating state to '%s'", state) -} - -func validateConfig(config *Config) error { - if len(config.AccessToken) == 0 { - return errEmptyAccessToken - } - return nil -} diff --git a/x-pack/libbeat/management/manager_test.go b/x-pack/libbeat/management/manager_test.go deleted file mode 100644 index c783989b708..00000000000 --- a/x-pack/libbeat/management/manager_test.go +++ /dev/null @@ -1,472 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "os" - "sync" - "testing" - "time" - - "github.com/gofrs/uuid" - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/elastic/beats/v7/libbeat/paths" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" -) - -type reloadable struct { - reloaded chan *reload.ConfigWithMeta -} - -func (r *reloadable) Reload(c *reload.ConfigWithMeta) error { - r.reloaded <- c - return nil -} - -func TestConfigManager(t *testing.T) { - registry := reload.NewRegistry() - id, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating id: %v", err) - } - accessToken := "footoken" - reloadable := reloadable{ - reloaded: make(chan *reload.ConfigWithMeta, 1), - } - registry.MustRegister("test.block", &reloadable) - - mux := http.NewServeMux() - i := 0 - responses := []string{ - // Initial load - `{"success": true, "list":[{"type":"test.block","config":{"module":"apache2"}}]}`, - - // No change, no reload - `{"success": true, "list":[{"type":"test.block","config":{"module":"apache2"}}]}`, - - // Changed, reload - `{"success": true, "list":[{"type":"test.block","config":{"module":"system"}}]}`, - } - mux.Handle(fmt.Sprintf("/api/beats/agent/%s/configuration", id), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, responses[i]) - i++ - })) - - reporter := addEventsReporterHandle(mux, id) - - server := httptest.NewServer(mux) - - c, err := api.ConfigFromURL(server.URL, common.NewConfig()) - if err != nil { - t.Fatal(err) - } - - config := &Config{ - Enabled: true, - Mode: ModeCentralManagement, - Period: 100 * time.Millisecond, - Kibana: c, - AccessToken: accessToken, - EventsReporter: EventReporterConfig{ - Period: 50 * time.Millisecond, - MaxBatchSize: 1, - }, - } - - manager, err := NewConfigManagerWithConfig(config, registry, id) - if err != nil { - t.Fatal(err) - } - - manager.Start(func() {}) - - // On first reload we will get apache2 module - config1 := <-reloadable.reloaded - assert.Equal(t, &reload.ConfigWithMeta{ - Config: common.MustNewConfigFrom(map[string]interface{}{ - "module": "apache2", - }), - }, config1) - - config2 := <-reloadable.reloaded - assert.Equal(t, &reload.ConfigWithMeta{ - Config: common.MustNewConfigFrom(map[string]interface{}{ - "module": "system", - }), - }, config2) - - // Cleanup - manager.Stop() - os.Remove(paths.Resolve(paths.Data, "management.yml")) - - events := []api.Event{&Starting, &InProgress, &Running, &InProgress, &Running, &Stopped} - assertEvents(t, events, reporter) -} - -func TestRemoveItems(t *testing.T) { - registry := reload.NewRegistry() - id, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating id: %v", err) - } - accessToken := "footoken" - reloadable := reloadable{ - reloaded: make(chan *reload.ConfigWithMeta, 1), - } - registry.MustRegister("test.blocks", &reloadable) - - mux := http.NewServeMux() - i := 0 - responses := []string{ - // Initial load - `{"success": true, "list":[{"type":"test.blocks","config":{"module":"apache2"}}]}`, - - // Return no blocks - `{"success": true, "list":[]}`, - } - mux.Handle(fmt.Sprintf("/api/beats/agent/%s/configuration", id), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, responses[i]) - i++ - })) - - reporter := addEventsReporterHandle(mux, id) - server := httptest.NewServer(mux) - - c, err := api.ConfigFromURL(server.URL, common.NewConfig()) - if err != nil { - t.Fatal(err) - } - - config := &Config{ - Enabled: true, - Mode: ModeCentralManagement, - Period: 100 * time.Millisecond, - Kibana: c, - AccessToken: accessToken, - EventsReporter: EventReporterConfig{ - Period: 50 * time.Millisecond, - MaxBatchSize: 1, - }, - } - - manager, err := NewConfigManagerWithConfig(config, registry, id) - if err != nil { - t.Fatal(err) - } - - manager.Start(func() {}) - - // On first reload we will get apache2 module - config1 := <-reloadable.reloaded - assert.Equal(t, &reload.ConfigWithMeta{ - Config: common.MustNewConfigFrom(map[string]interface{}{ - "module": "apache2", - }), - }, config1) - - // Get a nil config, even if the block is not part of the payload - config2 := <-reloadable.reloaded - var nilConfig *reload.ConfigWithMeta - assert.Equal(t, nilConfig, config2) - - // Cleanup - manager.Stop() - os.Remove(paths.Resolve(paths.Data, "management.yml")) - - events := []api.Event{&Starting, &InProgress, &Running, &InProgress, &Running, &Stopped} - assertEvents(t, events, reporter) -} - -func responseText(s string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, s) - } -} - -func TestUnEnroll(t *testing.T) { - registry := reload.NewRegistry() - id, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating id: %v", err) - } - accessToken := "footoken" - reloadable := reloadable{ - reloaded: make(chan *reload.ConfigWithMeta, 1), - } - registry.MustRegister("test.blocks", &reloadable) - - mux := http.NewServeMux() - i := 0 - responses := []http.HandlerFunc{ // Initial load - responseText(`{"success": true, "list":[{"type":"test.blocks","config":{"module":"apache2"}}]}`), - http.NotFound, - } - - mux.Handle(fmt.Sprintf("/api/beats/agent/%s/configuration", id), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - responses[i](w, r) - i++ - })) - - reporter := addEventsReporterHandle(mux, id) - server := httptest.NewServer(mux) - - c, err := api.ConfigFromURL(server.URL, common.NewConfig()) - if err != nil { - t.Fatal(err) - } - - config := &Config{ - Enabled: true, - Mode: ModeCentralManagement, - Period: 100 * time.Millisecond, - Kibana: c, - AccessToken: accessToken, - EventsReporter: EventReporterConfig{ - Period: 50 * time.Millisecond, - MaxBatchSize: 1, - }, - } - - manager, err := NewConfigManagerWithConfig(config, registry, id) - if err != nil { - t.Fatal(err) - } - - manager.Start(func() {}) - - // On first reload we will get apache2 module - config1 := <-reloadable.reloaded - assert.Equal(t, &reload.ConfigWithMeta{ - Config: common.MustNewConfigFrom(map[string]interface{}{ - "module": "apache2", - }), - }, config1) - - // Get a nil config, even if the block is not part of the payload - config2 := <-reloadable.reloaded - var nilConfig *reload.ConfigWithMeta - assert.Equal(t, nilConfig, config2) - - // Cleanup - manager.Stop() - os.Remove(paths.Resolve(paths.Data, "management.yml")) - - events := []api.Event{&Starting, &InProgress, &Running, &InProgress, &Running, &Stopped} - assertEvents(t, events, reporter) -} - -func TestBadConfig(t *testing.T) { - registry := reload.NewRegistry() - id, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating id: %v", err) - } - accessToken := "footoken" - reloadable := reloadable{ - reloaded: make(chan *reload.ConfigWithMeta, 1), - } - registry.MustRegister("test.blocks", &reloadable) - - mux := http.NewServeMux() - i := 0 - responses := []http.HandlerFunc{ // Initial load - responseText(`{"success": true, "list":[{"type":"output","config":{"_sub_type": "console", "path": "/tmp/bad"}}]}`), - // will not resend new events - responseText(`{"success": true, "list":[{"type":"output","config":{"_sub_type": "console", "path": "/tmp/bad"}}]}`), - // recover on call - http.NotFound, - } - - mux.Handle(fmt.Sprintf("/api/beats/agent/%s/configuration", id), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - responses[i](w, r) - i++ - })) - - reporter := addEventsReporterHandle(mux, id) - server := httptest.NewServer(mux) - - c, err := api.ConfigFromURL(server.URL, common.NewConfig()) - if err != nil { - t.Fatal(err) - } - - config := &Config{ - Enabled: true, - Mode: ModeCentralManagement, - Period: 100 * time.Millisecond, - Kibana: c, - AccessToken: accessToken, - EventsReporter: EventReporterConfig{ - Period: 50 * time.Millisecond, - MaxBatchSize: 1, - }, - Blacklist: ConfigBlacklistSettings{ - Patterns: map[string]string{ - "output": "console|file", - }, - }, - } - - manager, err := NewConfigManagerWithConfig(config, registry, id) - if err != nil { - t.Fatal(err) - } - - manager.Start(func() {}) - - // On first reload we will get apache2 module - config1 := <-reloadable.reloaded - assert.Nil(t, config1) - - // Cleanup - manager.Stop() - os.Remove(paths.Resolve(paths.Data, "management.yml")) - - events := []api.Event{ - &Starting, - &InProgress, - &Error{Type: ConfigError, Err: errors.New("Config for 'output' is blacklisted")}, - &Failed, - &InProgress, // recovering on NotFound, to get out of the blocking. - &Running, - &Stopped, - } - assertEvents(t, events, reporter) -} - -type testEventRequest struct { - EventType api.EventType - Event api.Event -} - -func (er *testEventRequest) UnmarshalJSON(b []byte) error { - resp := struct { - EventType api.EventType `json:"type"` - Event json.RawMessage `json:"event"` - }{} - - if err := json.Unmarshal(b, &resp); err != nil { - return err - } - - switch resp.EventType { - case ErrorEvent: - event := &Error{} - if err := json.Unmarshal(resp.Event, event); err != nil { - return err - } - *er = testEventRequest{EventType: resp.EventType, Event: event} - return nil - case StateEvent: - event := State("") - if err := json.Unmarshal(resp.Event, &event); err != nil { - return err - } - *er = testEventRequest{EventType: resp.EventType, Event: &event} - return nil - } - return fmt.Errorf("unknown event type of '%s'", resp.EventType) -} - -// collect in the background any events generated from the managers. -type collectEventRequest struct { - sync.Mutex - requests []testEventRequest -} - -func (r *collectEventRequest) Requests() []testEventRequest { - r.Lock() - defer r.Unlock() - return r.requests -} - -func (r *collectEventRequest) Add(requests ...testEventRequest) { - r.Lock() - defer r.Unlock() - r.requests = append(r.requests, requests...) -} - -func addEventsReporterHandle(mux *http.ServeMux, uuid uuid.UUID) *collectEventRequest { - reporter := &collectEventRequest{} - path := "/api/beats/" + uuid.String() + "/events" - fn := func(w http.ResponseWriter, r *http.Request) { - var requests []testEventRequest - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&requests); err != nil { - http.Error(w, "could not decode JSON", 500) - } - - reporter.Add(requests...) - resp := api.EventAPIResponse{Response: make([]api.EventResponse, len(requests))} - - for i := 0; i < len(requests); i++ { - resp.Response[i] = api.EventResponse{BaseResponse: api.BaseResponse{Success: true}} - } - - json.NewEncoder(w).Encode(resp) - } - mux.Handle(path, http.HandlerFunc(fn)) - return reporter -} - -func assertEvents(t *testing.T, events []api.Event, reporter *collectEventRequest) { - requests := reporter.Requests() - if !assert.Equal(t, len(events), len(requests)) { - return - } - - for i := 0; i < len(events); i++ { - switch v := requests[i].Event.(type) { - case *State: - assert.Equal(t, events[i], requests[i].Event) - case *Error: - comparable := events[i].(*Error) - assert.Error(t, comparable.Err, v.Err) - default: - t.Fatalf("cannot assert unknown type: %T", requests[i].Event) - } - } -} - -func TestConfigValidate(t *testing.T) { - tests := map[string]struct { - config *common.Config - err bool - }{ - "missing access_token": { - config: common.MustNewConfigFrom(map[string]interface{}{}), - err: true, - }, - "access_token is present": { - config: common.MustNewConfigFrom(map[string]interface{}{"access_token": "abc1234"}), - err: false, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - c := defaultConfig() - err := test.config.Unpack(c) - if assert.NoError(t, err) { - return - } - - err = validateConfig(c) - if test.err { - assert.Error(t, err) - return - } - assert.NoError(t, err) - }) - } -} diff --git a/x-pack/libbeat/management/plugin.go b/x-pack/libbeat/management/plugin.go deleted file mode 100644 index b1e7b7d13b9..00000000000 --- a/x-pack/libbeat/management/plugin.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/feature" - "github.com/elastic/beats/v7/libbeat/management" -) - -func init() { - management.Register("x-pack", NewManagerPlugin, feature.Beta) -} - -// NewManagerPlugin creates a plugin function returning factory if configuration matches the criteria -func NewManagerPlugin(config *common.Config) management.FactoryFunc { - c := defaultConfig() - if config.Enabled() { - if err := config.Unpack(&c); err != nil { - return nil - } - - if c.Mode == ModeCentralManagement { - return NewConfigManager - } - } - - return nil -} diff --git a/x-pack/libbeat/management/state.go b/x-pack/libbeat/management/state.go deleted file mode 100644 index 00fdc7c169b..00000000000 --- a/x-pack/libbeat/management/state.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "encoding/json" - "fmt" - - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" -) - -// StateEvent is a state change notification. -var StateEvent = api.EventType("STATE") - -var ( - // Starting is when the Manager is created and no config are currently active. - Starting = State("STARTING") - // InProgress we have received a new config from the Remote endpoint and we are trying to apply it. - InProgress = State("IN_PROGRESS") - // Running is set when all the config are successfully applied. - Running = State("RUNNING") - // Failed is set if an unpack failed, a a blacklisted option is set or when a reload fails. - Failed = State("FAILED") - // Stopped is set when CM is shutting down, on close the event reported will flush any pending states. - Stopped = State("STOPPED") -) - -var translateState = map[string]State{ - "STARTING": Starting, - "IN_PROGRESS": InProgress, - "RUNNING": Running, - "FAILED": Failed, - "STOPPED": Stopped, -} - -// State represents the internal State of the CM Manager, it does not yet represent -// the full status of beats, because if the manager is marked as Failed it is possible that -// Beat is in fact partially working. A failed state represents an error while unpacking the config -// or when a module failed to reload. -type State string - -// MarshalJSON marshals a status into a valid JSON document. -func (s *State) MarshalJSON() ([]byte, error) { - res := struct { - Type string `json:"type"` - Message string `json:"message"` - }{ - Type: string(*s), - Message: fmt.Sprintf("State change: %s", *s), - } - return json.Marshal(&res) -} - -// EventType returns the type of event. -func (s *State) EventType() api.EventType { - return StateEvent -} - -// UnmarshalJSON unmarshals the State. -func (s *State) UnmarshalJSON(b []byte) error { - raw := struct { - Type string `json:"type"` - }{} - - if err := json.Unmarshal(b, &raw); err != nil { - return err - } - - v, ok := translateState[raw.Type] - if !ok { - return fmt.Errorf("unknown state %s", raw.Type) - } - - *s = v - return nil -} - -func (s *State) String() string { - return string(*s) -} diff --git a/x-pack/libbeat/management/state_test.go b/x-pack/libbeat/management/state_test.go deleted file mode 100644 index 2f0ab7df866..00000000000 --- a/x-pack/libbeat/management/state_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSerializationOfState(t *testing.T) { - t.Run("serialize ok", func(t *testing.T) { - e := &Starting - - b, err := json.Marshal(&e) - if assert.NoError(t, err) { - return - } - - resp := &struct { - Message string `json:"message"` - Type string `json:"type"` - }{} - - err = json.Unmarshal(b, resp) - if assert.NoError(t, err) { - return - } - - assert.Equal(t, e.String(), resp.Type) - assert.NotEmpty(t, resp.Message) - }) - t.Run("ensure that json general fields are present", ensureJSONhasGeneralfield(t, &Starting)) -} - -// Ensure that all events have a Message key that can by used by the GUI. -func ensureJSONhasGeneralfield(t *testing.T, obj json.Marshaler) func(*testing.T) { - return func(t *testing.T) { - serialized, err := json.Marshal(obj) - if !assert.NoError(t, err) { - return - } - - message := struct { - Message string `json:"message"` - }{} - - err = json.Unmarshal(serialized, &message) - - if !assert.NoError(t, err) { - return - } - assert.NotEmpty(t, message) - } -} diff --git a/x-pack/metricbeat/cmd/root.go b/x-pack/metricbeat/cmd/root.go index 242657049be..f5a3ce207f2 100644 --- a/x-pack/metricbeat/cmd/root.go +++ b/x-pack/metricbeat/cmd/root.go @@ -16,9 +16,9 @@ import ( "github.com/elastic/beats/v7/metricbeat/beater" mbcmd "github.com/elastic/beats/v7/metricbeat/cmd" "github.com/elastic/beats/v7/metricbeat/cmd/test" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" // Register the includes. + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" _ "github.com/elastic/beats/v7/x-pack/metricbeat/include" // Import OSS modules. @@ -57,5 +57,4 @@ func init() { RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), settings) RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", mbcmd.BuildModulesManager)) RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", beater.DefaultTestModulesCreator())) - xpackcmd.AddXPack(RootCmd, Name) } diff --git a/x-pack/osquerybeat/cmd/root.go b/x-pack/osquerybeat/cmd/root.go index adaa112619f..c4548bac200 100644 --- a/x-pack/osquerybeat/cmd/root.go +++ b/x-pack/osquerybeat/cmd/root.go @@ -9,7 +9,8 @@ import ( cmd "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" + + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // Name of this beat @@ -24,6 +25,5 @@ func Osquerybeat() *cmd.BeatsRootCmd { } command := cmd.GenRootCmdWithSettings(beater.New, settings) - xpackcmd.AddXPack(command, Name) return command } diff --git a/x-pack/packetbeat/cmd/root.go b/x-pack/packetbeat/cmd/root.go index e5dd5bb7918..c7c15b058be 100644 --- a/x-pack/packetbeat/cmd/root.go +++ b/x-pack/packetbeat/cmd/root.go @@ -7,7 +7,8 @@ package cmd import ( "github.com/elastic/beats/v7/libbeat/cmd" packetbeatCmd "github.com/elastic/beats/v7/packetbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" + + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // Name of this beat. @@ -20,5 +21,4 @@ func init() { settings := packetbeatCmd.PacketbeatSettings() settings.ElasticLicensed = true RootCmd = packetbeatCmd.Initialize(settings) - xpackcmd.AddXPack(RootCmd, packetbeatCmd.Name) } diff --git a/x-pack/winlogbeat/cmd/root.go b/x-pack/winlogbeat/cmd/root.go index bc6cd4b0afd..ae56c4de83f 100644 --- a/x-pack/winlogbeat/cmd/root.go +++ b/x-pack/winlogbeat/cmd/root.go @@ -7,9 +7,9 @@ package cmd import ( "github.com/elastic/beats/v7/libbeat/cmd" winlogbeatCmd "github.com/elastic/beats/v7/winlogbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" // Register fields. + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" _ "github.com/elastic/beats/v7/x-pack/winlogbeat/include" ) @@ -23,5 +23,4 @@ func init() { settings := winlogbeatCmd.WinlogbeatSettings() settings.ElasticLicensed = true RootCmd = winlogbeatCmd.Initialize(settings) - xpackcmd.AddXPack(RootCmd, winlogbeatCmd.Name) } From 834dbffe6cad65b1453999850d6653ab3e6b4379 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Wed, 12 May 2021 10:51:52 -0700 Subject: [PATCH 2/8] Flatten directories so fleet code is directly in management. --- x-pack/libbeat/include/include.go | 2 +- .../libbeat/management/api/configuration.go | 60 -------------- x-pack/libbeat/management/api/doc.go | 70 ---------------- x-pack/libbeat/management/api/response.go | 53 ------------ x-pack/libbeat/management/blacklist.go | 5 +- x-pack/libbeat/management/blacklist_test.go | 43 +++++----- x-pack/libbeat/management/config.go | 77 +++++++++++++++-- .../configuration_test.go => config_test.go} | 2 +- x-pack/libbeat/management/fleet/config.go | 27 ------ .../libbeat/management/{fleet => }/manager.go | 83 +++++++++---------- .../management/{fleet => }/manager_test.go | 18 ++-- .../libbeat/management/{fleet => }/plugin.go | 11 ++- 12 files changed, 151 insertions(+), 300 deletions(-) delete mode 100644 x-pack/libbeat/management/api/configuration.go delete mode 100644 x-pack/libbeat/management/api/doc.go delete mode 100644 x-pack/libbeat/management/api/response.go rename x-pack/libbeat/management/{api/configuration_test.go => config_test.go} (99%) delete mode 100644 x-pack/libbeat/management/fleet/config.go rename x-pack/libbeat/management/{fleet => }/manager.go (77%) rename x-pack/libbeat/management/{fleet => }/manager_test.go (87%) rename x-pack/libbeat/management/{fleet => }/plugin.go (65%) diff --git a/x-pack/libbeat/include/include.go b/x-pack/libbeat/include/include.go index c82fc7b07af..bf61ff599d9 100644 --- a/x-pack/libbeat/include/include.go +++ b/x-pack/libbeat/include/include.go @@ -6,7 +6,7 @@ package include import ( // Register Fleet - _ "github.com/elastic/beats/v7/x-pack/libbeat/management/fleet" + _ "github.com/elastic/beats/v7/x-pack/libbeat/management" // register processors _ "github.com/elastic/beats/v7/x-pack/libbeat/processors/add_cloudfoundry_metadata" diff --git a/x-pack/libbeat/management/api/configuration.go b/x-pack/libbeat/management/api/configuration.go deleted file mode 100644 index 3ae1ef913d0..00000000000 --- a/x-pack/libbeat/management/api/configuration.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "github.com/mitchellh/hashstructure" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common/reload" - - "github.com/elastic/beats/v7/libbeat/common" -) - -// ConfigBlock stores a piece of config from central management -type ConfigBlock struct { - Raw map[string]interface{} -} - -// ConfigBlocksWithType is a list of config blocks with the same type -type ConfigBlocksWithType struct { - Type string - Blocks []*ConfigBlock -} - -// ConfigBlocks holds a list of type + configs objects -type ConfigBlocks []ConfigBlocksWithType - -// Config returns a common.Config object holding the config from this block -func (c *ConfigBlock) Config() (*common.Config, error) { - return common.NewConfigFrom(c.Raw) -} - -// ConfigWithMeta returns a reload.ConfigWithMeta object holding the config from this block, meta will be nil -func (c *ConfigBlock) ConfigWithMeta() (*reload.ConfigWithMeta, error) { - config, err := c.Config() - if err != nil { - return nil, err - } - return &reload.ConfigWithMeta{ - Config: config, - }, nil -} - -// ConfigBlocksEqual returns true if the given config blocks are equal, false if not -func ConfigBlocksEqual(a, b ConfigBlocks) (bool, error) { - // If there is an errors when hashing the config blocks its because the format changed. - aHash, err := hashstructure.Hash(a, nil) - if err != nil { - return false, errors.Wrap(err, "could not hash config blocks") - } - - bHash, err := hashstructure.Hash(b, nil) - if err != nil { - return false, errors.Wrap(err, "could not hash config blocks") - } - - return aHash == bHash, nil -} diff --git a/x-pack/libbeat/management/api/doc.go b/x-pack/libbeat/management/api/doc.go deleted file mode 100644 index c0a95acfc8c..00000000000 --- a/x-pack/libbeat/management/api/doc.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -/* -The Kibana CM Api returns a configuration format which cannot be ingested directly by our -configuration parser, it need to be transformed from the generic format into an adapted format -which is dependant on the type of configuration. - - -Translations: - -Type: output - -{ - "success": true, - "list": [ - - { - "config": { - "_sub_type": "elasticsearch" - "_id": "12312341231231" - "hosts": [ "localhost" ], - "password": "foobar" - "username": "elastic" - }, - "type": "output" - } - ] -} - -YAML representation: - -{ - "elasticsearch": { - "hosts": [ "localhost" ], - "password": "foobar" - "username": "elastic" - } -} - - -Type: *.modules - -{ - "success": true, - "list": [ - { - "config": { - "_sub_type": "system" - "_id": "12312341231231" - "path" "foobar" - }, - "type": "filebeat.module" - } - ] -} - -YAML representation: - -[ -{ - "module": "system" - "path": "foobar" -} -] - -*/ - -package api diff --git a/x-pack/libbeat/management/api/response.go b/x-pack/libbeat/management/api/response.go deleted file mode 100644 index cb28924bb25..00000000000 --- a/x-pack/libbeat/management/api/response.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import "fmt" - -// Action are the actions executed on the API. -type Action int - -// List of the valid Actions executed by the API. -//go:generate stringer -type=LicenseType -linecomment=true -const ( - Created Action = iota + 1 // created -) - -var mapStringToAction = map[string]Action{ - "created": Created, -} - -// UnmarshalJSON unmarshal an action string into a constant. -func (a *Action) UnmarshalJSON(b []byte) error { - k := string(b) - if len(b) <= 2 { - return fmt.Errorf( - "invalid string for action type, received: '%s'", - k, - ) - } - v, found := mapStringToAction[k[1:len(k)-1]] - if !found { - return fmt.Errorf( - "unknown action '%s' returned from the API, valid actions are: 'created'", - k, - ) - } - *a = v - return nil -} - -// BaseResponse the common response from all the API calls. -type BaseResponse struct { - Action Action `json:"action,omitempty"` - Success bool `json:"success"` - Error ErrorResponse `json:"error,omitempty"` -} - -// ErrorResponse contains human readable and machine readable information when an error happens. -type ErrorResponse struct { - Message string `json:"message"` - Code int `json:"code"` -} diff --git a/x-pack/libbeat/management/blacklist.go b/x-pack/libbeat/management/blacklist.go index 9e3326aa537..d344dad727d 100644 --- a/x-pack/libbeat/management/blacklist.go +++ b/x-pack/libbeat/management/blacklist.go @@ -12,7 +12,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/match" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" ) // ConfigBlacklist takes a ConfigBlocks object and filter it based on the given @@ -61,7 +60,7 @@ func NewConfigBlacklist(cfg ConfigBlacklistSettings) (*ConfigBlacklist, error) { } // Detect an error if any of the given config blocks is blacklisted -func (c *ConfigBlacklist) Detect(configBlocks api.ConfigBlocks) Errors { +func (c *ConfigBlacklist) Detect(configBlocks ConfigBlocks) Errors { var errs Errors for _, configs := range configBlocks { for _, block := range configs.Blocks { @@ -76,7 +75,7 @@ func (c *ConfigBlacklist) Detect(configBlocks api.ConfigBlocks) Errors { return errs } -func (c *ConfigBlacklist) isBlacklisted(blockType string, block *api.ConfigBlock) bool { +func (c *ConfigBlacklist) isBlacklisted(blockType string, block *ConfigBlock) bool { cfg, err := block.ConfigWithMeta() if err != nil { return false diff --git a/x-pack/libbeat/management/blacklist_test.go b/x-pack/libbeat/management/blacklist_test.go index 4bd2e36352e..016dd675179 100644 --- a/x-pack/libbeat/management/blacklist_test.go +++ b/x-pack/libbeat/management/blacklist_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" ) func TestConfigBlacklistSettingsUnpack(t *testing.T) { @@ -67,16 +66,16 @@ func TestConfigBlacklist(t *testing.T) { tests := []struct { name string patterns map[string]string - blocks api.ConfigBlocks + blocks ConfigBlocks blacklisted bool }{ { name: "No patterns", blacklisted: false, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "output", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "output": "console", @@ -92,10 +91,10 @@ func TestConfigBlacklist(t *testing.T) { patterns: map[string]string{ "output": "^console$", }, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "output", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "console": map[string]interface{}{ @@ -120,10 +119,10 @@ func TestConfigBlacklist(t *testing.T) { patterns: map[string]string{ "metricbeat.modules.module": "k.{8}s", }, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "metricbeat.modules", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "module": "kubernetes", @@ -140,10 +139,10 @@ func TestConfigBlacklist(t *testing.T) { patterns: map[string]string{ "metricbeat.modules.metricsets": "event", }, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "metricbeat.modules", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "module": "kubernetes", @@ -171,10 +170,10 @@ func TestConfigBlacklist(t *testing.T) { patterns: map[string]string{ "filebeat.inputs.containers.ids": "1ffeb0dbd13", }, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "metricbeat.modules", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "module": "kubernetes", @@ -186,9 +185,9 @@ func TestConfigBlacklist(t *testing.T) { }, }, }, - api.ConfigBlocksWithType{ + ConfigBlocksWithType{ Type: "filebeat.inputs", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "type": "docker", @@ -220,10 +219,10 @@ func TestConfigBlacklist(t *testing.T) { "list.of.elements": "forbidden", "list.of.elements.disallowed": "yes", }, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "list", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "of": map[string]interface{}{ diff --git a/x-pack/libbeat/management/config.go b/x-pack/libbeat/management/config.go index 740f28165fb..3fef422ecb1 100644 --- a/x-pack/libbeat/management/config.go +++ b/x-pack/libbeat/management/config.go @@ -4,10 +4,77 @@ package management -const ( - // ModeCentralManagement is a default CM mode, using existing processes. - ModeCentralManagement = "x-pack-cm" // TODO remove? +import ( + "github.com/mitchellh/hashstructure" + "github.com/pkg/errors" - // ModeFleet is a management mode where fleet is used to retrieve configurations. - ModeFleet = "x-pack-fleet" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/reload" ) + +// ModeFleet is a management mode where fleet is used to retrieve configurations. +const ModeFleet = "x-pack-fleet" + +// Config for central management +type Config struct { + Enabled bool `config:"enabled" yaml:"enabled"` + Mode string `config:"mode" yaml:"mode"` + Blacklist ConfigBlacklistSettings `config:"blacklist" yaml:"blacklist"` +} + +// ConfigBlock stores a piece of config from central management +type ConfigBlock struct { + Raw map[string]interface{} +} + +// ConfigBlocksWithType is a list of config blocks with the same type +type ConfigBlocksWithType struct { + Type string + Blocks []*ConfigBlock +} + +// ConfigBlocks holds a list of type + configs objects +type ConfigBlocks []ConfigBlocksWithType + +func defaultConfig() *Config { + return &Config{ + Mode: ModeFleet, + Blacklist: ConfigBlacklistSettings{ + Patterns: map[string]string{ + "output": "console|file", + }, + }, + } +} + +// Config returns a common.Config object holding the config from this block +func (c *ConfigBlock) Config() (*common.Config, error) { + return common.NewConfigFrom(c.Raw) +} + +// ConfigWithMeta returns a reload.ConfigWithMeta object holding the config from this block, meta will be nil +func (c *ConfigBlock) ConfigWithMeta() (*reload.ConfigWithMeta, error) { + config, err := c.Config() + if err != nil { + return nil, err + } + return &reload.ConfigWithMeta{ + Config: config, + }, nil +} + +// ConfigBlocksEqual returns true if the given config blocks are equal, false if not +func ConfigBlocksEqual(a, b ConfigBlocks) (bool, error) { + // If there is an errors when hashing the config blocks its because the format changed. + aHash, err := hashstructure.Hash(a, nil) + if err != nil { + return false, errors.Wrap(err, "could not hash config blocks") + } + + bHash, err := hashstructure.Hash(b, nil) + if err != nil { + return false, errors.Wrap(err, "could not hash config blocks") + } + + return aHash == bHash, nil +} diff --git a/x-pack/libbeat/management/api/configuration_test.go b/x-pack/libbeat/management/config_test.go similarity index 99% rename from x-pack/libbeat/management/api/configuration_test.go rename to x-pack/libbeat/management/config_test.go index aaee9f80955..647cdd32026 100644 --- a/x-pack/libbeat/management/api/configuration_test.go +++ b/x-pack/libbeat/management/config_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package api +package management import ( "testing" diff --git a/x-pack/libbeat/management/fleet/config.go b/x-pack/libbeat/management/fleet/config.go deleted file mode 100644 index 3ab745ef069..00000000000 --- a/x-pack/libbeat/management/fleet/config.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package fleet - -import ( - xmanagement "github.com/elastic/beats/v7/x-pack/libbeat/management" -) - -// Config for central management -type Config struct { - Enabled bool `config:"enabled" yaml:"enabled"` - Mode string `config:"mode" yaml:"mode"` - Blacklist xmanagement.ConfigBlacklistSettings `config:"blacklist" yaml:"blacklist"` -} - -func defaultConfig() *Config { - return &Config{ - Mode: xmanagement.ModeCentralManagement, // TODO Fleet mode? - Blacklist: xmanagement.ConfigBlacklistSettings{ - Patterns: map[string]string{ - "output": "console|file", - }, - }, - } -} diff --git a/x-pack/libbeat/management/fleet/manager.go b/x-pack/libbeat/management/manager.go similarity index 77% rename from x-pack/libbeat/management/fleet/manager.go rename to x-pack/libbeat/management/manager.go index 8ec84c10579..57d6eef656b 100644 --- a/x-pack/libbeat/management/fleet/manager.go +++ b/x-pack/libbeat/management/manager.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package fleet +package management import ( "context" @@ -21,10 +21,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/libbeat/management" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" - - xmanagement "github.com/elastic/beats/v7/x-pack/libbeat/management" + lbmanagement "github.com/elastic/beats/v7/libbeat/management" ) // Manager handles internal config updates. By retrieving @@ -34,10 +31,10 @@ type Manager struct { logger *logp.Logger beatUUID uuid.UUID registry *reload.Registry - blacklist *xmanagement.ConfigBlacklist + blacklist *ConfigBlacklist client client.Client lock sync.Mutex - status management.Status + status lbmanagement.Status msg string payload map[string]interface{} @@ -45,7 +42,7 @@ type Manager struct { } // NewFleetManager returns a X-Pack Beats Fleet Management manager. -func NewFleetManager(config *common.Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { +func NewFleetManager(config *common.Config, registry *reload.Registry, beatUUID uuid.UUID) (lbmanagement.Manager, error) { c := defaultConfig() if config.Enabled() { if err := config.Unpack(&c); err != nil { @@ -56,8 +53,8 @@ func NewFleetManager(config *common.Config, registry *reload.Registry, beatUUID } // NewFleetManagerWithConfig returns a X-Pack Beats Fleet Management manager. -func NewFleetManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { - log := logp.NewLogger(management.DebugK) +func NewFleetManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (lbmanagement.Manager, error) { + log := logp.NewLogger(lbmanagement.DebugK) m := &Manager{ config: c, @@ -67,11 +64,11 @@ func NewFleetManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uu } var err error - var blacklist *xmanagement.ConfigBlacklist + var blacklist *ConfigBlacklist var eac client.Client - if c.Enabled && c.Mode == xmanagement.ModeFleet { + if c.Enabled && c.Mode == ModeFleet { // Initialize configs blacklist - blacklist, err = xmanagement.NewConfigBlacklist(c.Blacklist) + blacklist, err = NewConfigBlacklist(c.Blacklist) if err != nil { return nil, errors.Wrap(err, "wrong settings for configurations blacklist") } @@ -90,7 +87,7 @@ func NewFleetManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uu // Enabled returns true if config management is enabled. func (cm *Manager) Enabled() bool { - return cm.config.Enabled && cm.config.Mode == xmanagement.ModeFleet + return cm.config.Enabled && cm.config.Mode == ModeFleet } // Start the config manager @@ -128,7 +125,7 @@ func (cm *Manager) CheckRawConfig(cfg *common.Config) error { } // UpdateStatus updates the manager with the current status for the beat. -func (cm *Manager) UpdateStatus(status management.Status, msg string) { +func (cm *Manager) UpdateStatus(status lbmanagement.Status, msg string) { cm.lock.Lock() defer cm.lock.Unlock() @@ -141,14 +138,14 @@ func (cm *Manager) UpdateStatus(status management.Status, msg string) { } func (cm *Manager) OnConfig(s string) { - cm.UpdateStatus(management.Configuring, "Updating configuration") + cm.UpdateStatus(lbmanagement.Configuring, "Updating configuration") var configMap common.MapStr uconfig, err := common.NewConfigFrom(s) if err != nil { err = errors.Wrap(err, "config blocks unsuccessfully generated") cm.logger.Error(err) - cm.UpdateStatus(management.Failed, err.Error()) + cm.UpdateStatus(lbmanagement.Failed, err.Error()) return } @@ -156,7 +153,7 @@ func (cm *Manager) OnConfig(s string) { if err != nil { err = errors.Wrap(err, "config blocks unsuccessfully generated") cm.logger.Error(err) - cm.UpdateStatus(management.Failed, err.Error()) + cm.UpdateStatus(lbmanagement.Failed, err.Error()) return } @@ -164,13 +161,13 @@ func (cm *Manager) OnConfig(s string) { if err != nil { err = errors.Wrap(err, "failed to parse configuration") cm.logger.Error(err) - cm.UpdateStatus(management.Failed, err.Error()) + cm.UpdateStatus(lbmanagement.Failed, err.Error()) return } if errs := cm.apply(blocks); !errs.IsEmpty() { // `cm.apply` already logs the errors; currently allow beat to run degraded - cm.UpdateStatus(management.Failed, errs.Error()) + cm.UpdateStatus(lbmanagement.Failed, errs.Error()) return } @@ -202,8 +199,8 @@ func (cm *Manager) OnError(err error) { cm.logger.Errorf("elastic-agent-client got error: %s", err) } -func (cm *Manager) apply(blocks api.ConfigBlocks) xmanagement.Errors { - var errors xmanagement.Errors +func (cm *Manager) apply(blocks ConfigBlocks) Errors { + var errors Errors missing := map[string]bool{} for _, name := range cm.registry.GetRegisteredNames() { missing[name] = true @@ -226,7 +223,7 @@ func (cm *Manager) apply(blocks api.ConfigBlocks) xmanagement.Errors { // Unset missing configs for name := range missing { if missing[name] { - if err := cm.reload(name, []*api.ConfigBlock{}); err != nil { + if err := cm.reload(name, []*ConfigBlock{}); err != nil { errors = append(errors, err) } } @@ -235,14 +232,14 @@ func (cm *Manager) apply(blocks api.ConfigBlocks) xmanagement.Errors { return errors } -func (cm *Manager) reload(t string, blocks []*api.ConfigBlock) *xmanagement.Error { +func (cm *Manager) reload(t string, blocks []*ConfigBlock) *Error { cm.logger.Infof("Applying settings for %s", t) if obj := cm.registry.GetReloadable(t); obj != nil { // Single object if len(blocks) > 1 { err := fmt.Errorf("got an invalid number of configs for %s: %d, expected: 1", t, len(blocks)) cm.logger.Error(err) - return xmanagement.NewConfigError(err) + return NewConfigError(err) } var config *reload.ConfigWithMeta @@ -251,13 +248,13 @@ func (cm *Manager) reload(t string, blocks []*api.ConfigBlock) *xmanagement.Erro config, err = blocks[0].ConfigWithMeta() if err != nil { cm.logger.Error(err) - return xmanagement.NewConfigError(err) + return NewConfigError(err) } } if err := obj.Reload(config); err != nil { cm.logger.Error(err) - return xmanagement.NewConfigError(err) + return NewConfigError(err) } } else if obj := cm.registry.GetReloadableList(t); obj != nil { // List @@ -266,22 +263,22 @@ func (cm *Manager) reload(t string, blocks []*api.ConfigBlock) *xmanagement.Erro config, err := block.ConfigWithMeta() if err != nil { cm.logger.Error(err) - return xmanagement.NewConfigError(err) + return NewConfigError(err) } configs = append(configs, config) } if err := obj.Reload(configs); err != nil { cm.logger.Error(err) - return xmanagement.NewConfigError(err) + return NewConfigError(err) } } return nil } -func (cm *Manager) toConfigBlocks(cfg common.MapStr) (api.ConfigBlocks, error) { - blocks := map[string][]*api.ConfigBlock{} +func (cm *Manager) toConfigBlocks(cfg common.MapStr) (ConfigBlocks, error) { + blocks := map[string][]*ConfigBlock{} // Extract all registered values beat can respond to for _, regName := range cm.registry.GetRegisteredNames() { @@ -291,11 +288,11 @@ func (cm *Manager) toConfigBlocks(cfg common.MapStr) (api.ConfigBlocks, error) { } if mapBlock, ok := iBlock.(map[string]interface{}); ok { - blocks[regName] = append(blocks[regName], &api.ConfigBlock{Raw: mapBlock}) + blocks[regName] = append(blocks[regName], &ConfigBlock{Raw: mapBlock}) } else if arrayBlock, ok := iBlock.([]interface{}); ok { for _, item := range arrayBlock { if mapBlock, ok := item.(map[string]interface{}); ok { - blocks[regName] = append(blocks[regName], &api.ConfigBlock{Raw: mapBlock}) + blocks[regName] = append(blocks[regName], &ConfigBlock{Raw: mapBlock}) } } } @@ -308,31 +305,31 @@ func (cm *Manager) toConfigBlocks(cfg common.MapStr) (api.ConfigBlocks, error) { } sort.Strings(keys) - res := api.ConfigBlocks{} + res := ConfigBlocks{} for _, t := range keys { b := blocks[t] - res = append(res, api.ConfigBlocksWithType{Type: t, Blocks: b}) + res = append(res, ConfigBlocksWithType{Type: t, Blocks: b}) } return res, nil } -func statusToProtoStatus(status management.Status) proto.StateObserved_Status { +func statusToProtoStatus(status lbmanagement.Status) proto.StateObserved_Status { switch status { - case management.Unknown: + case lbmanagement.Unknown: // unknown is reported as healthy, as the status is unknown return proto.StateObserved_HEALTHY - case management.Starting: + case lbmanagement.Starting: return proto.StateObserved_STARTING - case management.Configuring: + case lbmanagement.Configuring: return proto.StateObserved_CONFIGURING - case management.Running: + case lbmanagement.Running: return proto.StateObserved_HEALTHY - case management.Degraded: + case lbmanagement.Degraded: return proto.StateObserved_DEGRADED - case management.Failed: + case lbmanagement.Failed: return proto.StateObserved_FAILED - case management.Stopping: + case lbmanagement.Stopping: return proto.StateObserved_STOPPING } // unknown status, still reported as healthy diff --git a/x-pack/libbeat/management/fleet/manager_test.go b/x-pack/libbeat/management/manager_test.go similarity index 87% rename from x-pack/libbeat/management/fleet/manager_test.go rename to x-pack/libbeat/management/manager_test.go index 7810886018a..113c19efe5c 100644 --- a/x-pack/libbeat/management/fleet/manager_test.go +++ b/x-pack/libbeat/management/manager_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package fleet +package management import ( "testing" @@ -13,7 +13,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/elastic/beats/v7/libbeat/management" + lbmanagement "github.com/elastic/beats/v7/libbeat/management" ) func TestConfigBlocks(t *testing.T) { @@ -58,13 +58,13 @@ output: } func TestStatusToProtoStatus(t *testing.T) { - assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(management.Unknown)) - assert.Equal(t, proto.StateObserved_STARTING, statusToProtoStatus(management.Starting)) - assert.Equal(t, proto.StateObserved_CONFIGURING, statusToProtoStatus(management.Configuring)) - assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(management.Running)) - assert.Equal(t, proto.StateObserved_DEGRADED, statusToProtoStatus(management.Degraded)) - assert.Equal(t, proto.StateObserved_FAILED, statusToProtoStatus(management.Failed)) - assert.Equal(t, proto.StateObserved_STOPPING, statusToProtoStatus(management.Stopping)) + assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(lbmanagement.Unknown)) + assert.Equal(t, proto.StateObserved_STARTING, statusToProtoStatus(lbmanagement.Starting)) + assert.Equal(t, proto.StateObserved_CONFIGURING, statusToProtoStatus(lbmanagement.Configuring)) + assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(lbmanagement.Running)) + assert.Equal(t, proto.StateObserved_DEGRADED, statusToProtoStatus(lbmanagement.Degraded)) + assert.Equal(t, proto.StateObserved_FAILED, statusToProtoStatus(lbmanagement.Failed)) + assert.Equal(t, proto.StateObserved_STOPPING, statusToProtoStatus(lbmanagement.Stopping)) } type dummyReloadable struct{} diff --git a/x-pack/libbeat/management/fleet/plugin.go b/x-pack/libbeat/management/plugin.go similarity index 65% rename from x-pack/libbeat/management/fleet/plugin.go rename to x-pack/libbeat/management/plugin.go index edb12ca78f1..cd411a85767 100644 --- a/x-pack/libbeat/management/fleet/plugin.go +++ b/x-pack/libbeat/management/plugin.go @@ -2,28 +2,27 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package fleet +package management import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/feature" - "github.com/elastic/beats/v7/libbeat/management" - xmanagement "github.com/elastic/beats/v7/x-pack/libbeat/management" + lbmanagement "github.com/elastic/beats/v7/libbeat/management" ) func init() { - management.Register("x-pack-fleet", NewFleetManagerPlugin, feature.Beta) + lbmanagement.Register("x-pack-fleet", NewFleetManagerPlugin, feature.Beta) } // NewFleetManagerPlugin creates a plugin function returning factory if configuration matches the criteria -func NewFleetManagerPlugin(config *common.Config) management.FactoryFunc { +func NewFleetManagerPlugin(config *common.Config) lbmanagement.FactoryFunc { c := defaultConfig() if config.Enabled() { if err := config.Unpack(&c); err != nil { return nil } - if c.Mode == xmanagement.ModeFleet { + if c.Mode == ModeFleet { return NewFleetManager } } From 2a84b53dc84d8e5a474ec3695dee05ac105b23df Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Thu, 13 May 2021 10:16:30 -0700 Subject: [PATCH 3/8] Remove central management python integ tests --- x-pack/libbeat/tests/system/base.py | 14 - .../tests/system/config/mockbeat.yml.j2 | 93 ------ x-pack/libbeat/tests/system/requirements.txt | 0 .../libbeat/tests/system/test_management.py | 291 ------------------ 4 files changed, 398 deletions(-) delete mode 100644 x-pack/libbeat/tests/system/base.py delete mode 100644 x-pack/libbeat/tests/system/config/mockbeat.yml.j2 delete mode 100644 x-pack/libbeat/tests/system/requirements.txt delete mode 100644 x-pack/libbeat/tests/system/test_management.py diff --git a/x-pack/libbeat/tests/system/base.py b/x-pack/libbeat/tests/system/base.py deleted file mode 100644 index ac2fdb84858..00000000000 --- a/x-pack/libbeat/tests/system/base.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys -import os -from beat.beat import TestCase - - -class BaseTest(TestCase): - - @classmethod - def setUpClass(self): - self.beat_name = "mockbeat" - self.beat_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), "../../")) - self.test_binary = self.beat_path + "/libbeat.test" - super(BaseTest, self).setUpClass() diff --git a/x-pack/libbeat/tests/system/config/mockbeat.yml.j2 b/x-pack/libbeat/tests/system/config/mockbeat.yml.j2 deleted file mode 100644 index 134545b9970..00000000000 --- a/x-pack/libbeat/tests/system/config/mockbeat.yml.j2 +++ /dev/null @@ -1,93 +0,0 @@ -############################# Mockbeat ###################################### -mockbeat: -############################# General ############################################ - -# The name of the shipper that publishes the network data. It can be used to group -# all the transactions sent by a single shipper in the web interface. -# If this options is not defined, the hostname is used. -name: - -# The tags of the shipper are included in their own field with each -# transaction published. Tags make it easy to group servers by different -# logical properties. -tags: [ -{%- if agent_tags -%} - {%- for tag in agent_tags -%} - "{{ tag }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} -{%- endif -%}] - -#================================ Queue ===================================== - -queue.mem: - events: 4096 - flush.min_events: 8 - flush.timeout: 0.1s - -############################# Output ############################################ - -# Configure what outputs to use when sending the data collected by mockbeat. -# Multiple outputs may NOT be enabled. -output: - {% if console -%} - console: - {% for k, v in console.items() -%} - {{ k }}: {{ v }} - {% endfor -%} - {%- endif %} - - {% if elasticsearch -%} - elasticsearch: - {% for k, v in elasticsearch.items() -%} - {{ k }}: {{ v }} - {% endfor -%} - {%- endif %} - - # Redis as output - # Options: - # host, port: where Redis is listening on - #redis: - # host: localhost - # port: 6379 - - # File as output - # Options - # path: where to save the files - # filename: name of the files - # rotate_every_kb: maximum size of the files in path - # number of files: maximum number of files in path - {% if not (console or elasticsearch) -%} - file: - path: {{ output_file_path|default(beat.working_dir + "/output") }} - filename: "{{ output_file_filename|default("mockbeat") }}" - rotate_every_kb: 1000 - #number_of_files: 7 - {%- endif %} - -#==================== Elasticsearch template setting ========================== -setup.template: - fields: {{ fields|default("fields.yml") }} - settings: - index.number_of_shards: 1 - index.codec: best_compression - name: {{ es_template_name }} - pattern: {{ es_template_pattern }} - overwrite: {{ template_overwrite|default("false") }} - json: - enabled: {{ template_json_enabled|default("false") }} - path: {{ template_json_path }} - name: {{ template_json_name }} - -#================================ Logging ===================================== - -{% if metrics_period -%} -logging.metrics.period: {{ metrics_period }} -{%- endif %} - -{% if keystore_path %} -#================================ keystore ===================================== -keystore.path: {{keystore_path}} -{% endif %} - -# vim: set ft=jinja: diff --git a/x-pack/libbeat/tests/system/requirements.txt b/x-pack/libbeat/tests/system/requirements.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/x-pack/libbeat/tests/system/test_management.py b/x-pack/libbeat/tests/system/test_management.py deleted file mode 100644 index 35a4f9949f9..00000000000 --- a/x-pack/libbeat/tests/system/test_management.py +++ /dev/null @@ -1,291 +0,0 @@ -import sys -import os -import glob -import json -import requests -import string -import random -import unittest -import time -from elasticsearch import Elasticsearch -from os import path - - -from base import BaseTest - - -# Disable because waiting artifacts from https://github.com/elastic/kibana/pull/31660 -INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) -# INTEGRATION_TESTS = False -TIMEOUT = 2 * 60 - - -class TestManagement(BaseTest): - - def setUp(self): - super(TestManagement, self).setUp() - # NOTES: Theses options are linked to the specific of the docker compose environment for - # CM. - self.es_host = os.getenv('ES_HOST', 'localhost') + ":" + os.getenv('ES_POST', '9200') - self.es_user = "myelastic" - self.es_pass = "changeme" - self.es = Elasticsearch([self.get_elasticsearch_url()], verify_certs=True) - self.keystore_path = self.working_dir + "/data/keystore" - - if path.exists(self.keystore_path): - os.Remove(self.keystore_path) - - @unittest.skipIf(not INTEGRATION_TESTS, - "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") - def test_enroll(self): - """ - Enroll the beat in Kibana Central Management - """ - - assert len(glob.glob(os.path.join(self.working_dir, "mockbeat.yml.*.bak"))) == 0 - - # We don't care about this as it will be replaced by enrollment - # process: - config_path = os.path.join(self.working_dir, "mockbeat.yml") - self.render_config_template("mockbeat", config_path, keystore_path=self.keystore_path) - - config_content = open(config_path, 'r').read() - - exit_code = self.enroll(self.es_user, self.es_pass) - - assert exit_code == 0 - - assert self.log_contains("Enrolled and ready to retrieve settings") - - # Enroll creates a keystore (to store access token) - assert os.path.isfile(os.path.join( - self.working_dir, "data/keystore")) - - # New settings file is in place now - new_content = open(config_path, 'r').read() - assert config_content != new_content - - # Settings backup has been created - backup_file = glob.glob(os.path.join(self.working_dir, "mockbeat.yml.*.bak"))[0] - assert os.path.isfile(backup_file) - backup_content = open(backup_file).read() - assert config_content == backup_content - - @unittest.skipIf(not INTEGRATION_TESTS, - "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") - def test_enroll_bad_pw(self): - """ - Try to enroll the beat in Kibana Central Management with a bad password - """ - # We don't care about this as it will be replaced by enrollment - # process: - config_path = os.path.join(self.working_dir, "mockbeat.yml") - self.render_config_template("mockbeat", config_path, keystore_path=self.keystore_path) - - config_content = open(config_path, 'r').read() - - exit_code = self.enroll("not", 'wrong password') - - assert exit_code == 1 - - # Keystore wasn't created - assert not os.path.isfile(os.path.join( - self.working_dir, "data/keystore")) - - # Settings hasn't changed - new_content = open(config_path, 'r').read() - assert config_content == new_content - - @unittest.skipIf(not INTEGRATION_TESTS, - "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") - def test_fetch_configs(self): - """ - Config is retrieved from Central Management and updates are applied - """ - # Enroll the beat - config_path = os.path.join(self.working_dir, "mockbeat.yml") - self.render_config_template("mockbeat", config_path, keystore_path=self.keystore_path) - exit_code = self.enroll(self.es_user, self.es_pass) - assert exit_code == 0 - - index = self.random_index() - # Configure an output - self.create_and_assing_tag([ - { - "type": "output", - "config": { - "_sub_type": "elasticsearch", - "hosts": [self.es_host], - "username": self.es_user, - "password": self.es_pass, - "index": index, - }, - "id": "myconfig", - } - ]) - - # Start beat - proc = self.start_beat(extra_args=[ - "-E", "management.period=1s", - "-E", "keystore.path=%s" % self.keystore_path, - ]) - - # Wait for beat to apply new conf - self.wait_log_contains("Applying settings for output") - - self.wait_until(lambda: self.log_contains("PublishEvents: "), max_timeout=TIMEOUT) - - self.wait_documents(index, 1) - - index2 = self.random_index() - - # Update output configuration - self.create_and_assing_tag([ - { - "type": "output", - "config": { - "_sub_type": "elasticsearch", - "hosts": [self.es_host], - "username": self.es_user, - "password": self.es_pass, - "index": index2, - }, - "id": "myconfig", - } - ]) - self.wait_log_contains("Applying settings for output") - self.wait_until(lambda: self.log_contains("PublishEvents: "), max_timeout=TIMEOUT) - self.wait_documents(index2, 1) - - proc.check_kill_and_wait() - - @unittest.skipIf(not INTEGRATION_TESTS, - "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") - def test_configs_cache(self): - """ - Config cache is used if Kibana is not available - """ - # Enroll the beat - config_path = os.path.join(self.working_dir, "mockbeat.yml") - self.render_config_template("mockbeat", config_path, keystore_path=self.keystore_path) - exit_code = self.enroll(self.es_user, self.es_pass) - assert exit_code == 0 - - index = self.random_index() - - # Update output configuration - self.create_and_assing_tag([ - { - "type": "output", - "config": { - "_sub_type": "elasticsearch", - "hosts": [self.es_host], - "username": self.es_user, - "password": self.es_pass, - "index": index, - } - } - ]) - - # Start beat - proc = self.start_beat(extra_args=[ - "-E", "management.period=1s", - "-E", "keystore.path=%s" % self.keystore_path, - ]) - - self.wait_until(lambda: self.log_contains("PublishEvents: "), max_timeout=TIMEOUT) - self.wait_documents(index, 1) - proc.check_kill_and_wait() - - # Remove the index - self.es.indices.delete(index) - - # Cache should exists already, start with wrong kibana settings: - proc = self.start_beat(extra_args=[ - "-E", "management.period=1s", - "-E", "management.kibana.host=wronghost", - "-E", "management.kibana.timeout=0.5s", - "-E", "keystore.path=%s" % self.keystore_path, - ]) - - self.wait_until(lambda: self.log_contains("PublishEvents: "), max_timeout=TIMEOUT) - self.wait_documents(index, 1) - proc.check_kill_and_wait() - - def enroll(self, user, password): - return self.run_beat( - extra_args=["enroll", self.get_kibana_url(), - "--password", "env:PASS", "--username", user, "--force"], - logging_args=["-v", "-d", "*"], - env={ - 'PASS': password, - }) - - def check_kibana_status(self): - headers = { - "kbn-xsrf": "1" - } - - # Create tag - url = self.get_kibana_url() + "/api/status" - - r = requests.get(url, headers=headers, - auth=(self.es_user, self.es_pass)) - - def create_and_assing_tag(self, blocks): - tag_name = "test%d" % int(time.time() * 1000) - headers = { - "kbn-xsrf": "1" - } - - # Create tag - url = self.get_kibana_url() + "/api/beats/tag/" + tag_name - data = { - "color": "#DD0A73", - "name": tag_name, - } - - r = requests.put(url, json=data, headers=headers, - auth=(self.es_user, self.es_pass)) - assert r.status_code in (200, 201) - - # Create blocks - url = self.get_kibana_url() + "/api/beats/configurations" - for b in blocks: - b["tag"] = tag_name - - r = requests.put(url, json=blocks, headers=headers, - auth=(self.es_user, self.es_pass)) - assert r.status_code in (200, 201) - - # Retrieve beat ID - meta = json.loads( - open(os.path.join(self.working_dir, 'data', 'meta.json'), 'r').read()) - - # Assign tag to beat - data = {"assignments": [{"beatId": meta["uuid"], "tag": tag_name}]} - url = self.get_kibana_url() + "/api/beats/agents_tags/assignments" - r = requests.post(url, json=data, headers=headers, - auth=(self.es_user, self.es_pass)) - - assert r.status_code == 200 - - def get_elasticsearch_url(self): - return 'http://' + self.es_user + ":" + self.es_pass + '@' + \ - os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '5601') - - def get_kibana_url(self): - return 'http://' + os.getenv('KIBANA_HOST', 'kibana') + ':' + os.getenv('KIBANA_PORT', '5601') - - def random_index(self): - return ''.join(random.choice(string.ascii_lowercase) for i in range(10)) - - def check_document_count(self, index, count): - try: - self.es.indices.refresh(index=index) - return self.es.search(index=index, body={"query": {"match_all": {}}})['hits']['total']['value'] >= count - except BaseException: - return False - - def wait_documents(self, index, count): - self.wait_until(lambda: self.check_document_count(index, count), max_timeout=TIMEOUT, poll_interval=1) From 8c10a8ce24af702c3dff046a526aa773fc73904d Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Tue, 18 May 2021 16:07:13 -0700 Subject: [PATCH 4/8] Add CHANGELOG --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2809b5e2752..25ae062d200 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -35,6 +35,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Set `cleanup_timeout` to zero by default in docker and kubernetes autodiscover in all beats except Filebeat where it is kept to 60 seconds. {pull}24681[24681] - Update to ECS 1.9.0. {pull}24909[24909] - Remove id_field_data {pull}25239[25239] +- Removed beats central management {pull}25696[25696], {issue}23908[23908] *Auditbeat* From f235c594b76c76d40ef46be763f0373d83de167b Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Wed, 19 May 2021 09:42:55 -0700 Subject: [PATCH 5/8] Remove mode attribute and unused code --- x-pack/libbeat/management/config.go | 24 ---- x-pack/libbeat/management/config_test.go | 153 ----------------------- x-pack/libbeat/management/error.go | 39 ------ x-pack/libbeat/management/error_test.go | 60 --------- x-pack/libbeat/management/manager.go | 4 +- x-pack/libbeat/management/plugin.go | 5 +- 6 files changed, 3 insertions(+), 282 deletions(-) delete mode 100644 x-pack/libbeat/management/config_test.go diff --git a/x-pack/libbeat/management/config.go b/x-pack/libbeat/management/config.go index 3fef422ecb1..cf285d5389d 100644 --- a/x-pack/libbeat/management/config.go +++ b/x-pack/libbeat/management/config.go @@ -5,20 +5,13 @@ package management import ( - "github.com/mitchellh/hashstructure" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/reload" ) -// ModeFleet is a management mode where fleet is used to retrieve configurations. -const ModeFleet = "x-pack-fleet" - // Config for central management type Config struct { Enabled bool `config:"enabled" yaml:"enabled"` - Mode string `config:"mode" yaml:"mode"` Blacklist ConfigBlacklistSettings `config:"blacklist" yaml:"blacklist"` } @@ -38,7 +31,6 @@ type ConfigBlocks []ConfigBlocksWithType func defaultConfig() *Config { return &Config{ - Mode: ModeFleet, Blacklist: ConfigBlacklistSettings{ Patterns: map[string]string{ "output": "console|file", @@ -62,19 +54,3 @@ func (c *ConfigBlock) ConfigWithMeta() (*reload.ConfigWithMeta, error) { Config: config, }, nil } - -// ConfigBlocksEqual returns true if the given config blocks are equal, false if not -func ConfigBlocksEqual(a, b ConfigBlocks) (bool, error) { - // If there is an errors when hashing the config blocks its because the format changed. - aHash, err := hashstructure.Hash(a, nil) - if err != nil { - return false, errors.Wrap(err, "could not hash config blocks") - } - - bHash, err := hashstructure.Hash(b, nil) - if err != nil { - return false, errors.Wrap(err, "could not hash config blocks") - } - - return aHash == bHash, nil -} diff --git a/x-pack/libbeat/management/config_test.go b/x-pack/libbeat/management/config_test.go deleted file mode 100644 index 647cdd32026..00000000000 --- a/x-pack/libbeat/management/config_test.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestConfigBlocksEqual(t *testing.T) { - tests := []struct { - name string - a, b ConfigBlocks - equal bool - }{ - { - name: "empty lists or nil", - a: nil, - b: ConfigBlocks{}, - equal: true, - }, - { - name: "single element", - a: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": "bar", - }, - }, - }, - }, - }, - b: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": "bar", - }, - }, - }, - }, - }, - equal: true, - }, - { - name: "single element with slices", - a: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": []string{"foo", "bar"}, - }, - }, - }, - }, - }, - b: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": []string{"foo", "bar"}, - }, - }, - }, - }, - }, - equal: true, - }, - { - name: "different number of blocks", - a: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": "bar", - }, - }, - &ConfigBlock{ - Raw: map[string]interface{}{ - "baz": "buzz", - }, - }, - }, - }, - }, - b: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": "bar", - }, - }, - }, - }, - }, - equal: false, - }, - { - name: "different block", - a: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "baz": "buzz", - }, - }, - }, - }, - }, - b: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": "bar", - }, - }, - }, - }, - }, - equal: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - check, err := ConfigBlocksEqual(test.a, test.b) - if !assert.NoError(t, err) { - return - } - assert.Equal(t, test.equal, check) - }) - } -} diff --git a/x-pack/libbeat/management/error.go b/x-pack/libbeat/management/error.go index b70aa5387b3..1d330a5938f 100644 --- a/x-pack/libbeat/management/error.go +++ b/x-pack/libbeat/management/error.go @@ -5,12 +5,10 @@ package management import ( - "encoding/json" "strconv" "strings" "github.com/gofrs/uuid" - "github.com/pkg/errors" ) // ErrorType is type of error that the events endpoint understand. @@ -26,43 +24,6 @@ type Error struct { Err error } -// MarshalJSON transform an error into a JSON document. -func (e *Error) MarshalJSON() ([]byte, error) { - return json.Marshal(&struct { - UUID string `json:"uuid"` - Type string `json:"type"` - Message string `json:"message"` - }{ - UUID: e.UUID.String(), - Type: string(e.Type), - Message: e.Err.Error(), - }) -} - -// UnmarshalJSON unmarshals a event of the type Error. -func (e *Error) UnmarshalJSON(b []byte) error { - res := &struct { - UUID string `json:"uuid,omitempty"` - Type string `json:"type"` - Message string `json:"message"` - }{} - - if err := json.Unmarshal(b, res); err != nil { - return err - } - - uuid, err := uuid.FromString(res.UUID) - if err != nil { - return err - } - *e = Error{ - Type: ErrorType(res.Type), - UUID: uuid, - Err: errors.New(res.Message), - } - return nil -} - func (e *Error) Error() string { return e.Err.Error() } diff --git a/x-pack/libbeat/management/error_test.go b/x-pack/libbeat/management/error_test.go index 4c0d6beb141..ee85b4d78bd 100644 --- a/x-pack/libbeat/management/error_test.go +++ b/x-pack/libbeat/management/error_test.go @@ -5,72 +5,12 @@ package management import ( - "encoding/json" "testing" - "github.com/gofrs/uuid" "github.com/pkg/errors" "github.com/stretchr/testify/assert" ) -// Ensure that all events have a Message key that can by used by the GUI. -func ensureJSONhasGeneralfield(t *testing.T, obj json.Marshaler) func(*testing.T) { - return func(t *testing.T) { - serialized, err := json.Marshal(obj) - if !assert.NoError(t, err) { - return - } - - message := struct { - Message string `json:"message"` - }{} - - err = json.Unmarshal(serialized, &message) - - if !assert.NoError(t, err) { - return - } - assert.NotEmpty(t, message) - } -} - -func TestErrorSerialization(t *testing.T) { - id, _ := uuid.NewV4() - t.Run("serialize ok", func(t *testing.T) { - e := Error{ - Type: ConfigError, - Err: errors.New("hello world"), - UUID: id, - } - - b, err := json.Marshal(&e) - if assert.NoError(t, err) { - return - } - - resp := &struct { - UUID string `json:"uuid"` - Message string `json:"message"` - Type string `json:"type"` - }{} - - err = json.Unmarshal(b, resp) - if assert.NoError(t, err) { - return - } - - assert.Equal(t, e.UUID.String(), resp.UUID) - assert.Equal(t, e.Err.Error(), resp.Message) - assert.Equal(t, e.Type, resp.Type) - }) - - t.Run("ensure that json general fields are present", ensureJSONhasGeneralfield(t, &Error{ - Type: ConfigError, - Err: errors.New("hello world"), - UUID: id, - })) -} - func TestErrors(t *testing.T) { t.Run("single error", func(t *testing.T) { errors := Errors{NewConfigError(errors.New("error1"))} diff --git a/x-pack/libbeat/management/manager.go b/x-pack/libbeat/management/manager.go index 57d6eef656b..9fa96bbc55e 100644 --- a/x-pack/libbeat/management/manager.go +++ b/x-pack/libbeat/management/manager.go @@ -66,7 +66,7 @@ func NewFleetManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uu var err error var blacklist *ConfigBlacklist var eac client.Client - if c.Enabled && c.Mode == ModeFleet { + if c.Enabled { // Initialize configs blacklist blacklist, err = NewConfigBlacklist(c.Blacklist) if err != nil { @@ -87,7 +87,7 @@ func NewFleetManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uu // Enabled returns true if config management is enabled. func (cm *Manager) Enabled() bool { - return cm.config.Enabled && cm.config.Mode == ModeFleet + return cm.config.Enabled } // Start the config manager diff --git a/x-pack/libbeat/management/plugin.go b/x-pack/libbeat/management/plugin.go index cd411a85767..34560513f93 100644 --- a/x-pack/libbeat/management/plugin.go +++ b/x-pack/libbeat/management/plugin.go @@ -21,10 +21,7 @@ func NewFleetManagerPlugin(config *common.Config) lbmanagement.FactoryFunc { if err := config.Unpack(&c); err != nil { return nil } - - if c.Mode == ModeFleet { - return NewFleetManager - } + return NewFleetManager } return nil From 33132a57542b8901845f399e9948f52b7e76b721 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Wed, 19 May 2021 15:07:42 -0700 Subject: [PATCH 6/8] Remove central management from docs --- .../docs/howto/load-ingest-pipelines.asciidoc | 8 +- filebeat/docs/index.asciidoc | 2 - .../docs/shared-central-management.asciidoc | 256 ------------------ metricbeat/docs/index.asciidoc | 2 - 4 files changed, 3 insertions(+), 265 deletions(-) delete mode 100644 libbeat/docs/shared-central-management.asciidoc diff --git a/filebeat/docs/howto/load-ingest-pipelines.asciidoc b/filebeat/docs/howto/load-ingest-pipelines.asciidoc index db0e3f00fb3..68471845a3a 100644 --- a/filebeat/docs/howto/load-ingest-pipelines.asciidoc +++ b/filebeat/docs/howto/load-ingest-pipelines.asciidoc @@ -3,10 +3,8 @@ The ingest pipelines used to parse log lines are set up automatically the first time you run {beatname_uc}, assuming the {es} output is enabled. If you're sending -events to {ls}, or plan to use -<>, you need to -load the ingest pipelines manually. To do this, run the `setup` command with -the `--pipelines` option specified. If you used the +events to {ls} you need to load the ingest pipelines manually. To do this, run the +`setup` command with the `--pipelines` option specified. If you used the <> command to enable modules in the `modules.d` directory, also specify the `--modules` flag. For example, the following command loads the ingest pipelines used by all filesets enabled in the system, nginx, @@ -44,4 +42,4 @@ PS > .{backslash}{beatname_lc}.exe setup --pipelines --modules system,nginx,mysq TIP: If you're loading ingest pipelines manually because you want to send events to {ls}, also see -{logstash-ref}/filebeat-modules.html[Working with {beatname_uc} modules]. \ No newline at end of file +{logstash-ref}/filebeat-modules.html[Working with {beatname_uc} modules]. diff --git a/filebeat/docs/index.asciidoc b/filebeat/docs/index.asciidoc index 5193aef8667..fc9eaefdc32 100644 --- a/filebeat/docs/index.asciidoc +++ b/filebeat/docs/index.asciidoc @@ -49,8 +49,6 @@ include::./configuring-howto.asciidoc[] include::{docdir}/howto/howto.asciidoc[] -include::{libbeat-dir}/shared-central-management.asciidoc[] - include::./modules.asciidoc[] include::./fields.asciidoc[] diff --git a/libbeat/docs/shared-central-management.asciidoc b/libbeat/docs/shared-central-management.asciidoc deleted file mode 100644 index 6789ec012ef..00000000000 --- a/libbeat/docs/shared-central-management.asciidoc +++ /dev/null @@ -1,256 +0,0 @@ -[[configuration-central-management]] -[role="xpack"] -= {beats} central management - -[partintro] --- - -include::{asciidoc-dir}/../../shared/discontinued.asciidoc[tag=cm-discontinued] - -[WARNING] -======================================= -When you use central management, configurations are stored centrally in {es}. To -prevent an attacker from leveraging the configurations to attack your -infrastructure, you must secure {es} and {kib} before using central management. -See {ref}/secure-cluster.html[Secure a cluster]. -======================================= - -{beats} central management provides a way to define and manage configurations in -a central location in {kib} and quickly deploy configuration changes to all -{beats} running across your enterprise. - -To learn more, see <>. - -To use central management, <>, then use -the {cm-ui} UI in {kib} to create and apply the configurations. - -[NOTE] -===== -This feature requires an Elastic license that includes {beats} central -management. - -include::shared-license-statement.asciidoc[] -===== - --- - -[[how-central-managment-works]] -[role="xpack"] -== How central management works - -include::{asciidoc-dir}/../../shared/discontinued.asciidoc[tag=cm-discontinued] - -{beats} central management uses a mechanism called configuration tags to group -related configurations. You define configuration tags in the {cm-ui} UI in {kib} -after <> - -A _configuration tag_ is a group of configuration blocks that you can apply to -one or more {beats}. A tag can have configuration blocks for different types of -{beats}. For example, you might have a tag called `development` that you use to -group all configurations that are valid for running {beats} in your development -environment. The `development` tag might have: - -* Two Metricbeat module configuration blocks: one that reads system metrics and -another that reads metrics from Apache HTTP servers -* Two Filebeat module configuration blocks: one that reads Apache HTTP server -logs and another that reads system logs -* One heartbeat monitor configuration block: checks that a public facing website -is live. -* One Elasticsearch output configuration block that sends the output to your -{es} development cluster - -[role="screenshot"] -image::./images/configuration-blocks.png[Screen showing configuration blocks in a tag] - -You apply the tag to any {beats} that will use the configurations defined in the -tag. - -[role="screenshot"] -image::./images/enrolled-beats.png[Screen showing {beats} with tags applied] - -When the enrolled {beats} run, each Beat uses the configuration blocks that are -valid for its type. - -You can add, modify, or remove configuration blocks from a tag. Any changes that -you make to the configuration blocks in a tag are automatically applied to all -{beats} that have that tag. - -You can add or remove tags to change the set of configuration blocks applied to -your {beats}. For example, after you've tested configurations in your -`development` environment, you can remove the `development` tag and add a -`production` tag that has an {es} output configuration block for sending the -data to your production cluster. - -[role="screenshot"] -image::./images/enrolled-beats-dev-prod.png[Screen showing {beats} with development and production tags applied] - -You can apply multiple tags to a Beat. For example, instead of defining the -apache modules under the `development` tag as described earlier, you could -create a separate tag called `apache` that contains the Apache module -configurations, then apply the tag to all {beats} running on Apache servers. -This would enable you to maintain your Apache module configurations under a -single tag, while also using a `development` tag to send output for some -instances to your development cluster. - -[role="screenshot"] -image::./images/enrolled-beats-apache.png[Screen showing {beats} with multiple tags] - -You can apply as many tags as you need. Just keep in mind that the -configurations for all assigned tags are merged, which means that you should not -specify conflicting configurations. If there are errors in the configuration, -you'll see an Error status in the {cm-ui} UI and need to look at the logs for -the Beat to troubleshoot the problem. - -[[enroll-beats]] -[role="xpack"] -== Enroll {beats} in central management - -include::{asciidoc-dir}/../../shared/discontinued.asciidoc[tag=cm-discontinued] - -You need to enroll {beats} to register them in -<> and establish -trust. Enrolled {beats} will have the credentials needed to retrieve -configurations from {kib}. - -During the enrollment process: - - . The Beat contacts {kib} and tries to register - . {kib} registers the Beat instance and returns an access token - for configuration polling - . The enroll command creates a backup of your configuration and then - **overwrites the current settings** so they can be managed centrally - -To enroll {beats}, use either <> -or <> enrollment. - - -[float] -=== Prerequisites - -* Verify that your Elastic license includes the {beats} central management -feature. -+ -include::shared-license-statement.asciidoc[] - -* {kibana-ref}/using-kibana-with-security.html[Enable security] in {kib} to -ensure that only users with sufficient privileges are able to access {beats} -configurations. -* Assign the `beats_admin` role to any users who need to enroll {beats} or -manage configuration settings in central management. -ifndef::no_dashboards[] -* If you plan to use the sample {kib} dashboards provided with {beatname_uc}, -<> before enrolling the -Beat. -endif::[] -ifeval::["{beatname_lc}"=="filebeat"] -* If you plan to define module configurations in central management, set up the -ingest pipelines before enrolling the Beat. For more information, see -<>. -endif::[] - -[float] -[[token-based-enrollment]] -=== Token-based enrollment - -Token-based enrollment is recommended if you are enrolling {beats} manually. - -To use token-based enrollment, go to {kib} -> Management -> {beats} and click -`Enroll Beat`. Select the Beat type and operating system, then copy and run the -command for enrolling the Beat. - -The command has this format: - -["source","shell",subs="attributes"] ----------------------------------------------------------------------- -{beatname_lc} enroll KIBANA_URL TOKEN ----------------------------------------------------------------------- - -*`KIBANA_URL`*:: -The URL of the {kib} instance you will use for central management. - -*`TOKEN`*:: -The enrollment token generated by the {cm-ui} UI. The enrollment token will -expire as soon as it's used. - -For example: -["source","shell",subs="attributes"] ----------------------------------------------------------------------- -{beatname_lc} enroll http://xyz.gov:5601 70f4b584e8024b96b682c46125a8d81 ----------------------------------------------------------------------- - -Repeat this process to enroll additional {beats}. - -// Maintainers: If you update the following note, also update the note that -// appears later in this file. - -[IMPORTANT] -===== -*Windows users:* If you installed {beatname_uc} as a service, you must also set -`-path.data` to +"C:{backslash}ProgramData{backslash}{beatname_lc}"+ when you -run the enroll command. For example: - -+.{backslash}{beatname_lc}.exe enroll http://xyz.gov:5601 70f4b584e8024b96b682c46125a8d81a --path.data "C:{backslash}ProgramData{backslash}{beatname_lc}"+ - -Why? The service installation script, +install-service-{beatname_lc}.ps1+, -changes the default data path to match the convention used for Windows. If you -run the enroll command without specifying the correct data path, {beatname_uc} -will be enrolled in central management with the wrong UUID and unable to receive -the configuration. -===== - -[float] -[[username-password-enrollment]] -=== Username and password-based enrollment - -You can also enroll by specifying a username and password. This is the -recommended way for scripted deploys: - -["source","shell",subs="attributes"] ----------------------------------------------------------------------- -{beatname_lc} enroll KIBANA_URL --username USER --password METHOD [--force] ----------------------------------------------------------------------- - -*`--username USER`*:: -The username to use for password-based enrollment. The default -username is `elastic`. - -*`--password METHOD`*:: -The method to use for getting the password. Available options are: - - * `env:VAR_NAME` gets the password from the environment variable `VAR_NAME` - * `stdin` prompts the user for a password. This is the default. - -*`--force`*:: -Overwrites the current settings without asking for confirmation. - -For example: - -["source","shell",subs="attributes"] ----------------------------------------------------------------------- -{beatname_lc} enroll http://xyz.gov:5601 --username myuser --password stdin ----------------------------------------------------------------------- - -[IMPORTANT] -===== -*Windows users:* If you installed {beatname_uc} as a service, you must also set -`-path.data` to +"C:{backslash}ProgramData{backslash}{beatname_lc}"+ when you -run the enroll command. For example: - -+.{backslash}{beatname_lc}.exe enroll http://xyz.gov:5601 --username myuser --password stdin --path.data "C:{backslash}ProgramData{backslash}{beatname_lc}"+ - -Why? The service installation script, +install-service-{beatname_lc}.ps1+, -changes the default data path to match the convention used for Windows. If you -run the enroll command without specifying the correct data path, {beatname_uc} -will be enrolled in central management with the wrong UUID and unable to receive -the configuration. -===== - - - -//[[central-management-API]] -//== Enrollment API (not documented for beta) -// -//available. - diff --git a/metricbeat/docs/index.asciidoc b/metricbeat/docs/index.asciidoc index 42a00fe97af..8fff64854e8 100644 --- a/metricbeat/docs/index.asciidoc +++ b/metricbeat/docs/index.asciidoc @@ -49,8 +49,6 @@ include::./configuring-howto.asciidoc[] include::{docdir}/howto/howto.asciidoc[] -include::{libbeat-dir}/shared-central-management.asciidoc[] - include::./modules.asciidoc[] include::./fields.asciidoc[] From 68f45f1ac643503123df69bb4a5fc13264d571fd Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Wed, 26 May 2021 09:41:30 -0700 Subject: [PATCH 7/8] Replace custom error code with go-multierror --- x-pack/libbeat/management/blacklist.go | 12 ++--- x-pack/libbeat/management/blacklist_test.go | 6 ++- x-pack/libbeat/management/error.go | 60 --------------------- x-pack/libbeat/management/error_test.go | 27 ---------- x-pack/libbeat/management/manager.go | 30 +++++------ 5 files changed, 25 insertions(+), 110 deletions(-) delete mode 100644 x-pack/libbeat/management/error.go delete mode 100644 x-pack/libbeat/management/error_test.go diff --git a/x-pack/libbeat/management/blacklist.go b/x-pack/libbeat/management/blacklist.go index d344dad727d..6ea5088a962 100644 --- a/x-pack/libbeat/management/blacklist.go +++ b/x-pack/libbeat/management/blacklist.go @@ -8,6 +8,7 @@ import ( "fmt" "strings" + "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" @@ -60,19 +61,16 @@ func NewConfigBlacklist(cfg ConfigBlacklistSettings) (*ConfigBlacklist, error) { } // Detect an error if any of the given config blocks is blacklisted -func (c *ConfigBlacklist) Detect(configBlocks ConfigBlocks) Errors { - var errs Errors +func (c *ConfigBlacklist) Detect(configBlocks ConfigBlocks) error { + var err *multierror.Error for _, configs := range configBlocks { for _, block := range configs.Blocks { if c.isBlacklisted(configs.Type, block) { - errs = append(errs, &Error{ - Type: ConfigError, - Err: fmt.Errorf("Config for '%s' is blacklisted", configs.Type), - }) + err = multierror.Append(err, fmt.Errorf("Config for '%s' is blacklisted", configs.Type)) } } } - return errs + return err.ErrorOrNil() } func (c *ConfigBlacklist) isBlacklisted(blockType string, block *ConfigBlock) bool { diff --git a/x-pack/libbeat/management/blacklist_test.go b/x-pack/libbeat/management/blacklist_test.go index 016dd675179..ead5060d961 100644 --- a/x-pack/libbeat/management/blacklist_test.go +++ b/x-pack/libbeat/management/blacklist_test.go @@ -273,7 +273,11 @@ func TestConfigBlacklist(t *testing.T) { } errs := bl.Detect(test.blocks) - assert.Equal(t, test.blacklisted, !errs.IsEmpty()) + if test.blacklisted { + assert.NotNil(t, errs) + } else { + assert.Nil(t, errs) + } }) } } diff --git a/x-pack/libbeat/management/error.go b/x-pack/libbeat/management/error.go deleted file mode 100644 index 1d330a5938f..00000000000 --- a/x-pack/libbeat/management/error.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "strconv" - "strings" - - "github.com/gofrs/uuid" -) - -// ErrorType is type of error that the events endpoint understand. -type ErrorType string - -// ConfigError is the type of error send when an unpack or a blacklist happen. -var ConfigError = ErrorType("CONFIG") - -// Error is a config error to be reported to kibana. -type Error struct { - Type ErrorType - UUID uuid.UUID - Err error -} - -func (e *Error) Error() string { - return e.Err.Error() -} - -// Errors contains mutiples config error. -type Errors []*Error - -// Errors makes sure we can display the error in the logger. -func (er *Errors) Error() string { - var s strings.Builder - if len(*er) == 1 { - s.WriteString("1 error: ") - } else { - s.WriteString(strconv.Itoa(len(*er))) - s.WriteString(" errors: ") - } - for idx, err := range *er { - if idx != 0 { - s.WriteString("; ") - } - s.WriteString(err.Error()) - } - return s.String() -} - -// IsEmpty returns true when we don't have any errors. -func (er *Errors) IsEmpty() bool { - return len(*er) == 0 -} - -// NewConfigError wraps an error to be a management error of a specific ConfigError Type -func NewConfigError(err error) *Error { - return &Error{Type: ConfigError, Err: err} -} diff --git a/x-pack/libbeat/management/error_test.go b/x-pack/libbeat/management/error_test.go deleted file mode 100644 index ee85b4d78bd..00000000000 --- a/x-pack/libbeat/management/error_test.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" -) - -func TestErrors(t *testing.T) { - t.Run("single error", func(t *testing.T) { - errors := Errors{NewConfigError(errors.New("error1"))} - assert.Equal(t, "1 error: error1", errors.Error()) - }) - - t.Run("multiple errors", func(t *testing.T) { - errors := Errors{ - NewConfigError(errors.New("error1")), - NewConfigError(errors.New("error2")), - } - assert.Equal(t, "2 errors: error1; error2", errors.Error()) - }) -} diff --git a/x-pack/libbeat/management/manager.go b/x-pack/libbeat/management/manager.go index 9fa96bbc55e..36a4f8c0245 100644 --- a/x-pack/libbeat/management/manager.go +++ b/x-pack/libbeat/management/manager.go @@ -12,6 +12,7 @@ import ( "sync" "github.com/gofrs/uuid" + "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/elastic/elastic-agent-client/v7/pkg/client" @@ -165,7 +166,7 @@ func (cm *Manager) OnConfig(s string) { return } - if errs := cm.apply(blocks); !errs.IsEmpty() { + if errs := cm.apply(blocks); errs != nil { // `cm.apply` already logs the errors; currently allow beat to run degraded cm.UpdateStatus(lbmanagement.Failed, errs.Error()) return @@ -199,23 +200,22 @@ func (cm *Manager) OnError(err error) { cm.logger.Errorf("elastic-agent-client got error: %s", err) } -func (cm *Manager) apply(blocks ConfigBlocks) Errors { - var errors Errors +func (cm *Manager) apply(blocks ConfigBlocks) error { missing := map[string]bool{} for _, name := range cm.registry.GetRegisteredNames() { missing[name] = true } // Detect unwanted configs from the list - if errs := cm.blacklist.Detect(blocks); !errs.IsEmpty() { - errors = append(errors, errs...) - return errors + if err := cm.blacklist.Detect(blocks); err != nil { + return err } + var errors *multierror.Error // Reload configs for _, b := range blocks { if err := cm.reload(b.Type, b.Blocks); err != nil { - errors = append(errors, err) + errors = multierror.Append(errors, err) } missing[b.Type] = false } @@ -224,22 +224,22 @@ func (cm *Manager) apply(blocks ConfigBlocks) Errors { for name := range missing { if missing[name] { if err := cm.reload(name, []*ConfigBlock{}); err != nil { - errors = append(errors, err) + errors = multierror.Append(errors, err) } } } - return errors + return errors.ErrorOrNil() } -func (cm *Manager) reload(t string, blocks []*ConfigBlock) *Error { +func (cm *Manager) reload(t string, blocks []*ConfigBlock) error { cm.logger.Infof("Applying settings for %s", t) if obj := cm.registry.GetReloadable(t); obj != nil { // Single object if len(blocks) > 1 { err := fmt.Errorf("got an invalid number of configs for %s: %d, expected: 1", t, len(blocks)) cm.logger.Error(err) - return NewConfigError(err) + return err } var config *reload.ConfigWithMeta @@ -248,13 +248,13 @@ func (cm *Manager) reload(t string, blocks []*ConfigBlock) *Error { config, err = blocks[0].ConfigWithMeta() if err != nil { cm.logger.Error(err) - return NewConfigError(err) + return err } } if err := obj.Reload(config); err != nil { cm.logger.Error(err) - return NewConfigError(err) + return err } } else if obj := cm.registry.GetReloadableList(t); obj != nil { // List @@ -263,14 +263,14 @@ func (cm *Manager) reload(t string, blocks []*ConfigBlock) *Error { config, err := block.ConfigWithMeta() if err != nil { cm.logger.Error(err) - return NewConfigError(err) + return err } configs = append(configs, config) } if err := obj.Reload(configs); err != nil { cm.logger.Error(err) - return NewConfigError(err) + return err } } From 593123ec4ec005548350f4af289da5637763f77f Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Wed, 26 May 2021 16:16:44 -0700 Subject: [PATCH 8/8] Remove management mode from specs --- x-pack/elastic-agent/pkg/agent/program/supported.go | 2 +- x-pack/elastic-agent/spec/apm-server.yml | 1 - x-pack/elastic-agent/spec/filebeat.yml | 1 - x-pack/elastic-agent/spec/heartbeat.yml | 2 +- x-pack/elastic-agent/spec/metricbeat.yml | 1 - x-pack/elastic-agent/spec/osquerybeat.yml | 2 +- x-pack/elastic-agent/spec/packetbeat.yml | 2 +- 7 files changed, 4 insertions(+), 7 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index 9cfac186565..c1c5ded612b 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -25,7 +25,7 @@ func init() { // spec/metricbeat.yml // spec/osquerybeat.yml // spec/packetbeat.yml - unpacked := packer.MustUnpack("eJzUelt3oziX9v33M/r2m5mXQ0g3s1Zf2E6DwQ4p4wQJ3SFhA7aE6RgfYNb891kSBwO2U0lVzbwzF7UqIUKHrb2f/exn8x+/7bMV+UeQsX/dr96Pq/d/Kxj97d9/w8zI0esuWnhjZ+45lKSIkijbYLB4tEzjhJdyiaCtIGjNfGhLAUCxr978W0rKXQROu8iaWLm7tPbWxM59oMVI8XIENGnOvIMP7D0CCz2c2jJaWvtJMoqsRDas5BRZTI4xM1IEZIrTxQEDfYte5Yc5HMe+ss8RO1MEF8l81FsjJqaUE8UrQuYVc+bGoWmsieoWCBg5KTSKmZFg09t+g24Rgrfh+6kv6ycE7Q3f47el+zu3xZvkLb2tt4RS+M2Vnh+fX0en9dKawUVWhOBMSbmbWZNRbpnOkUzpOjT1NTZpGT6J5xH/F0CX4tddVO+7fR6afzxaJj2QqScRYS/+fHzEVN9gRWPYpJvwaRdhph+J2KsUYWCcXqJ27MmH7u4lGTNsejQctc+7e6ufjXJralMMdAV5+juC20dh80nnn5nT1esu8oGzQdApoWKcgmrsD81DmHfCKvcVLe2OmS/HBQLyMWTeOlA87XKe5t9YzGuZ3Ac0ib87X45ZAOP2Pl+S8e/rRWUTxOg+NL0CqijGU693bqJ4ewQcCavWLVtX60xdjZhvl3thxj4EXtnflxT5cJy3ewFnCUHrcvapQ7HpbUJTL67sfrVuNR9WPYk/H9imvstMDUzv8JKM9whoaWhGO3uaZzgdy+HU0WfL0f+3nkaRD7StZcYxkXK6WkbblaLL4ZSPkfbWJKTYNMrQpBuieDFhzs4utrPf/qUCgVUaZrskzQcQ4AJtS0w9w+kielO8TQjtLJxuZ74ib1+SMcXMPWGFHsKJXCLgyIRRabXIYpK6GWKGcFl0mSNHpqdMUgEpma+8PVpPvvryFM184EgB0A9QqcwDVX4Nwuy5ZXoHNB0fA3797HxEsi5cvTLteOtDWw3Aw6M1sY6vJk0IM4rVUjea486ly/tz1ZF86NK5cj6iQu/sX/p7zucuLD7nPgCavHraRVaiH8l0cXTBOSaqm/mFblze0cvQNCS01PdYIcfuOWeJxp8l/DpDhR6QqascCq3t8yM0zgvC9JQwI7f+Qhk2vRIa53a/4udmDeNMiMrhyyPQ5Gc/k5vrMGeHgPMu7Ke6MTZPj5NEihCMqS/r3EVp48LENKSAn4117AId6qteEUBXs+pxNXzPGhe1eBpglK2W1uVZIuXY1NPmnflylBDV3SJoF82z0KQ5ArrMfeG5HM2IqZehwffvSD447+s7fkDAWfPwRa/1mtNxHJrRozWxb/tZsw/TKJD63IRfbk3sdu7uvuZLub2TelwZmi4lqdV5ZuVz6J2QasfIfBs8tylRdJkwh5KiY4M7duyP1x4DOKrnG0sBT2eqJ70kI+X5aTQjU5tC1TsEQOM+tcdPu9l8OaYr09tAhfvIW32+BuZHSdcPyCU2mzViwsIubPHzypi1/pFcoOb6Hm/b58a+U2+P4HPla4w+QNWRCPNinuJWqjSrn9+G5A5MviTjGs6eZ920EU7pCS12N6GY26f1i5GwVw3F9no49iPoF2lHYFlFCeqzKj44y9XZmlQnqII4Uzf1NvHUS+lTW8Zmb6/301yTImu6MhHxgDI8dSnZXNuqG5N9GiFFAdBOIXTLds9mnQJGnX1AlBGFHnG0m4VKTPFmF2GOsaq7m03c36s53UFaOVPMQimY8LRS20+VMuvpIXqejGPMFlFgGuVS8TQ+R5PO1stTZCve3occ350SAaPwlSidLTJOa8pwasc8bjg22tO8CIEmfGzO+LhYtyZ/6dYkjAmTFGdC2nS1TugKr4KrdMXhA9jUh4smRQno85kXh6OsColkjHsMM3VoOPVOc0b3eNlhhoC7q0Ot5MTNm8zf3pL5ZJQQxZNCODqEppcT8xyH5tsBAS32udmeZOaDc/mLWOwXmfL5iJQww4wcsOkpCJx0ZHpJCMh32a27pc/um20sPHftSvp0Ue7Oz09j3YqycMKMIjQp4yn6JRknCBgSKfSuXQsEeVq1JajwFF67nKFvAtM4IJ7m69D5HlzMl+MMs4z6qrsOgLZFMGpgT1rBMeUMCCthWYUIPQTM24SGzt2I+kBek6l99BWvJIpeNCGGFW3tK/oBsXNWVSn0IELN0GOUupQ0aYpDzSsPBT0lJwEnuQ9HA9gdwNR1mG+wOtagYuyxoUtY1vcBdKReuE/d40sybvZcdhnf9V61I1ZHHQbJU4JDyXTxaBn75FLRiHvN0ET83/pw5d/jAisOJapzJOlzAheDvaruESrnjKiLHntvKpWuTXus9UvnaG2eIICOhFUwSoRPyy01Qal95Gv2YWws4fLyrD3zqU6RU7fgvlmdwV2LfbewPbw3Z7Dfccbj+Poc/TUrf5Nj8jSgJoNUcvHvS8rwgVZiThNfu5VAuy/h1z3bAeNETK8YVkLcL2t6JH7uV1pjGjKPU9KK6ok4IbN+JcYxoaLAyKQHcWbTzfBgHQQRx6hjaDonvj+/53/debwtedpFIXBPg71UdNjUN4Hi8dS+xYrzjqA1mEdQ8AIBNyOyXmJTV/m5XpJx9ex0ff656miEU/dyFxGVlh07fFiJ1ileJoq3HlRan3lv60M3brFq8fmUjkxD8r0Gn6SDyFPQOfnAoT+8/lITv/M81KMKVB9QlnGJOEar7pFsrmLqutqdDitmXuWG5Rw4e6x621AxJF+J+n+Dzx2qYTyG0KZzUPlHMPUu9Nms5kBMlzFzi1UXgxRN8gE9XPxDihrfubwfbshEPoXALkNwGRdMXYkYepcGNvPuwql7ClLniC/Y/O4D9O5f1u7EiBRhaEid9w8Iuoz75GUPf5TOxotD4GY9OtXx1fmyH1P89xV0euvwuLrcvbcll7k4PX+4/I3n8vMRXeYWsd3g30/fvdnyibv3L3jGYtfLyz4IKU5FfmYcIxua7yv6adVgnIkKrEhX+FipXKLkFTzia1T3wu3uqCa5iDvIab5NJ2m4Q+DhsU9dL2vP2c/T2PlklBLmbQP4nFYcK6x8bEn21iQUXCU0jTKYkGwS/flnS1/papXfFl3digpGb001wJwcXaqLVtTktIxXvZwGzJcyp4m8Os6x4lKLSgNhFu0R8ER1Wou5A8ESZauWRkmP1jTXJ9H/rKB2db5ftf7QzT/YQ+PuH8JyDe2t69X7bPYCOQyZf9wTAyNQ1KLeRGuE8kMz15xduVjU0LUOZTiG0D2FcDGrBcgm1HlIctpywz62KHtaP0hrP0i0E1Y4RdseArC4tVZDdw/Pk3bs19a9fc+HizgfH4m6GIikN/ap2rfF0Y4diersL3YfULTi4dTOFzV+36F09yr5a1rXheO+8Dv5rgDcOeNF0B7+nacvrDqtb90R+vulyP19fk5gn3ol8vTWt6wfEPkHdOdXzHGLMn3tXGa/+VM3g0r0g2fslh8NdvbS1+f8/2tNgehmA6Dy/2k/fXWVnCbtr5fb6FsyOlm8HJ+Mdz505ghu+Rz1fbs6T2WoUblVh/rQ3gzTVrwK3vMbssvS9GKSulVKr/NV0HvWyVUDGSUA57wrbSBm7IlSjfmq5HJfOpGPyHzTVxOZ+9H7HFT4IsrjOxJNZ36JpB7tj70vtcBB90NIJNPnY88eqc1pHP99zUvl5p6h4ux8oKVIKJ88F30iT/eaUBflE6phFprxmjAvRTDu4sfNvEzY26NlaOWKU3fTEzloXjy838pjA+z5gIZ+R+W9q77eoqNNnFZS0o/E3kAdbXDkNuZ9l0r+8+gjW+XvCbkRiK/Akwijm1qXqzv2Mg2nduYrtS56uytfIujKZKJl2JS+r2c2mmnqUgzHe9FSuxXEH+qZd4Oyp7diYBxEzQE4wf1u554bLQ2Als7ZmYbM238DLvVTLx3OixRnTUyv8DlxfpK2K0k2PMO2F9KXdNAcQbcIgPPVwD7xBN9v1wlnzLGKaL/l02uFcYIvNMc5WxyJSksOivOU5nhidb4mMAqkiDZNgoG+FSBSaFuhn6guxUtNw+B0QCYt6ztLJmmrZd4BGhFAfzckF5new8tQO639r9bm2pqxSZpQ7WiRTdCa+gEqzhEztA8q22ZYEDVeayKp1QUqwLr4s/r8qzTED3TeK71w8DVFXfd6eozN880vPpq1O2t2SMXV2Q9Y0Xt6GoLxBsGxJAh8WmtA8Fn4cAAWlS838VrpvyfCdPHVBgdBXgwO9npblzaphMRd9/TiHz3H5Q6ZxwSI1m3ecGrH3CeEnQxd6K+XluCAtKvteWf9PQ6eV1p0m1xuth1lXQ2gu4OqTZHiPbTE/0arsUqGD4fZJb5Le/RhK/Gf3n4c+PuVn3K749QRWH2fHHTuYEAQ+Jw8xmES/ePt6Sx062/Jw/tseW2jah6+Bscst6txV0S7ylfduRlhen5LD680I/mIpt4eXbS73Ac5hYpREGZoN/24xQmn8zVU6yvtnlG3Xfp9Lbbz3kWHIsyrcVv40uy6CBmKEJ95p1+4/BK9+MfnEBhL1HHsK2+fP8OgqOznkKoAFrmE/dGITvXXc82eWr8Y9N16d3jREoe+1eqQPQ50uJUb2jzZi2XrWk9NOzE54FZXhdstXfJ2Mdc7i+BG3N/hz3zdJfjOhWBORmmgGCxQ/hI/iz4n5yqiHb9ryeVu//dh9V7cYpeqcw6BV6z6HfYjUQ0ZQVsbdtm/0GH/OrO8zxCFBPoNRjkxjU1QyLVM9FF5eN1Z/xSjpNKMMI9T9CI09COmjae7a1+JY8xCyhGNl4932OOV7XzVzThzrD10LbJS3THHjf0Xu/9rnfSfyYydjHAjK5p6SaBHSbr9JLp8rtvwwYc0Nz+a+SVSC3OOWDA8I8OpW74ko+1KtWWcuhkGbwcxbipF1qsU2YpR4Fdfsouq7Lz/8QwpX578tFs+ZgHZrm7pOG88YBRP6pWPUw75OQ3NQflYkNyt6tTvlI58zNVYCQH5JNriNwDBLk6RXciG+F/51KczAtLmb28VtH1cMvbHmnqKuJMU2l60vp/kLQK2jIrwXsknylUEf17L+UnNpE+V7uol9f2NmnXbYK96RoaeYYaO9UfU/y26yv8O/WT352//+f/+KwAA//9DyXVy") + unpacked := packer.MustUnpack("eJy8WkuXm7i2nt+f0dNzHzzi6uaudQaGCi+7qBinkNAMSTZgC+wuY2O46/73s8QbbKeSdPoMspLIQtra2o9vf1v/99vpuCH/ExyT/zpt3i+b9/8uEvbb//6GEz1DXw/hylOdpecwkiJGwuMOg9WTZeg5XoslgraEoLXwoS0EAEW+fPe3lJSHEOSH0NKszF1bJ0uzMx/MIiR5GQIzYZl4Zx/YJwRWCjVtEa2tkxbPQysWdSvOQysZrXlGhi74nlJS02Y+EMuPv6c7KKuMJA7D6UqxzUx9+yx+dT0buJ69dQXFXJWH68uzqljhkWqpLWLDK19DLrMQ+pKSbzxFwKJyCqAj1OPz0NLUIzW87DVWE2x4jM67cQGXhzAAs5xCt6xk4eOGcoaSc8EJOgXAEV5j9YwlJee/L9dq5sP5UzfXVCNqhE+WgU4IeEI/PpStGdOEkCRehmXEoJSxzdfDov+t/hNI3uw1ViNfchiRna0P1WM1d/VT6xQIqheSukeckOGczDJthoEiIU95R3Dfn6f9Y1Trhj6g/C4W1TcGOm709j6FJ8vMlEYnSQCuAoL2lib6iYLhudUSgSvzZfdCdvd0Xe9DTZaj7oyq5IOriODLSK7lWo2IIXSyYNNlZNefnUjeCQFHwLJ9o/ebfev1LhS6OYWrsW7au0zpAYFPT5ZxZTihQqCF+43EzsT0BCILR+v5U/iiqRFOVmFg6OVa8mYLzf0dy57A52zXeWhL3smHjhAAp0RAL3wpTBerwz9/+8/aoTcpPR7iNJu4swtme2IoR5yuwjfJ21FoH6m5X/iSuH+NVYYTN8cSO1NNLBFwRJIwYbM6RvyqUaLv6PMhRP0aGTI8SUur8HD0pbcn69mXX5/DhQ+4aNzc62NB2Z2R2mwzy/DOyFQvAZgJWnK9IFHJfegeSMlVru59aMsBV49mXb4aLCaJXmzWio4NvaQG2y2F/vul7Ag+dNlSul5QoQzkF/5c8rULi695CsBM3DwfQitWLsRcXVxwjYjsHv1C0ftvlJIauoDWyglL5DI85yKe8bGYXzOV2BkZiszDmrV/eYL6dUUSJSWJnlmf0ZG7J9SvnbzVv9s99CshshtRwyPQ4Ge/krv7JM4BAee90p/sRtjIn7RYCBGMmC8q3C1Ya4rE0IWAny0Z6AU6zJe9IoDuzGrmNaF40ZquxUN6wpLN2urHYiHDhpK23yzX85jI7h5Bu2jHqMEyBBSR28JLOV8QQympzuV3BB9cT80df0LA2aKEnVAbRtqQptn37ayVw9ALJHcumlma3a09lGu5Frs7aeaV1HAZSa3BmJUtoZcj2Y6Q8TYZtxmRFJGnBVIMdPBAj+P5s6cAzpv1VCEAIuPu+RrPpZfn+YKYNoOydw7AjNvUCT8fFsu1yjaGt4MSt5G35nxqZfuv8Twe2gHpfbPdIyIJHYYffl4RJ519xH0Iur3H+/q5I3fqnerwKIQoYZ+g7Agk8SL89RBuZGHRjJ+o4RVQRhE2PdbZlOkwbHg7aijFa6wecaqK1HxZPArH05DO9dPZxbzS18Pw34dk68kymtCZH4YpiuFEj7Hh7ZuzTkN/ZpluQcFbdSYM9HzqTyi5MtSG8EnaXa7VAgHxQhNvW+03TAvNWYnkFTTxCq3yh0laGehq6JOjPW/TS2YZikhNVWyhRiUHREcisQsODwsqRQzvDiHmMVZ2Dzxt1Gu6ymI9/4f1PA99MNv/+tRz3GFpxlN4xP2Gx0bbzAoKZpWNLRM+L1Is7bNiaTQiiSA5Glm06Wobsw3eBDfpiocPYDMfrtoUVYU+P/EiOj/WLhGreIT2UodR08uXCTvh9awzgy+Am6vDrDjn6o2Xb2/xUpvHRPIECudnjuSIcY2o8XZGYBb5XG3PYuKDa3mLKMUIJ3qKuPukq+F8gaTezR7cFRFPG8XshCBi+FncI2CLqPgQqRrrt6u+2nuqpyvmV4E+v+4+5y+mEGtMmCBvrie3XFbhxYsR0AUttRkPDyR1t1iabdtrhJJz8MEsRZVL2iJaHQsKrpUrV24Hoy2R3QIBnaPb37crbsLsHCTergr1ics2z60p89T+xtHMBckvlTsFYPYnd88ufHhKThJlh6BTcpdtXPKCmcLNJsEGq6AFD5cI2gKU9ISHmDZMbaDKXmP1hCVa1u4kRuR5klYmYaAPZa3rOBdisi1PI1B2L1C6Hom8GiFBavzxZJmNzPBlgN5uZcWJciHzodt7n3zJy/lvoLC76qS+V7av/+4qldr2TPviS15JJKUghU2nslJD2WKDlfR5iMS7qmOgU7v82XP0OrcZSpQCrSobKLhNY9ClqYQkSnYT1k330o853Zm1BgLwcODLbn0GXank7tPF5N7kibwGE1BVDUzGR3u+PAzf49CpdvbdhlYuG06dE4ePo/DdylXb9VB3mQ/VHEFrUtVUdllDsspGybhqMjypqnKbtF/5SX4YVUVVTEhXFw61KuhrOgIy2HmyD6OJx2Gq4MtzLt9uZH+DdShw89dYFZE5n8hSweQ9lpx3fg7LcC++lDESjtfh8Wop2wwZrISyc8Iy5ed64tUdH7s9P7kQmZX8u9dYLTfQGejhW1WlWlWJlumVyFMmVdN3fGdwKK13sapP7Y+r0AYe7H3oRl18Ws/OPhAZkXmF/PbT+y+T6v8lT+F/M1SKqPyS+dKV37XsQ3cXzMe/kfKlO4cPjyJJ3rLaPtwDBT3cbdZIsMxhrz0bxiCcujydd/axXKut7fSQRXLyJVRFP3VEv593oKabQ2lQ6nXrRgI11T+JpJz7sSxCSRb1/+99ZLlWMwLdwfczRg10wnJvX7h8kRygi8hgwtAGBraaTXyK/39GpNE+3K/6+ADcvJ/rnQMY9r9J7MxtvZepLvsaBuMv332HJ+YP77/CGXWc7fJyhCTvXOdndGnyNo/lOyyrHURFqX3hcH4SHyvGCg3w1vAMt7DbHsoywGSHBwyIUPkdkd0LSd4ewNRu73Mr23a9D7/E89wy9DPS1IMPnSWC+4NtZs36rrLU5ilqy3nZYT60d4FGTpZGCwTcIylIhbFsqbYxu+AQlmMVR/Chc7CLfQ9D2WaT3SdC3RqZh28tqk+cDPVVQtah/aSuXi39xCvU+ko0McOSy6wbyFYTQrzKbAjWYZq6T4w1pORD8/qgOupD7ji93RJso4okG1Qzv2Z/o4NCH8pQQ9RGJ49Ce+M6LcRt5WxlgbyqNP4YEKyjCiwERUPiarOOaG7XWiY3FU8Ie1KxqeB7k2/Iy5ag3PLwh+/qx67Kl84O0sYO4lmOpevRl/fnAKzu7dWGjfOL1s39sX3v3/O5vedlEl2IfJ/AHMkp28KE/LzRI5Gd0y2x3dhF8Snv1gtbux/A+0cV+S3EH4Yj4RamPaq6R38yy+Bl3Uy4+bZJa1h2Otsaws47d/ORnI8ZhA+h0ccwZ5RORg0T4Vesca7iInRyHzjsp871Dej0M/K165D88FNE/tS26jWmsauBG12MOcqB4Z057EdgllIj5Kmptn9zzKYMU90vSG9HLfxnR/ZHm+A9u0OfrA2Pl/41FdDkq2A0NshVEzokANds2FhDiX4iUj3nR6mTH2n8DebysioNwCxdJlde+py+AJf5qZfe5tGW+oi4foSGHioQ5Cmel7bKGUoNc6Yru4DrW3p7ahnAoX8+oD/u5zxRkQPoHiAvlSTv0zBmNWVGRV0Mmne8RNwQmeeiiFGD7Ujxx3mRf9zU+hXNsCms/EZDrIaX8YN4+jiOfuwz433eEWQlXwet7vpSSy1M2MsoIkLGNutwv2nv1RQ45GNt14VIXkQSDu/y0G5KWbuoaLk0kPQkkD6n3Lc4jK8oskkDLdlk7zG541RfgSeQhO0aI2s64iKjpn30pYarvN/1LhF0RaLNjtgQPuYYWx4zdRmGasVT3HXI+V/prF8vSKJHnJAzrniKXEGGF1NApuumvqjkCNo7vu6Xtfv71zfv7W3Pnn+MmxzriSReSQ29oLpywaxNcu7Wl6IIJ5Q7S22IqXohNaDp2ikNUKgSzajNNGqJiRdkVs59RppSbqAjICCcN0A89a0RlQeJFMHVE9cfltyK6l8mq4rT4MFtmbIMa7N9AJ36zjTrw1bJcq0ecXJsgdgewfBpwmdmCLoFB5WTOq7l0bZDfrB1dizNtr6knFHCQd+qBlIctPH6L3W7Wr3iDzggbuyVJ8Zfw+t9g3udcni3bZOqFoWSfsL6vRcV3d79noOAcXv22QXLQ45LZRvDYcRcVUmtAwVFZcNHpFV/dz2B2s/UAtevIi4kfYlbsN3J+oArDqDL+F2PONz8Z8/R3WGMAKpq5KbdK5BE5zZR6QlKTEBALB9xnQMQNJZxMl7xw32gvttqgzI9UiPaksRLEYw6oHub2NSC2xmMP713/i2/tG23+4lp2BJjyiQJfSuZPSgezWkR2heN45c1D0D6rZ1yvZ+JxEGNt6USEwJdKRCgbGPOR3bQ3sH4pUmf7BfPyupLXWz/Yxmfjrc6aopfvsfzIbRHhXnFQVbc5LjFiAosCfc46orHwUDZU3BlPT8lRoHkbX1oF/6UD21spIsT0jhJ17bSysxz0ssP8KOD7+bf3zqdApXv+uZukfPXONyfX4PHWK8khr5Dq+8/A4UO64Bq3QLvckj9YqrKJQm/47YdX71Oa2Vq7WICwkZ3KPX83tS2Wm4QDzEOxyu3uaHLkz/S3h6t+51c4f3XUqOznFt7/yst7Brv9JyhpdF3H6B3f139u+o9cruYFl+H05/nzXtxDynKzpUCr9iMO9gXIusigvZs2sX+gQ72j6PEYTca6OfKmoB3ptpgfViVQ+O5DzvXNv2B7vLoAVl1bvPlgqf6+eajMaUk0GMk3S8G7zMfvB0cdKCrx1rep9cpKuKwPx97WOu9UB4ggtaib99uHnFFW3HPRELXEbnTDf8bu0CdLX1nJ2BSyg0ybTrslI+jx9BD/763iz/1gKQuBzX3d5J65+ph2npWlX3b9Tz6sp6nL2Uo2Xlf1h0Dst/c40reDH0XSJ4wKutMNfKljFFjUtYVJHPrsPBBScfn3MzlEC7H1UvDW+eu6tVC1Os2xbefmYznPizlUvjAAcn4zD/Pl/x7HunmPnDeUc+vf5Aa/wbu4j7f9+/mKBa//f9//CsAAP//rSoRAA==") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/elastic-agent/spec/apm-server.yml b/x-pack/elastic-agent/spec/apm-server.yml index 7e9d2af76dc..67aac6349e3 100644 --- a/x-pack/elastic-agent/spec/apm-server.yml +++ b/x-pack/elastic-agent/spec/apm-server.yml @@ -3,7 +3,6 @@ cmd: apm-server artifact: apm-server args: [ "-E", "management.enabled=true", - "-E", "management.mode=x-pack-fleet", "-E", "apm-server.data_streams.enabled=true", "-E", "gc_percent=${APMSERVER_GOGC:100}" ] diff --git a/x-pack/elastic-agent/spec/filebeat.yml b/x-pack/elastic-agent/spec/filebeat.yml index 0bf33f15e7d..8207c803284 100644 --- a/x-pack/elastic-agent/spec/filebeat.yml +++ b/x-pack/elastic-agent/spec/filebeat.yml @@ -3,7 +3,6 @@ cmd: filebeat args: [ "-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", - "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true", "-E", "logging.level=debug", "-E", "gc_percent=${FILEBEAT_GOGC:100}" diff --git a/x-pack/elastic-agent/spec/heartbeat.yml b/x-pack/elastic-agent/spec/heartbeat.yml index b4f5b14e5a8..adb0c414f10 100644 --- a/x-pack/elastic-agent/spec/heartbeat.yml +++ b/x-pack/elastic-agent/spec/heartbeat.yml @@ -1,6 +1,6 @@ name: Heartbeat cmd: heartbeat -args: ["-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true", "-E", "logging.level=debug"] +args: ["-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", "-E", "management.enabled=true", "-E", "logging.level=debug"] artifact: beats/heartbeat restart_on_output_change: true rules: diff --git a/x-pack/elastic-agent/spec/metricbeat.yml b/x-pack/elastic-agent/spec/metricbeat.yml index cb4c1e8ca2b..7968de13e8d 100644 --- a/x-pack/elastic-agent/spec/metricbeat.yml +++ b/x-pack/elastic-agent/spec/metricbeat.yml @@ -3,7 +3,6 @@ cmd: metricbeat args: [ "-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", - "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true", "-E", "logging.level=debug", "-E", "gc_percent=${METRICBEAT_GOGC:100}" diff --git a/x-pack/elastic-agent/spec/osquerybeat.yml b/x-pack/elastic-agent/spec/osquerybeat.yml index bb6e7e50a89..4735180343d 100644 --- a/x-pack/elastic-agent/spec/osquerybeat.yml +++ b/x-pack/elastic-agent/spec/osquerybeat.yml @@ -1,6 +1,6 @@ name: Osquerybeat cmd: osquerybeat -args: ["-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true", "-E", "logging.level=debug"] +args: ["-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", "-E", "management.enabled=true", "-E", "logging.level=debug"] restart_on_output_change: true artifact: beats/osquerybeat action_input_types: diff --git a/x-pack/elastic-agent/spec/packetbeat.yml b/x-pack/elastic-agent/spec/packetbeat.yml index bf05f901c48..37c2629f130 100644 --- a/x-pack/elastic-agent/spec/packetbeat.yml +++ b/x-pack/elastic-agent/spec/packetbeat.yml @@ -1,6 +1,6 @@ name: Packetbeat cmd: packetbeat -args: ['-E', 'setup.ilm.enabled=false', '-E', 'setup.template.enabled=false', '-E', 'management.mode=x-pack-fleet', '-E', 'management.enabled=true', '-E', 'logging.level=debug'] +args: ['-E', 'setup.ilm.enabled=false', '-E', 'setup.template.enabled=false', '-E', 'management.enabled=true', '-E', 'logging.level=debug'] artifact: beats/packetbeat restart_on_output_change: true rules: