diff --git a/e2e/_suites/fleet/features/apm_integration.feature b/e2e/_suites/fleet/features/apm_integration.feature new file mode 100644 index 0000000000..cff83f53d7 --- /dev/null +++ b/e2e/_suites/fleet/features/apm_integration.feature @@ -0,0 +1,30 @@ +@apm_server +Feature: APM Integration +Scenarios for APM + +@install +Scenario Outline: Deploying a stand-alone agent with fleet server mode + Given a "" stand-alone agent is deployed with fleet server mode + And the stand-alone agent is listed in Fleet as "online" + When the "Elastic APM" integration is added in the policy + Then the "Elastic APM" datasource is shown in the policy as added + And the "apm-server" process is in the "started" state on the host + + +@default +Examples: default + | image | + | default | + + + +@cloud +Scenario Outline: Deploying a stand-alone agent with fleet server mode on cloud + When a "" stand-alone agent is deployed with fleet server mode on cloud + Then the "apm-server" process is in the "started" state on the host + + +@default +Examples: default + | image | + | default | diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index f632d9a21f..c6aba70195 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -31,7 +31,7 @@ const actionREMOVED = "removed" // FleetTestSuite represents the scenarios for Fleet-mode type FleetTestSuite struct { // integrations - Cleanup bool + StandAlone bool CurrentToken string // current enrollment token CurrentTokenID string // current enrollment tokenID ElasticAgentStopped bool // will be used to signal when the agent process can be called again in the tear-down stage @@ -45,12 +45,20 @@ type FleetTestSuite struct { PolicyUpdatedAt string // the moment the policy was updated Version string // current elastic-agent version kibanaClient *kibana.Client +<<<<<<< HEAD +======= + // fleet server + FleetServerHostname string // hostname of the fleet server. If empty, it means the agent is the first one, bootstrapping fleet server + // date controls for queries + AgentStoppedDate time.Time + RuntimeDependenciesStartDate time.Time +>>>>>>> 77a2c554... Unify fleet and stand-alone suites (#1112) } // afterScenario destroys the state created by a scenario func (fts *FleetTestSuite) afterScenario() { - serviceManager := compose.NewServiceManager() +<<<<<<< HEAD serviceName := fts.Image if log.IsLevelEnabled(log.DebugLevel) { @@ -64,6 +72,24 @@ func (fts *FleetTestSuite) afterScenario() { }).Warn("Could not get agent logs in the container") } +======= + serviceName := common.ElasticAgentServiceName + serviceManager := compose.NewServiceManager() + + if !fts.StandAlone { + agentInstaller := fts.getInstaller() + serviceName = fts.getServiceName(agentInstaller) + + if log.IsLevelEnabled(log.DebugLevel) { + err := agentInstaller.PrintLogsFn(fts.Hostname) + if err != nil { + log.WithFields(log.Fields{ + "containerName": fts.Hostname, + "error": err, + }).Warn("Could not get agent logs in the container") + } + } +>>>>>>> 77a2c554... Unify fleet and stand-alone suites (#1112) // only call it when the elastic-agent is present if !fts.ElasticAgentStopped { err := agentInstaller.UninstallFn() @@ -71,6 +97,8 @@ func (fts *FleetTestSuite) afterScenario() { log.Warnf("Could not uninstall the agent after the scenario: %v", err) } } + } else if log.IsLevelEnabled(log.DebugLevel) { + _ = fts.getContainerLogs() } err := fts.unenrollHostname() @@ -122,11 +150,16 @@ func (fts *FleetTestSuite) afterScenario() { fts.CurrentToken = "" fts.Image = "" fts.Hostname = "" +<<<<<<< HEAD +======= + fts.FleetServerHostname = "" + fts.StandAlone = false +>>>>>>> 77a2c554... Unify fleet and stand-alone suites (#1112) } // beforeScenario creates the state needed by a scenario func (fts *FleetTestSuite) beforeScenario() { - fts.Cleanup = false + fts.StandAlone = false fts.ElasticAgentStopped = false fts.Version = common.AgentVersion @@ -177,8 +210,45 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) { s.Step(`^the policy is updated to have "([^"]*)" in "([^"]*)" mode$`, fts.thePolicyIsUpdatedToHaveMode) s.Step(`^the policy will reflect the change in the Security App$`, fts.thePolicyWillReflectTheChangeInTheSecurityApp) +<<<<<<< HEAD // fleet server steps s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer in fleet-server mode$`, fts.anAgentIsDeployedToFleetWithInstallerInFleetMode) +======= + // stand-alone only steps + s.Step(`^a "([^"]*)" stand-alone agent is deployed$`, fts.aStandaloneAgentIsDeployed) + s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode$`, fts.bootstrapFleetServerFromAStandaloneAgent) + s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode on cloud$`, fts.aStandaloneAgentIsDeployedWithFleetServerModeOnCloud) + s.Step(`^there is new data in the index from agent$`, fts.thereIsNewDataInTheIndexFromAgent) + s.Step(`^the "([^"]*)" docker container is stopped$`, fts.theDockerContainerIsStopped) + s.Step(`^there is no new data in the index after agent shuts down$`, fts.thereIsNoNewDataInTheIndexAfterAgentShutsDown) + s.Step(`^the stand-alone agent is listed in Fleet as "([^"]*)"$`, fts.theStandaloneAgentIsListedInFleetWithStatus) +} + +func (fts *FleetTestSuite) theStandaloneAgentIsListedInFleetWithStatus(desiredStatus string) error { + waitForAgents := func() error { + agents, err := fts.kibanaClient.ListAgents() + if err != nil { + return err + } + + if len(agents) == 0 { + return errors.New("No agents found") + } + + agentZero := agents[0] + hostname := agentZero.LocalMetadata.Host.HostName + + return theAgentIsListedInFleetWithStatus(desiredStatus, hostname) + } + maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + exp := common.GetExponentialBackOff(maxTimeout) + + err := backoff.Retry(waitForAgents, exp) + if err != nil { + return err + } + return nil +>>>>>>> 77a2c554... Unify fleet and stand-alone suites (#1112) } func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error { @@ -327,7 +397,6 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i var fleetConfig *kibana.FleetConfig fleetConfig, err = deployAgentToFleet(agentInstaller, containerName, fts.CurrentToken, bootstrapFleetServer) - fts.Cleanup = true if err != nil { return err } @@ -678,6 +747,7 @@ func (fts *FleetTestSuite) theEnrollmentTokenIsRevoked() error { return nil } +<<<<<<< HEAD func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) error { log.WithFields(log.Fields{ "policyID": fts.FleetPolicy.ID, @@ -714,6 +784,8 @@ func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) return nil } +======= +>>>>>>> 77a2c554... Unify fleet and stand-alone suites (#1112) func (fts *FleetTestSuite) theIntegrationIsOperatedInThePolicy(packageName string, action string) error { log.WithFields(log.Fields{ "action": action, @@ -1220,3 +1292,72 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe return cfg, agentInstaller.PostInstallFn() } +<<<<<<< HEAD +======= + +func inputs(integration string) []kibana.Input { + switch integration { + case "apm": + return []kibana.Input{ + { + Type: "apm", + Enabled: true, + Streams: []interface{}{}, + Vars: map[string]kibana.Var{ + "apm-server": { + Value: "host", + Type: "localhost:8200", + }, + }, + }, + } + case "linux": + return []kibana.Input{ + { + Type: "linux/metrics", + Enabled: true, + Streams: []interface{}{ + map[string]interface{}{ + "id": "linux/metrics-linux.memory-" + uuid.New().String(), + "enabled": true, + "data_stream": map[string]interface{}{ + "dataset": "linux.memory", + "type": "metrics", + }, + }, + }, + Vars: map[string]kibana.Var{ + "period": { + Value: "1s", + Type: "string", + }, + }, + }, + } + } + return []kibana.Input{} +} + +func (fts *FleetTestSuite) getContainerLogs() error { + serviceManager := compose.NewServiceManager() + + profile := common.FleetProfileName + serviceName := common.ElasticAgentServiceName + + composes := []string{ + profile, // profile name + serviceName, // agent service + } + err := serviceManager.RunCommand(profile, composes, []string{"logs", serviceName}, common.ProfileEnv) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "service": serviceName, + }).Error("Could not retrieve Elastic Agent logs") + + return err + } + + return nil +} +>>>>>>> 77a2c554... Unify fleet and stand-alone suites (#1112) diff --git a/e2e/_suites/fleet/ingest_manager_test.go b/e2e/_suites/fleet/ingest_manager_test.go index 44c748d234..f2f006f923 100644 --- a/e2e/_suites/fleet/ingest_manager_test.go +++ b/e2e/_suites/fleet/ingest_manager_test.go @@ -89,37 +89,23 @@ func setUpSuite() { kibanaClient: kibanaClient, Installers: map[string]installer.ElasticAgentInstaller{}, // do not pre-initialise the map }, - StandAlone: &StandAloneTestSuite{ - kibanaClient: kibanaClient, - }, } } func InitializeIngestManagerTestScenario(ctx *godog.ScenarioContext) { ctx.BeforeScenario(func(*messages.Pickle) { log.Trace("Before Fleet scenario") - - imts.StandAlone.Cleanup = false - imts.Fleet.beforeScenario() }) ctx.AfterScenario(func(*messages.Pickle, error) { log.Trace("After Fleet scenario") - - if imts.StandAlone.Cleanup { - imts.StandAlone.afterScenario() - } - - if imts.Fleet.Cleanup { - imts.Fleet.afterScenario() - } + imts.Fleet.afterScenario() }) ctx.Step(`^the "([^"]*)" process is in the "([^"]*)" state on the host$`, imts.processStateOnTheHost) imts.Fleet.contributeSteps(ctx) - imts.StandAlone.contributeSteps(ctx) } func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) { @@ -175,7 +161,7 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) { imts.Fleet.setup() - imts.StandAlone.RuntimeDependenciesStartDate = time.Now().UTC() + imts.Fleet.RuntimeDependenciesStartDate = time.Now().UTC() }) ctx.AfterSuite(func() { diff --git a/e2e/_suites/fleet/stand-alone.go b/e2e/_suites/fleet/stand-alone.go index fd4d723cfe..2a19df1d63 100644 --- a/e2e/_suites/fleet/stand-alone.go +++ b/e2e/_suites/fleet/stand-alone.go @@ -7,23 +7,29 @@ package main import ( "context" "fmt" +<<<<<<< HEAD "strings" "time" +======= +>>>>>>> 77a2c554... Unify fleet and stand-alone suites (#1112) "github.com/cenkalti/backoff/v4" - "github.com/cucumber/godog" + "github.com/elastic/e2e-testing/cli/config" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/compose" "github.com/elastic/e2e-testing/internal/docker" - "github.com/elastic/e2e-testing/internal/elasticsearch" "github.com/elastic/e2e-testing/internal/installer" - "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" - "github.com/pkg/errors" + "path" + "strings" + "time" + + "github.com/elastic/e2e-testing/internal/elasticsearch" log "github.com/sirupsen/logrus" ) +<<<<<<< HEAD // StandAloneTestSuite represents the scenarios for Stand-alone-mode type StandAloneTestSuite struct { Cleanup bool @@ -33,23 +39,18 @@ type StandAloneTestSuite struct { AgentStoppedDate time.Time RuntimeDependenciesStartDate time.Time kibanaClient *kibana.Client +======= +func (fts *FleetTestSuite) aStandaloneAgentIsDeployed(image string) error { + return fts.startStandAloneAgent(image, "", nil) +>>>>>>> 77a2c554... Unify fleet and stand-alone suites (#1112) } -// afterScenario destroys the state created by a scenario -func (sats *StandAloneTestSuite) afterScenario() { - serviceManager := compose.NewServiceManager() - serviceName := common.ElasticAgentServiceName - - if log.IsLevelEnabled(log.DebugLevel) { - _ = sats.getContainerLogs() - } - - developerMode := shell.GetEnvBool("DEVELOPER_MODE") - if !developerMode { - _ = serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv) - } else { - log.WithField("service", serviceName).Info("Because we are running in development mode, the service won't be stopped") +func (fts *FleetTestSuite) bootstrapFleetServerFromAStandaloneAgent(image string) error { + fleetPolicy, err := fts.kibanaClient.GetDefaultPolicy(true) + if err != nil { + return err } +<<<<<<< HEAD } func (sats *StandAloneTestSuite) contributeSteps(s *godog.ScenarioContext) { @@ -59,34 +60,49 @@ func (sats *StandAloneTestSuite) contributeSteps(s *godog.ScenarioContext) { s.Step(`^the "([^"]*)" docker container is stopped$`, sats.theDockerContainerIsStopped) s.Step(`^there is no new data in the index after agent shuts down$`, sats.thereIsNoNewDataInTheIndexAfterAgentShutsDown) s.Step(`^the stand-alone agent is listed in Fleet as "([^"]*)"$`, sats.theStandaloneAgentIsListedInFleetWithStatus) +======= + fts.FleetServerPolicy = fleetPolicy + return fts.startStandAloneAgent(image, "", map[string]string{"fleetServerMode": "1"}) } -func (sats *StandAloneTestSuite) theStandaloneAgentIsListedInFleetWithStatus(desiredStatus string) error { - waitForAgents := func() error { - agents, err := sats.kibanaClient.ListAgents() - if err != nil { - return err - } - - if len(agents) == 0 { - return errors.New("No agents found") - } +func (fts *FleetTestSuite) aStandaloneAgentIsDeployedWithFleetServerModeOnCloud(image string) error { + fleetPolicy, err := fts.kibanaClient.GetDefaultPolicy(true) + if err != nil { + return err + } + fts.FleetServerPolicy = fleetPolicy + volume := path.Join(config.OpDir(), "compose", "services", "elastic-agent", "apm-legacy") + return fts.startStandAloneAgent(image, "docker-compose-cloud.yml", map[string]string{"apmVolume": volume}) +>>>>>>> 77a2c554... Unify fleet and stand-alone suites (#1112) +} - agentZero := agents[0] - hostname := agentZero.LocalMetadata.Host.HostName +func (fts *FleetTestSuite) thereIsNewDataInTheIndexFromAgent() error { + maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + minimumHitsCount := 50 - return theAgentIsListedInFleetWithStatus(desiredStatus, hostname) + result, err := searchAgentData(fts.Hostname, fts.RuntimeDependenciesStartDate, minimumHitsCount, maxTimeout) + if err != nil { + return err } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) - err := backoff.Retry(waitForAgents, exp) + log.Tracef("Search result: %v", result) + + return elasticsearch.AssertHitsArePresent(result) +} + +func (fts *FleetTestSuite) theDockerContainerIsStopped(serviceName string) error { + serviceManager := compose.NewServiceManager() + + err := serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv) if err != nil { return err } + fts.AgentStoppedDate = time.Now().UTC() + return nil } +<<<<<<< HEAD func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployedWithFleetServerMode(image string) error { return sats.startAgent(image, map[string]string{"fleetServerMode": "1"}) } @@ -97,6 +113,29 @@ func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployed(image string) error func (sats *StandAloneTestSuite) startAgent(image string, env map[string]string) error { +======= +func (fts *FleetTestSuite) thereIsNoNewDataInTheIndexAfterAgentShutsDown() error { + maxTimeout := time.Duration(30) * time.Second + minimumHitsCount := 1 + + result, err := searchAgentData(fts.Hostname, fts.AgentStoppedDate, minimumHitsCount, maxTimeout) + if err != nil { + if strings.Contains(err.Error(), "type:index_not_found_exception") { + return err + } + + log.WithFields(log.Fields{ + "error": err, + }).Info("No documents were found for the Agent in the index after it stopped") + return nil + } + + return elasticsearch.AssertHitsAreNotPresent(result) +} + +func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename string, env map[string]string) error { + fts.StandAlone = true +>>>>>>> 77a2c554... Unify fleet and stand-alone suites (#1112) log.Trace("Deploying an agent to Fleet") dockerImageTag := common.AgentVersion @@ -145,11 +184,10 @@ func (sats *StandAloneTestSuite) startAgent(image string, env map[string]string) return err } - sats.Image = image - sats.Hostname = hostname - sats.Cleanup = true + fts.Image = image + fts.Hostname = hostname - err = sats.installTestTools(containerName) + err = fts.installTestTools(containerName) if err != nil { return err } @@ -157,23 +195,36 @@ func (sats *StandAloneTestSuite) startAgent(image string, env map[string]string) return nil } -func (sats *StandAloneTestSuite) getContainerLogs() error { - serviceManager := compose.NewServiceManager() +func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) error { + log.WithFields(log.Fields{ + "policyID": fts.FleetServerPolicy.ID, + "package": packageName, + }).Trace("Checking if the policy shows the package added") - profile := common.FleetProfileName - serviceName := common.ElasticAgentServiceName + maxTimeout := time.Minute + retryCount := 1 + + exp := common.GetExponentialBackOff(maxTimeout) - composes := []string{ - profile, // profile name - serviceName, // agent service + configurationIsPresentFn := func() error { + packagePolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy(packageName, fts.FleetServerPolicy) + if err != nil { + log.WithFields(log.Fields{ + "packagePolicy": packagePolicy, + "policy": fts.FleetServerPolicy, + "retry": retryCount, + "error": err, + }).Warn("The integration was not found in the policy") + retryCount++ + return err + } + + retryCount++ + return err } - err := serviceManager.RunCommand(profile, composes, []string{"logs", serviceName}, common.ProfileEnv) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "service": serviceName, - }).Error("Could not retrieve Elastic Agent logs") + err := backoff.Retry(configurationIsPresentFn, exp) + if err != nil { return err } @@ -183,8 +234,8 @@ func (sats *StandAloneTestSuite) getContainerLogs() error { // installTestTools we need the container name because we use the Docker Client instead of Docker Compose // we are going to install those tools we use in the test framework for checking // and verifications -func (sats *StandAloneTestSuite) installTestTools(containerName string) error { - if sats.Image != "ubi8" { +func (fts *FleetTestSuite) installTestTools(containerName string) error { + if fts.Image != "ubi8" { return nil } @@ -213,51 +264,6 @@ func (sats *StandAloneTestSuite) installTestTools(containerName string) error { return nil } -func (sats *StandAloneTestSuite) thereIsNewDataInTheIndexFromAgent() error { - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - minimumHitsCount := 50 - - result, err := searchAgentData(sats.Hostname, sats.RuntimeDependenciesStartDate, minimumHitsCount, maxTimeout) - if err != nil { - return err - } - - log.Tracef("Search result: %v", result) - - return elasticsearch.AssertHitsArePresent(result) -} - -func (sats *StandAloneTestSuite) theDockerContainerIsStopped(serviceName string) error { - serviceManager := compose.NewServiceManager() - - err := serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv) - if err != nil { - return err - } - sats.AgentStoppedDate = time.Now().UTC() - - return nil -} - -func (sats *StandAloneTestSuite) thereIsNoNewDataInTheIndexAfterAgentShutsDown() error { - maxTimeout := time.Duration(30) * time.Second - minimumHitsCount := 1 - - result, err := searchAgentData(sats.Hostname, sats.AgentStoppedDate, minimumHitsCount, maxTimeout) - if err != nil { - if strings.Contains(err.Error(), "type:index_not_found_exception") { - return err - } - - log.WithFields(log.Fields{ - "error": err, - }).Info("No documents were found for the Agent in the index after it stopped") - return nil - } - - return elasticsearch.AssertHitsAreNotPresent(result) -} - func searchAgentData(hostname string, startDate time.Time, minimumHitsCount int, maxTimeout time.Duration) (elasticsearch.SearchResult, error) { timezone := "America/New_York" diff --git a/e2e/_suites/fleet/world.go b/e2e/_suites/fleet/world.go index 6fa758a44f..69f73f41ec 100644 --- a/e2e/_suites/fleet/world.go +++ b/e2e/_suites/fleet/world.go @@ -13,17 +13,26 @@ import ( // IngestManagerTestSuite represents a test suite, holding references to the pieces needed to run the tests type IngestManagerTestSuite struct { - Fleet *FleetTestSuite - StandAlone *StandAloneTestSuite + Fleet *FleetTestSuite } func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state string) error { profile := common.FleetProfileName serviceName := common.ElasticAgentServiceName +<<<<<<< HEAD containerName := fmt.Sprintf("%s_%s_%s_%d", profile, imts.Fleet.Image+"-systemd", serviceName, 1) if imts.StandAlone.Hostname != "" { containerName = fmt.Sprintf("%s_%s_%d", profile, serviceName, 1) +======= + var containerName string + + if imts.Fleet.StandAlone { + containerName = fmt.Sprintf("%s_%s_%d", profile, common.ElasticAgentServiceName, 1) + } else { + agentInstaller := imts.Fleet.getInstaller() + containerName = imts.Fleet.getContainerName(agentInstaller, 1) +>>>>>>> 77a2c554... Unify fleet and stand-alone suites (#1112) } return docker.CheckProcessStateOnTheHost(containerName, process, state, common.TimeoutFactor)