Unify fleet and stand-alone suites (#1112)

* fix agent uninstall

* unify fleet and stand alone suites

* move things around a bit more

* fixe bad merge

* simplify some things

e2e/_suites/fleet/features/apm_integration.feature

@@ -6,8 +6,8 @@ Scenarios for APM
 Scenario Outline: Deploying a <image> stand-alone agent with fleet server mode
   Given a "<image>" stand-alone agent is deployed with fleet server mode
     And the stand-alone agent is listed in Fleet as "online"
-  When the "Elastic APM" integration is added to the policy
-  Then the "Elastic APM" datasource is shown in the policy
+  When the "Elastic APM" integration is added in the policy
+  Then the "Elastic APM" datasource is shown in the policy as added
     And the "apm-server" process is in the "started" state on the host
 
 

e2e/_suites/fleet/fleet.go

@@ -31,7 +31,7 @@ const actionREMOVED = "removed"
 // FleetTestSuite represents the scenarios for Fleet-mode
 type FleetTestSuite struct {
 	// integrations
-	Cleanup             bool
+	StandAlone          bool
 	CurrentToken        string // current enrollment token
 	CurrentTokenID      string // current enrollment tokenID
 	ElasticAgentStopped bool   // will be used to signal when the agent process can be called again in the tear-down stage
@@ -47,32 +47,39 @@ type FleetTestSuite struct {
 	kibanaClient        *kibana.Client
 	// fleet server
 	FleetServerHostname string // hostname of the fleet server. If empty, it means the agent is the first one, bootstrapping fleet server
+	// date controls for queries
+	AgentStoppedDate             time.Time
+	RuntimeDependenciesStartDate time.Time
 }
 
 // afterScenario destroys the state created by a scenario
 func (fts *FleetTestSuite) afterScenario() {
-	serviceManager := compose.NewServiceManager()
 
-	agentInstaller := fts.getInstaller()
+	serviceName := common.ElasticAgentServiceName
+	serviceManager := compose.NewServiceManager()
 
-	serviceName := fts.getServiceName(agentInstaller)
+	if !fts.StandAlone {
+		agentInstaller := fts.getInstaller()
+		serviceName = fts.getServiceName(agentInstaller)
 
-	if log.IsLevelEnabled(log.DebugLevel) {
-		err := agentInstaller.PrintLogsFn(fts.Hostname)
-		if err != nil {
-			log.WithFields(log.Fields{
-				"containerName": fts.Hostname,
-				"error":         err,
-			}).Warn("Could not get agent logs in the container")
+		if log.IsLevelEnabled(log.DebugLevel) {
+			err := agentInstaller.PrintLogsFn(fts.Hostname)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"containerName": fts.Hostname,
+					"error":         err,
+				}).Warn("Could not get agent logs in the container")
+			}
 		}
-	}
-
-	// only call it when the elastic-agent is present
-	if !fts.ElasticAgentStopped {
-		err := agentInstaller.UninstallFn()
-		if err != nil {
-			log.Warnf("Could not uninstall the agent after the scenario: %v", err)
+		// only call it when the elastic-agent is present
+		if !fts.ElasticAgentStopped {
+			err := agentInstaller.UninstallFn()
+			if err != nil {
+				log.Warnf("Could not uninstall the agent after the scenario: %v", err)
+			}
 		}
+	} else if log.IsLevelEnabled(log.DebugLevel) {
+		_ = fts.getContainerLogs()
 	}
 
 	err := fts.unenrollHostname()
@@ -106,11 +113,12 @@ func (fts *FleetTestSuite) afterScenario() {
 	fts.Image = ""
 	fts.Hostname = ""
 	fts.FleetServerHostname = ""
+	fts.StandAlone = false
 }
 
 // beforeScenario creates the state needed by a scenario
 func (fts *FleetTestSuite) beforeScenario() {
-	fts.Cleanup = false
+	fts.StandAlone = false
 	fts.ElasticAgentStopped = false
 
 	fts.Version = common.AgentVersion
@@ -160,6 +168,41 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
 	s.Step(`^the policy response will be shown in the Security App$`, fts.thePolicyResponseWillBeShownInTheSecurityApp)
 	s.Step(`^the policy is updated to have "([^"]*)" in "([^"]*)" mode$`, fts.thePolicyIsUpdatedToHaveMode)
 	s.Step(`^the policy will reflect the change in the Security App$`, fts.thePolicyWillReflectTheChangeInTheSecurityApp)
+
+	// stand-alone only steps
+	s.Step(`^a "([^"]*)" stand-alone agent is deployed$`, fts.aStandaloneAgentIsDeployed)
+	s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode$`, fts.bootstrapFleetServerFromAStandaloneAgent)
+	s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode on cloud$`, fts.aStandaloneAgentIsDeployedWithFleetServerModeOnCloud)
+	s.Step(`^there is new data in the index from agent$`, fts.thereIsNewDataInTheIndexFromAgent)
+	s.Step(`^the "([^"]*)" docker container is stopped$`, fts.theDockerContainerIsStopped)
+	s.Step(`^there is no new data in the index after agent shuts down$`, fts.thereIsNoNewDataInTheIndexAfterAgentShutsDown)
+	s.Step(`^the stand-alone agent is listed in Fleet as "([^"]*)"$`, fts.theStandaloneAgentIsListedInFleetWithStatus)
+}
+
+func (fts *FleetTestSuite) theStandaloneAgentIsListedInFleetWithStatus(desiredStatus string) error {
+	waitForAgents := func() error {
+		agents, err := fts.kibanaClient.ListAgents()
+		if err != nil {
+			return err
+		}
+
+		if len(agents) == 0 {
+			return errors.New("No agents found")
+		}
+
+		agentZero := agents[0]
+		hostname := agentZero.LocalMetadata.Host.HostName
+
+		return theAgentIsListedInFleetWithStatus(desiredStatus, hostname)
+	}
+	maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2
+	exp := common.GetExponentialBackOff(maxTimeout)
+
+	err := backoff.Retry(waitForAgents, exp)
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error {
@@ -310,7 +353,6 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i
 	var fleetConfig *kibana.FleetConfig
 	fleetConfig, err = deployAgentToFleet(agentInstaller, containerName, fts.CurrentToken, fts.FleetServerHostname)
 
-	fts.Cleanup = true
 	if err != nil {
 		return err
 	}
@@ -665,46 +707,6 @@ func (fts *FleetTestSuite) theEnrollmentTokenIsRevoked() error {
 	return nil
 }
 
-func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) error {
-	return thePolicyShowsTheDatasourceAdded(fts.kibanaClient, fts.FleetServerPolicy, packageName)
-}
-
-func thePolicyShowsTheDatasourceAdded(client *kibana.Client, policy kibana.Policy, packageName string) error {
-	log.WithFields(log.Fields{
-		"policyID": policy.ID,
-		"package":  packageName,
-	}).Trace("Checking if the policy shows the package added")
-
-	maxTimeout := time.Minute
-	retryCount := 1
-
-	exp := common.GetExponentialBackOff(maxTimeout)
-
-	configurationIsPresentFn := func() error {
-		packagePolicy, err := client.GetIntegrationFromAgentPolicy(packageName, policy)
-		if err != nil {
-			log.WithFields(log.Fields{
-				"packagePolicy": packagePolicy,
-				"policy":        policy,
-				"retry":         retryCount,
-				"error":         err,
-			}).Warn("The integration was not found in the policy")
-			retryCount++
-			return err
-		}
-
-		retryCount++
-		return err
-	}
-
-	err := backoff.Retry(configurationIsPresentFn, exp)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (fts *FleetTestSuite) theIntegrationIsOperatedInThePolicy(packageName string, action string) error {
 	return theIntegrationIsOperatedInThePolicy(fts.kibanaClient, fts.FleetServerPolicy, packageName, action)
 }
@@ -1227,3 +1229,26 @@ func inputs(integration string) []kibana.Input {
 	}
 	return []kibana.Input{}
 }
+
+func (fts *FleetTestSuite) getContainerLogs() error {
+	serviceManager := compose.NewServiceManager()
+
+	profile := common.FleetProfileName
+	serviceName := common.ElasticAgentServiceName
+
+	composes := []string{
+		profile,     // profile name
+		serviceName, // agent service
+	}
+	err := serviceManager.RunCommand(profile, composes, []string{"logs", serviceName}, common.ProfileEnv)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":   err,
+			"service": serviceName,
+		}).Error("Could not retrieve Elastic Agent logs")
+
+		return err
+	}
+
+	return nil
+}

e2e/_suites/fleet/ingest_manager_test.go

@@ -89,37 +89,23 @@ func setUpSuite() {
 			kibanaClient: kibanaClient,
 			Installers:   map[string]installer.ElasticAgentInstaller{}, // do not pre-initialise the map
 		},
-		StandAlone: &StandAloneTestSuite{
-			kibanaClient: kibanaClient,
-		},
 	}
 }
 
 func InitializeIngestManagerTestScenario(ctx *godog.ScenarioContext) {
 	ctx.BeforeScenario(func(*messages.Pickle) {
 		log.Trace("Before Fleet scenario")
-
-		imts.StandAlone.Cleanup = false
-
 		imts.Fleet.beforeScenario()
 	})
 
 	ctx.AfterScenario(func(*messages.Pickle, error) {
 		log.Trace("After Fleet scenario")
-
-		if imts.StandAlone.Cleanup {
-			imts.StandAlone.afterScenario()
-		}
-
-		if imts.Fleet.Cleanup {
-			imts.Fleet.afterScenario()
-		}
+		imts.Fleet.afterScenario()
 	})
 
 	ctx.Step(`^the "([^"]*)" process is in the "([^"]*)" state on the host$`, imts.processStateOnTheHost)
 
 	imts.Fleet.contributeSteps(ctx)
-	imts.StandAlone.contributeSteps(ctx)
 }
 
 func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) {
@@ -181,7 +167,7 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) {
 
 		imts.Fleet.setup()
 
-		imts.StandAlone.RuntimeDependenciesStartDate = time.Now().UTC()
+		imts.Fleet.RuntimeDependenciesStartDate = time.Now().UTC()
 	})
 
 	ctx.AfterSuite(func() {