[7.x](backport #1153) feat: check how many processes of a process are…

… running in the host (#1154)

* feat: check how many processes of a process are running in the host (#1153)

* fix: use docker's stdcopy to separate stdout from stderr

This will allow removing the initial bytes when reading outputs from command
execution in a container

* feat: support checking the number of occurrences of a process in a container

It uses pgrep to get all pids for a process, and then iterates through them
to get the runnable status for each pid. If the process must be started
in the host, then it will check that the pid is in the S status (to skip
zombie processes)

* fix: check for only one filebeat instance

* fix: check for empty response when listing agent's workdir

(cherry picked from commit 78a0d49)

* fix: check for 1 filebeat process only

* fix: check for 1 metricbeat process only

Co-authored-by: Manuel de la Peña <mdelapenya@gmail.com>

e2e/_suites/fleet/features/backend_processes.feature

@@ -6,8 +6,8 @@ Feature: Backend Processes
 Scenario Outline: Deploying the <os> agent
   Given a "<os>" agent is deployed to Fleet with "tar" installer
   When the "elastic-agent" process is in the "started" state on the host
-  Then the "filebeat" process is in the "started" state on the host
-    And the "metricbeat" process is in the "started" state on the host
+  Then there are "1" instances of the "filebeat" process in the "started" state
+    And there are "1" instances of the "metricbeat" process in the "started" state
 
 @centos
 Examples: Centos
@@ -23,8 +23,8 @@ Examples: Debian
 Scenario Outline: Deploying the <os> agent with enroll and then run on rpm and deb
   Given a "<os>" agent is deployed to Fleet with "systemd" installer
   When the "elastic-agent" process is in the "started" state on the host
-  Then the "filebeat" process is in the "started" state on the host
-    And the "metricbeat" process is in the "started" state on the host
+  Then there are "1" instances of the "filebeat" process in the "started" state
+    And there are "1" instances of the "metricbeat" process in the "started" state
 
 @centos
 Examples: Centos
@@ -57,8 +57,8 @@ Examples: Debian
 Scenario Outline: Restarting the installed <os> agent
   Given a "<os>" agent is deployed to Fleet with "tar" installer
   When the "elastic-agent" process is "restarted" on the host
-  Then the "filebeat" process is in the "started" state on the host
-    And the "metricbeat" process is in the "started" state on the host
+  Then there are "1" instances of the "filebeat" process in the "started" state
+    And there are "1" instances of the "metricbeat" process in the "started" state
 
 @centos
 Examples: Centos
@@ -75,8 +75,8 @@ Scenario Outline: Restarting the <os> host with persistent agent restarts backen
   Given a "<os>" agent is deployed to Fleet with "tar" installer
   When the host is restarted
   Then the "elastic-agent" process is in the "started" state on the host
-    And the "filebeat" process is in the "started" state on the host
-    And the "metricbeat" process is in the "started" state on the host
+    And there are "1" instances of the "filebeat" process in the "started" state
+    And there are "1" instances of the "metricbeat" process in the "started" state
 
 @centos
 Examples: Centos

e2e/_suites/fleet/features/stand_alone_agent.feature

@@ -7,8 +7,8 @@ Feature: Stand-alone Agent
 @start-agent
 Scenario Outline: Starting the <image> agent starts backend processes
   When a "<image>" stand-alone agent is deployed
-  Then the "filebeat" process is in the "started" state on the host
-    And the "metricbeat" process is in the "started" state on the host
+  Then there are "1" instances of the "filebeat" process in the "started" state
+    And there are "2" instances of the "metricbeat" process in the "started" state
 
 @default
 Examples: default

e2e/_suites/fleet/fleet.go

@@ -461,7 +461,7 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st
 
 	containerName := fts.getContainerName(agentInstaller, 1)
 
-	return docker.CheckProcessStateOnTheHost(containerName, process, "stopped", common.TimeoutFactor)
+	return docker.CheckProcessStateOnTheHost(containerName, process, "stopped", 1, common.TimeoutFactor)
 }
 
 func (fts *FleetTestSuite) setup() error {
@@ -563,7 +563,7 @@ func (fts *FleetTestSuite) theFileSystemAgentFolderIsEmpty() error {
 		return err
 	}
 
-	if strings.Contains(content, "No such file or directory") {
+	if content == "" || strings.Contains(content, "No such file or directory") {
 		return nil
 	}
 

e2e/_suites/fleet/ingest_manager_test.go

@@ -105,6 +105,7 @@ func InitializeIngestManagerTestScenario(ctx *godog.ScenarioContext) {
 	})
 
 	ctx.Step(`^the "([^"]*)" process is in the "([^"]*)" state on the host$`, imts.processStateOnTheHost)
+	ctx.Step(`^there are "([^"]*)" instances of the "([^"]*)" process in the "([^"]*)" state$`, imts.thereAreInstancesOfTheProcessInTheState)
 
 	imts.Fleet.contributeSteps(ctx)
 }

e2e/_suites/fleet/world.go

@@ -6,6 +6,7 @@ package main
 
 import (
 	"fmt"
+	"strconv"
 
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/docker"
@@ -17,6 +18,10 @@ type IngestManagerTestSuite struct {
 }
 
 func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state string) error {
+	return imts.thereAreInstancesOfTheProcessInTheState("1", process, state)
+}
+
+func (imts *IngestManagerTestSuite) thereAreInstancesOfTheProcessInTheState(ocurrences string, process string, state string) error {
 	profile := common.FleetProfileName
 
 	var containerName string
@@ -28,5 +33,10 @@ func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state
 		containerName = imts.Fleet.getContainerName(agentInstaller, 1)
 	}
 
-	return docker.CheckProcessStateOnTheHost(containerName, process, state, common.TimeoutFactor)
+	count, err := strconv.Atoi(ocurrences)
+	if err != nil {
+		return err
+	}
+
+	return docker.CheckProcessStateOnTheHost(containerName, process, state, count, common.TimeoutFactor)
 }