make policy change handler try all fleet hosts before failing (#1329)

It changes the remote client to:
 - when creating a new client:
   - succeed if at least one host is health
   - shuffle the hosts, avoiding all the agents reaching to the same fleet-server on the first request
 - makes `(remote.*Client).Send` try all the hosts before failing, returning a multi-error if all hosts fail
   - if debug logs are enabled, `Send` will log each error with debug level
 - modifies `remote.requestClient`:
   - now `requestClient` holds its host
   - remove `requestFunc`
   - `(remopte.requestClient).newRequest uses the new `host` property to build the final URL for the request

changelog/fragments/1666281194-Fix-how-multiple-Fleet-Server-hosts-are-handled.yaml

@@ -0,0 +1,35 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Fix how multiple Fleet Server hosts are handled
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+description: It fixes the bug when the Elastic Agent would be enrolled using
+  a valid Fleet Server URL, but the policy would contain more than one, being
+  the first URL unreachable. In that case the Elastic Agent would enroll with
+  Fleet Server, but become unhealthy as it'd get stuck trying only the first,
+  unreachable Fleet Server host.
+
+# Affected component; a word indicating the component this changeset affects.
+#component:
+
+# PR number; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: 1329
+
+# Issue number; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: 1234

internal/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"sort"
 	"time"
 
@@ -142,30 +143,35 @@ func (h *PolicyChange) handleFleetServerHosts(ctx context.Context, c *config.Con
 			err, "fail to create API client with updated hosts",
 			errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Client.Hosts))
 	}
+
 	ctx, cancel := context.WithTimeout(ctx, apiStatusTimeout)
 	defer cancel()
-	resp, err := client.Send(ctx, "GET", "/api/status", nil, nil, nil)
+
+	resp, err := client.Send(ctx, http.MethodGet, "/api/status", nil, nil, nil)
 	if err != nil {
 		return errors.New(
-			err, "fail to communicate with updated API client hosts",
+			err, "fail to communicate with Fleet Server API client hosts",
 			errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Client.Hosts))
 	}
+
 	// discard body for proper cancellation and connection reuse
 	_, _ = io.Copy(ioutil.Discard, resp.Body)
 	resp.Body.Close()
 
 	reader, err := fleetToReader(h.agentInfo, h.config)
 	if err != nil {
 		return errors.New(
-			err, "fail to persist updated API client hosts",
+			err, "fail to persist new Fleet Server API client hosts",
 			errors.TypeUnexpected, errors.M("hosts", h.config.Fleet.Client.Hosts))
 	}
+
 	err = h.store.Save(reader)
 	if err != nil {
 		return errors.New(
-			err, "fail to persist updated API client hosts",
+			err, "fail to persist new Fleet Server API client hosts",
 			errors.TypeFilesystem, errors.M("hosts", h.config.Fleet.Client.Hosts))
 	}
+
 	for _, setter := range h.setters {
 		setter.SetClient(client)
 	}

internal/pkg/fleetapi/client/client.go

@@ -87,7 +87,7 @@ func NewWithConfig(log *logger.Logger, cfg remote.Config) (*remote.Client, error
 
 // ExtractError extracts error from a fleet-server response
 func ExtractError(resp io.Reader) error {
-	// Lets try to extract a high level fleet-server error.
+	// Let's try to extract a high level fleet-server error.
 	e := &struct {
 		StatusCode int    `json:"statusCode"`
 		Error      string `json:"error"`