Recreate Elasticsearch client in ContinuousHealthCheck (#1928)

test/e2e/es/mutation_test.go

@@ -31,7 +31,7 @@ func TestMutationHTTPToHTTPS(t *testing.T) {
 // then mutates it to a 3 node cluster running without TLS on the HTTP layer.
 func TestMutationHTTPSToHTTP(t *testing.T) {
 	// create a 3 md node cluster
-	b := elasticsearch.NewBuilder("test-mutation-http-to-https").
+	b := elasticsearch.NewBuilder("test-mutation-https-to-http").
 		WithESMasterDataNodes(3, elasticsearch.DefaultResources)
 
 	// mutate to http

test/e2e/test/elasticsearch/steps_mutation.go

@@ -152,24 +152,22 @@ type ContinuousHealthCheckFailure struct {
 // ContinuousHealthCheck continuously runs health checks against Elasticsearch
 // during the whole mutation process
 type ContinuousHealthCheck struct {
-	b            Builder
-	SuccessCount int
-	FailureCount int
-	Failures     []ContinuousHealthCheckFailure
-	stopChan     chan struct{}
-	esClient     esclient.Client
+	b               Builder
+	SuccessCount    int
+	FailureCount    int
+	Failures        []ContinuousHealthCheckFailure
+	stopChan        chan struct{}
+	esClientFactory func() (esclient.Client, error)
 }
 
 // NewContinuousHealthCheck sets up a ContinuousHealthCheck struct
 func NewContinuousHealthCheck(b Builder, k *test.K8sClient) (*ContinuousHealthCheck, error) {
-	esClient, err := NewElasticsearchClient(b.Elasticsearch, k)
-	if err != nil {
-		return nil, err
-	}
 	return &ContinuousHealthCheck{
 		b:        b,
 		stopChan: make(chan struct{}),
-		esClient: esClient,
+		esClientFactory: func() (esclient.Client, error) {
+			return NewElasticsearchClient(b.Elasticsearch, k)
+		},
 	}, nil
 }
 
@@ -193,8 +191,14 @@ func (hc *ContinuousHealthCheck) Start() {
 				return
 			case <-ticker.C:
 				ctx, cancel := context.WithTimeout(context.Background(), continuousHealthCheckTimeout)
+				// recreate the Elasticsearch client at each iteration, since we may have switched protocol from http to https during the mutation
+				client, err := hc.esClientFactory()
+				if err != nil {
+					// treat client creation failure same as unavailable cluster
+					hc.AppendErr(err)
+				}
 				defer cancel()
-				health, err := hc.esClient.GetClusterHealth(ctx)
+				health, err := client.GetClusterHealth(ctx)
 				if err != nil {
 					// Could not retrieve cluster health, can happen when the master node is killed
 					// during a rolling upgrade. We allow it, unless it lasts for too long.