Prevent StatefulSet rolling updates on endpoint certificate updates. F…

…ixes #2125

controllers/constants/constants.go

@@ -45,11 +45,18 @@ var (
 	JGroupsFastMerge = strings.ToUpper(GetEnvWithDefault("TEST_ENVIRONMENT", "false")) == "TRUE"
 
 	// 14.0.24.Final required to enable expiry in Gossip Router
-	GossipRouterHeartBeatMinVersion = semver.Version{
+	MinVersionGossipRouterHeartBeat = semver.Version{
 		Major: 14,
 		Minor: 0,
 		Patch: 24,
 	}
+
+	// 15.0.7.Final required to support the automatic reloading of TLS certs
+	MinVersionAutomaticCertificateReloading = semver.Version{
+		Major: 15,
+		Minor: 0,
+		Patch: 7,
+	}
 )
 
 const (

pkg/reconcile/pipeline/infinispan/handler/manage/statefulset_updates.go

@@ -181,11 +181,16 @@ func StatefulSetRollingUpgrade(i *ispnv1.Infinispan, ctx pipeline.Context) {
 	}
 
 	if i.IsEncryptionEnabled() {
-		provision.AddVolumesForEncryption(i, spec)
-		updateNeeded = updateStatefulSetEnv(container, statefulSet, "KEYSTORE_HASH", hash.HashByte(configFiles.Keystore.PemFile)+hash.HashByte(configFiles.Keystore.File)) || updateNeeded
+		updateNeeded = provision.AddVolumesForEncryption(i, spec) || updateNeeded
 
-		if i.IsClientCertEnabled() {
-			updateNeeded = updateStatefulSetEnv(container, statefulSet, "TRUSTSTORE_HASH", hash.HashByte(configFiles.Truststore.File)) || updateNeeded
+		// Only trigger a StatefulSet rolling upgrade for Keystore and Truststore updates from 15.0.7 onwards as
+		// Infinispan and JGroups automatically reload certificate changes
+		if requestedOperand.UpstreamVersion.LT(consts.MinVersionAutomaticCertificateReloading) {
+			updateNeeded = updateStatefulSetEnv(container, statefulSet, "KEYSTORE_HASH", hash.HashByte(configFiles.Keystore.PemFile)+hash.HashByte(configFiles.Keystore.File)) || updateNeeded
+
+			if i.IsClientCertEnabled() {
+				updateNeeded = updateStatefulSetEnv(container, statefulSet, "TRUSTSTORE_HASH", hash.HashByte(configFiles.Truststore.File)) || updateNeeded
+			}
 		}
 	}
 
@@ -224,6 +229,7 @@ func updateStatefulSetEnv(ispnContainer *corev1.Container, statefulSet *appsv1.S
 	}
 	return false
 }
+
 func updateStartupArgs(ispnContainer *corev1.Container, config *pipeline.ConfigFiles) (bool, error) {
 	newArgs := provision.BuildServerContainerArgs(config)
 	if len(newArgs) == len(ispnContainer.Args) {

pkg/reconcile/pipeline/infinispan/handler/provision/gossiprouter.go

@@ -72,7 +72,7 @@ func GossipRouter(i *ispnv1.Infinispan, ctx pipeline.Context) {
 		}
 
 		// arguments available since 14.0.24.Final
-		if upstreamVersion.GTE(consts.GossipRouterHeartBeatMinVersion) {
+		if upstreamVersion.GTE(consts.MinVersionGossipRouterHeartBeat) {
 			if *i.Spec.Service.Sites.Local.Discovery.Heartbeats.Enabled {
 				args = append(args, []string{
 					"-expiry", strconv.FormatInt(*i.Spec.Service.Sites.Local.Discovery.Heartbeats.Timeout, 10),

pkg/reconcile/pipeline/infinispan/handler/provision/pods.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"reflect"
 
 	"github.com/blang/semver"
 	ispnv1 "github.com/infinispan/infinispan-operator/api/v1"
@@ -202,16 +203,17 @@ func chmodInitContainer(containerName, volumeName, mountPath string) corev1.Cont
 	}
 }
 
-func AddVolumesForEncryption(i *ispnv1.Infinispan, spec *corev1.PodSpec) {
-	AddSecretVolume(i.GetKeystoreSecretName(), EncryptKeystoreVolumeName, consts.ServerEncryptKeystoreRoot, spec, InfinispanContainer)
+func AddVolumesForEncryption(i *ispnv1.Infinispan, spec *corev1.PodSpec) bool {
+	updated := AddSecretVolume(i.GetKeystoreSecretName(), EncryptKeystoreVolumeName, consts.ServerEncryptKeystoreRoot, spec, InfinispanContainer)
 
 	if i.IsClientCertEnabled() {
-		AddSecretVolume(i.GetTruststoreSecretName(), EncryptTruststoreVolumeName, consts.ServerEncryptTruststoreRoot, spec, InfinispanContainer)
+		updated = AddSecretVolume(i.GetTruststoreSecretName(), EncryptTruststoreVolumeName, consts.ServerEncryptTruststoreRoot, spec, InfinispanContainer) || updated
 	}
+	return updated
 }
 
 // AddSecretVolume creates a volume to a secret
-func AddSecretVolume(secretName, volumeName, mountPath string, spec *corev1.PodSpec, containerName string) {
+func AddSecretVolume(secretName, volumeName, mountPath string, spec *corev1.PodSpec, containerName string) (updated bool) {
 	v := &spec.Volumes
 
 	if _, index := findSecretInVolume(spec, volumeName); index < 0 {
@@ -222,6 +224,7 @@ func AddSecretVolume(secretName, volumeName, mountPath string, spec *corev1.PodS
 				},
 			},
 		})
+		updated = true
 	}
 
 	volumeMount := corev1.VolumeMount{
@@ -240,9 +243,12 @@ func AddSecretVolume(secretName, volumeName, mountPath string, spec *corev1.PodS
 
 	if index < 0 {
 		*volumeMounts = append(*volumeMounts, volumeMount)
+		updated = true
 	} else {
+		updated = reflect.DeepEqual((*volumeMounts)[index], volumeMount) || updated
 		(*volumeMounts)[index] = volumeMount
 	}
+	return
 }
 
 func findSecretInVolume(pod *corev1.PodSpec, volumeName string) (string, int) {

pkg/reconcile/pipeline/infinispan/handler/provision/statefulsets.go

@@ -317,6 +317,11 @@ func BuildServerContainerArgs(config *pipeline.ConfigFiles) []string {
 func addTLS(ctx pipeline.Context, i *ispnv1.Infinispan, statefulSet *appsv1.StatefulSet) {
 	if i.IsEncryptionEnabled() {
 		AddVolumesForEncryption(i, &statefulSet.Spec.Template.Spec)
+		// Only add the _HASH env variables for Infinispan servers that don't support automatic cert reloading
+		if ctx.Operand().UpstreamVersion.GTE(consts.MinVersionAutomaticCertificateReloading) {
+			return
+		}
+
 		configFiles := ctx.ConfigFiles()
 		ispnContainer := kube.GetContainer(InfinispanContainer, &statefulSet.Spec.Template.Spec)
 		ispnContainer.Env = append(ispnContainer.Env,

test/e2e/infinispan/encryption_test.go

@@ -119,6 +119,7 @@ func TestUpdateEncryptionSecrets(t *testing.T) {
 		i.Spec.Security = ispnv1.InfinispanSecurity{
 			EndpointEncryption: tutils.EndpointEncryption(i.Name),
 		}
+		i.Spec.Logging.Categories["org.infinispan.SECURITY"] = ispnv1.LoggingLevelDebug
 	})
 
 	// Create secret
@@ -160,21 +161,30 @@ func TestUpdateEncryptionSecrets(t *testing.T) {
 	keystoreSecret.Data[cconsts.EncryptTruststoreKey] = newKeystore
 	testKube.UpdateSecret(truststoreSecret)
 
-	// Wait for a new generation to appear
-	err := wait.Poll(tutils.DefaultPollPeriod, tutils.SinglePodTimeout, func() (done bool, err error) {
-		tutils.ExpectNoError(testKube.Kubernetes.Client.Get(context.TODO(), namespacedName, &ss))
-		return ss.Status.ObservedGeneration >= originalGeneration+1, nil
-	})
-	tutils.ExpectNoError(err)
-
-	// Wait that current and update revisions match. This ensures that the rolling upgrade completes
-	err = wait.Poll(tutils.DefaultPollPeriod, tutils.SinglePodTimeout, func() (done bool, err error) {
-		tutils.ExpectNoError(testKube.Kubernetes.Client.Get(context.TODO(), namespacedName, &ss))
-		return ss.Status.CurrentRevision == ss.Status.UpdateRevision, nil
-	})
-	tutils.ExpectNoError(err)
-
 	// Ensure that we can connect to the endpoint with the new TLS settings
 	client_ = tutils.HTTPSClientForCluster(spec, newTlsConfig, testKube)
-	checkRestConnection(client_)
+	if tutils.CurrentOperand.UpstreamVersion.LT(cconsts.MinVersionAutomaticCertificateReloading) {
+		// Wait for a new StatefulSet generation to appear
+		err := wait.Poll(tutils.DefaultPollPeriod, tutils.SinglePodTimeout, func() (done bool, err error) {
+			tutils.ExpectNoError(testKube.Kubernetes.Client.Get(context.TODO(), namespacedName, &ss))
+			return ss.Status.ObservedGeneration >= originalGeneration+1, nil
+		})
+		tutils.ExpectNoError(err)
+
+		// Wait that current and update revisions match. This ensures that the rolling upgrade completes
+		err = wait.Poll(tutils.DefaultPollPeriod, tutils.SinglePodTimeout, func() (done bool, err error) {
+			tutils.ExpectNoError(testKube.Kubernetes.Client.Get(context.TODO(), namespacedName, &ss))
+			return ss.Status.CurrentRevision == ss.Status.UpdateRevision, nil
+		})
+		tutils.ExpectNoError(err)
+		checkRestConnection(client_)
+	} else {
+		// Client connect attempt should eventually succeed once the Secret changes have been propagated to the Server pods
+		tutils.ExpectNoError(
+			wait.Poll(tutils.DefaultPollPeriod, tutils.SinglePodTimeout, func() (done bool, err error) {
+				_, err = ispnClient.New(tutils.CurrentOperand, client_).Container().Members()
+				return err == nil, nil
+			}),
+		)
+	}
 }