Add e2e test for leader election

test/e2e/controller/crash_test.go

@@ -20,8 +20,10 @@ import (
 	"time"
 
 	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
+	"agones.dev/agones/pkg/util/runtime"
 	e2eframework "agones.dev/agones/test/e2e/framework"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -58,6 +60,66 @@ func TestGameServerUnhealthyAfterDeletingPodWhileControllerDown(t *testing.T) {
 	assert.NoError(t, waitForAgonesControllerRunning(ctx))
 }
 
+func TestLeaderElectionAfterDeletingLeader(t *testing.T) {
+	logger := e2eframework.TestLogger(t)
+	gs := framework.DefaultGameServer(defaultNs)
+	ctx := context.Background()
+
+	if !runtime.FeatureEnabled(runtime.FeatureSplitControllerAndExtensions) {
+		t.Skip("Skip test. SplitControllerAndExtensions feature is not enabled")
+	}
+
+	err := waitForAgonesControllerRunning(ctx)
+	require.NoError(t, err, "Could not ensure controller running")
+
+	list, err := getAgonesControllerPods(ctx)
+	require.NoError(t, err, "Could not get list of Extension pods")
+	if len(list.Items) == 1 {
+		t.Skip("Skip test. Leader Election is not enabled since there is only 1 controller")
+	}
+
+	replication := len(list.Items)
+
+	// Deleting one of the pods which cause the other one to most definitely be leader
+	willBeLeader := list.Items[0]
+	for i := range list.Items[1:] {
+		err = deleteAgonesControllerPod(ctx, &list.Items[i])
+		require.NoError(t, err, "Could not delete controller pod")
+	}
+
+	err = deleteAgonesControllerPod(ctx, &willBeLeader)
+	require.NoError(t, err, "Could not delete controller pod")
+
+	err = wait.PollImmediate(time.Second, 5*time.Minute, func() (bool, error) {
+		list, err := getAgonesControllerPods(ctx)
+		if err != nil {
+			return true, err
+		}
+
+		if len(list.Items) != replication {
+			return true, nil
+		}
+
+		for i := range list.Items {
+			for _, c := range list.Items[i].Status.ContainerStatuses {
+				if c.State.Running == nil {
+					return false, nil
+				}
+			}
+		}
+
+		return true, nil
+	})
+	require.NoError(t, err, "Could not get controller ready after delete")
+
+	// Do something that will test controller like make game server
+	readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, defaultNs, gs)
+	if err != nil {
+		t.Fatalf("Could not get a GameServer ready: %v", err)
+	}
+	logger.WithField("gsKey", readyGs.ObjectMeta.Name).Info("GameServer Ready")
+}
+
 // deleteAgonesControllerPods deletes all the Controller pods for the Agones controller,
 // faking a controller crash.
 func deleteAgonesControllerPods(ctx context.Context) error {
@@ -66,10 +128,8 @@ func deleteAgonesControllerPods(ctx context.Context) error {
 		return err
 	}
 
-	policy := metav1.DeletePropagationBackground
 	for i := range list.Items {
-		err = framework.KubeClient.CoreV1().Pods("agones-system").Delete(ctx, list.Items[i].ObjectMeta.Name,
-			metav1.DeleteOptions{PropagationPolicy: &policy})
+		err = deleteAgonesControllerPod(ctx, &list.Items[i])
 		if err != nil {
 			return err
 		}
@@ -101,3 +161,11 @@ func getAgonesControllerPods(ctx context.Context) (*corev1.PodList, error) {
 	opts := metav1.ListOptions{LabelSelector: labels.Set{"agones.dev/role": "controller"}.String()}
 	return framework.KubeClient.CoreV1().Pods("agones-system").List(ctx, opts)
 }
+
+// deleteAgonesControllerPod deletes a Agones controller pod
+func deleteAgonesControllerPod(ctx context.Context, pod *corev1.Pod) error {
+	policy := metav1.DeletePropagationBackground
+	err := framework.KubeClient.CoreV1().Pods("agones-system").Delete(ctx, pod.ObjectMeta.Name,
+		metav1.DeleteOptions{PropagationPolicy: &policy})
+	return err
+}