E2E test for Unhealthy GameServer on process crash (#1038)

* E2E test for Unhealthy GameServer on process crash

More preparation for #956

* Include logic to test LastTerminatedState, since
it looks like an update event can skip the State and
move directly to LastTerminatedState.

pkg/gameservers/health.go

@@ -131,8 +131,9 @@ func (hc *HealthController) evictedPod(pod *corev1.Pod) bool {
 func (hc *HealthController) failedContainer(pod *corev1.Pod) bool {
 	container := pod.Annotations[agonesv1.GameServerContainerAnnotation]
 	for _, cs := range pod.Status.ContainerStatuses {
-		if cs.Name == container && cs.State.Terminated != nil {
-			return true
+		if cs.Name == container {
+			// sometimes on a restart, the cs.State can be running and the last state will be merged
+			return cs.State.Terminated != nil || cs.LastTerminationState.Terminated != nil
 		}
 	}
 	return false

pkg/gameservers/health_test.go

@@ -50,6 +50,9 @@ func TestHealthControllerFailedContainer(t *testing.T) {
 	pod.Status.ContainerStatuses[0].State.Terminated = nil
 	assert.False(t, hc.failedContainer(pod))
 
+	pod.Status.ContainerStatuses[0].LastTerminationState.Terminated = &corev1.ContainerStateTerminated{}
+	assert.True(t, hc.failedContainer(pod))
+
 	pod2.Status.ContainerStatuses[0].Name = "Not a matching name"
 	assert.False(t, hc.failedContainer(pod2))
 }

test/e2e/gameserver_test.go

@@ -16,6 +16,7 @@ package e2e
 
 import (
 	"fmt"
+	"net"
 	"testing"
 	"time"
 
@@ -212,6 +213,34 @@ func TestGameServerUnhealthyAfterDeletingPod(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestGameServerUnhealthyAfterReadyCrash(t *testing.T) {
+	t.Parallel()
+
+	l := logrus.WithField("test", "TestGameServerUnhealthyAfterReadyCrash")
+
+	gs := defaultGameServer()
+	readyGs, err := framework.CreateGameServerAndWaitUntilReady(defaultNs, gs)
+	if err != nil {
+		t.Fatalf("Could not get a GameServer ready: %v", err)
+	}
+
+	l.WithField("gs", readyGs.ObjectMeta.Name).Info("GameServer created")
+
+	gsClient := framework.AgonesClient.AgonesV1().GameServers(defaultNs)
+	defer gsClient.Delete(readyGs.ObjectMeta.Name, nil) // nolint: errcheck
+
+	address := fmt.Sprintf("%s:%d", readyGs.Status.Address, readyGs.Status.Ports[0].Port)
+	conn, err := net.Dial("udp", address)
+	assert.NoError(t, err)
+	defer conn.Close() // nolint: errcheck
+	_, err = conn.Write([]byte("CRASH"))
+	assert.NoError(t, err)
+	l.WithField("address", address).Info("sent UDP packet")
+
+	_, err = framework.WaitForGameServerState(readyGs, agonesv1.GameServerStateUnhealthy, 3*time.Minute)
+	assert.NoError(t, err)
+}
+
 func TestDevelopmentGameServerLifecycle(t *testing.T) {
 	t.Parallel()
 	gs := &agonesv1.GameServer{

test/e2e/main_test.go

@@ -39,7 +39,7 @@ func TestMain(m *testing.M) {
 	pullSecret := flag.String("pullsecret", "",
 		"optional secret to be used for pulling the gameserver and/or Agones SDK sidecar images")
 	stressTestLevel := flag.Int("stress", 0, "enable stress test at given level 0-100")
-	perfOutputDir := flag.String("perf-output", "", "write performance statistics to the specified directrory")
+	perfOutputDir := flag.String("perf-output", "", "write performance statistics to the specified directory")
 
 	flag.Parse()