e2e flow for High Density GameServers (#2516)

e2e flow for High Density GameServers

This PR includes two things, such that I believe we can close the below
issue:

1. An e2e test to confirm that the described flow in
https://agones.dev/site/docs/integration-patterns/high-density-gameservers/
works as expected (it does 😄).
2. Add a section in the integration documentation to highlight the
consistency models (i.e. things are not atomic operations, but
eventually consistent), and that people need to keep this in mind.

Closes #2408

site/content/en/docs/Integration Patterns/high-density-gameservers.md

@@ -17,7 +17,7 @@ systems, since it works around the common Kubernetes and/or Agones container lif
 
 Utilising the new allocation `gameServerState` filter as well as the existing ability to edit the 
 `GameServer` labels at both [allocation time]({{% ref "/docs/Reference/gameserverallocation.md" %}}), and from 
-within the game server process, [via the SDK]({{% ref "/docs/Guides/Client SDKs/_index.md#setlabelkey-value" %}}), 
+within the game server process, [via the SDK][sdk], 
 means Agones is able to atomically remove a `GameServer` from the list of potentially allocatable 
 `GameServers` at allocation time, and then return it back into the pool of allocatable `GameServers` if and when the 
 game server process deems that is has room to host another game session. 
@@ -72,11 +72,20 @@ spec:
 {{< alert title="Info" color="info">}}
 It's important to note that the labels that the `GameServer` process use to add itself back into the pool of 
 allocatable instances, must start with the prefix `agones.dev/sdk-`, since only labels that have this prefix are 
-available to be [updated from the SDK]({{% ref "/docs/Guides/Client SDKs/_index.md#setlabelkey-value" %}}).
+available to be [updated from the SDK][sdk].
 {{< /alert >}}
 
+## Consistency
+
+Agones, and Kubernetes itself are built as eventually consistent, self-healing systems. To that end, it is worth 
+noting that there may be minor delays between each of the operations in the above flow.  For example, depending on the 
+cluster load, it may take up to a second for an [SDK driven label change][sdk] on a `GameServer` record to be 
+visible to the Agones allocation system. We recommend building your integrations with Agones with this in mind.
+
 ## Next Steps
 
-* View the details about [using the SDK]({{% ref "/docs/Guides/Client SDKs/_index.md#setlabelkey-value" %}}) to set 
+* View the details about [using the SDK][sdk] to set 
   labels on the `GameServer`.
 * Check all the options available on [`GameServerAllocation`]({{% ref "/docs/Reference/gameserverallocation.md" %}}).
+
+[sdk]: {{% ref "/docs/Guides/Client SDKs/_index.md#setlabelkey-value" %}}

test/e2e/gameserverallocation_test.go

@@ -129,6 +129,79 @@ func TestCreateFleetAndGameServerStateFilterAllocation(t *testing.T) {
 	require.NotEqual(t, gs1.ObjectMeta.Annotations["agones.dev/last-allocated"], gs2.ObjectMeta.Annotations["agones.dev/last-allocated"])
 }
 
+func TestHighDensityGameServerFlow(t *testing.T) {
+	if !runtime.FeatureEnabled(runtime.FeatureStateAllocationFilter) {
+		t.SkipNow()
+	}
+	t.Parallel()
+	log := e2e.TestLogger(t)
+	ctx := context.Background()
+
+	fleets := framework.AgonesClient.AgonesV1().Fleets(framework.Namespace)
+	fleet := defaultFleet(framework.Namespace)
+	lockLabel := "agones.dev/sdk-available"
+	// to start they are all available
+	fleet.Spec.Template.ObjectMeta.Labels = map[string]string{lockLabel: "true"}
+
+	flt, err := fleets.Create(ctx, fleet, metav1.CreateOptions{})
+	require.NoError(t, err)
+	defer fleets.Delete(ctx, flt.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck
+
+	framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(flt.Spec.Replicas))
+
+	fleetSelector := metav1.LabelSelector{MatchLabels: map[string]string{agonesv1.FleetNameLabel: flt.ObjectMeta.Name}}
+	allocatedSelector := fleetSelector.DeepCopy()
+
+	allocated := agonesv1.GameServerStateAllocated
+	allocatedSelector.MatchLabels[lockLabel] = "true"
+	gsa := &allocationv1.GameServerAllocation{
+		Spec: allocationv1.GameServerAllocationSpec{
+			MetaPatch: allocationv1.MetaPatch{Labels: map[string]string{lockLabel: "false"}},
+			Selectors: []allocationv1.GameServerSelector{
+				{LabelSelector: *allocatedSelector, GameServerState: &allocated},
+				{LabelSelector: fleetSelector},
+			},
+		}}
+
+	// standard allocation
+	result, err := framework.AgonesClient.AllocationV1().GameServerAllocations(fleet.ObjectMeta.Namespace).Create(ctx, gsa, metav1.CreateOptions{})
+	require.NoError(t, err)
+	require.Equal(t, string(allocationv1.GameServerAllocationAllocated), string(result.Status.State))
+
+	gs, err := framework.AgonesClient.AgonesV1().GameServers(fleet.ObjectMeta.Namespace).Get(ctx, result.Status.GameServerName, metav1.GetOptions{})
+	require.NoError(t, err)
+	require.Equal(t, allocated, gs.Status.State)
+
+	// set the label to being available again
+	_, err = framework.SendGameServerUDP(t, gs, "LABEL available true")
+	require.NoError(t, err)
+
+	// wait for the label to be applied!
+	require.Eventuallyf(t, func() bool {
+		gs, err := framework.AgonesClient.AgonesV1().GameServers(fleet.ObjectMeta.Namespace).Get(ctx, result.Status.GameServerName, metav1.GetOptions{})
+		require.NoError(t, err)
+		log.WithField("labels", gs.ObjectMeta.Labels).Info("checking labels")
+		return gs.ObjectMeta.Labels[lockLabel] == "true"
+	}, time.Minute, time.Second, "GameServer did not unlock")
+
+	// Run the same allocation again, we should get back the preferred item.
+	expected := result.Status.GameServerName
+
+	// we will run this as an Eventually, as caches are eventually consistent
+	require.Eventuallyf(t, func() bool {
+		result, err = framework.AgonesClient.AllocationV1().GameServerAllocations(fleet.ObjectMeta.Namespace).Create(ctx, gsa, metav1.CreateOptions{})
+		require.NoError(t, err)
+		require.Equal(t, string(allocationv1.GameServerAllocationAllocated), string(result.Status.State))
+
+		if expected != result.Status.GameServerName {
+			log.WithField("expected", expected).WithField("gsa", result).Info("Re-allocation attempt failed. Retrying.")
+			return false
+		}
+
+		return true
+	}, time.Minute, time.Second, "Could not re-allocation")
+}
+
 func TestCreateFleetAndGameServerPlayerCapacityAllocation(t *testing.T) {
 	if !(runtime.FeatureEnabled(runtime.FeatureStateAllocationFilter) && runtime.FeatureEnabled(runtime.FeaturePlayerAllocationFilter)) {
 		t.SkipNow()