From ed173583c4e288270344debd25dd668a09b92f28 Mon Sep 17 00:00:00 2001 From: Mark Mandel Date: Wed, 16 Mar 2022 21:08:37 -0700 Subject: [PATCH] e2e flow for High Density GameServers (#2516) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit e2e flow for High Density GameServers This PR includes two things, such that I believe we can close the below issue: 1. An e2e test to confirm that the described flow in https://agones.dev/site/docs/integration-patterns/high-density-gameservers/ works as expected (it does 😄). 2. Add a section in the integration documentation to highlight the consistency models (i.e. things are not atomic operations, but eventually consistent), and that people need to keep this in mind. Closes #2408 --- .../high-density-gameservers.md | 15 +++- test/e2e/gameserverallocation_test.go | 73 +++++++++++++++++++ 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/site/content/en/docs/Integration Patterns/high-density-gameservers.md b/site/content/en/docs/Integration Patterns/high-density-gameservers.md index bcdf606af3..c6a42866f7 100644 --- a/site/content/en/docs/Integration Patterns/high-density-gameservers.md +++ b/site/content/en/docs/Integration Patterns/high-density-gameservers.md @@ -17,7 +17,7 @@ systems, since it works around the common Kubernetes and/or Agones container lif Utilising the new allocation `gameServerState` filter as well as the existing ability to edit the `GameServer` labels at both [allocation time]({{% ref "/docs/Reference/gameserverallocation.md" %}}), and from -within the game server process, [via the SDK]({{% ref "/docs/Guides/Client SDKs/_index.md#setlabelkey-value" %}}), +within the game server process, [via the SDK][sdk], means Agones is able to atomically remove a `GameServer` from the list of potentially allocatable `GameServers` at allocation time, and then return it back into the pool of allocatable `GameServers` if and when the game server process deems that is has room to host another game session. @@ -72,11 +72,20 @@ spec: {{< alert title="Info" color="info">}} It's important to note that the labels that the `GameServer` process use to add itself back into the pool of allocatable instances, must start with the prefix `agones.dev/sdk-`, since only labels that have this prefix are -available to be [updated from the SDK]({{% ref "/docs/Guides/Client SDKs/_index.md#setlabelkey-value" %}}). +available to be [updated from the SDK][sdk]. {{< /alert >}} +## Consistency + +Agones, and Kubernetes itself are built as eventually consistent, self-healing systems. To that end, it is worth +noting that there may be minor delays between each of the operations in the above flow. For example, depending on the +cluster load, it may take up to a second for an [SDK driven label change][sdk] on a `GameServer` record to be +visible to the Agones allocation system. We recommend building your integrations with Agones with this in mind. + ## Next Steps -* View the details about [using the SDK]({{% ref "/docs/Guides/Client SDKs/_index.md#setlabelkey-value" %}}) to set +* View the details about [using the SDK][sdk] to set labels on the `GameServer`. * Check all the options available on [`GameServerAllocation`]({{% ref "/docs/Reference/gameserverallocation.md" %}}). + +[sdk]: {{% ref "/docs/Guides/Client SDKs/_index.md#setlabelkey-value" %}} \ No newline at end of file diff --git a/test/e2e/gameserverallocation_test.go b/test/e2e/gameserverallocation_test.go index 57a6de7fee..0c6d86109c 100644 --- a/test/e2e/gameserverallocation_test.go +++ b/test/e2e/gameserverallocation_test.go @@ -129,6 +129,79 @@ func TestCreateFleetAndGameServerStateFilterAllocation(t *testing.T) { require.NotEqual(t, gs1.ObjectMeta.Annotations["agones.dev/last-allocated"], gs2.ObjectMeta.Annotations["agones.dev/last-allocated"]) } +func TestHighDensityGameServerFlow(t *testing.T) { + if !runtime.FeatureEnabled(runtime.FeatureStateAllocationFilter) { + t.SkipNow() + } + t.Parallel() + log := e2e.TestLogger(t) + ctx := context.Background() + + fleets := framework.AgonesClient.AgonesV1().Fleets(framework.Namespace) + fleet := defaultFleet(framework.Namespace) + lockLabel := "agones.dev/sdk-available" + // to start they are all available + fleet.Spec.Template.ObjectMeta.Labels = map[string]string{lockLabel: "true"} + + flt, err := fleets.Create(ctx, fleet, metav1.CreateOptions{}) + require.NoError(t, err) + defer fleets.Delete(ctx, flt.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck + + framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(flt.Spec.Replicas)) + + fleetSelector := metav1.LabelSelector{MatchLabels: map[string]string{agonesv1.FleetNameLabel: flt.ObjectMeta.Name}} + allocatedSelector := fleetSelector.DeepCopy() + + allocated := agonesv1.GameServerStateAllocated + allocatedSelector.MatchLabels[lockLabel] = "true" + gsa := &allocationv1.GameServerAllocation{ + Spec: allocationv1.GameServerAllocationSpec{ + MetaPatch: allocationv1.MetaPatch{Labels: map[string]string{lockLabel: "false"}}, + Selectors: []allocationv1.GameServerSelector{ + {LabelSelector: *allocatedSelector, GameServerState: &allocated}, + {LabelSelector: fleetSelector}, + }, + }} + + // standard allocation + result, err := framework.AgonesClient.AllocationV1().GameServerAllocations(fleet.ObjectMeta.Namespace).Create(ctx, gsa, metav1.CreateOptions{}) + require.NoError(t, err) + require.Equal(t, string(allocationv1.GameServerAllocationAllocated), string(result.Status.State)) + + gs, err := framework.AgonesClient.AgonesV1().GameServers(fleet.ObjectMeta.Namespace).Get(ctx, result.Status.GameServerName, metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, allocated, gs.Status.State) + + // set the label to being available again + _, err = framework.SendGameServerUDP(t, gs, "LABEL available true") + require.NoError(t, err) + + // wait for the label to be applied! + require.Eventuallyf(t, func() bool { + gs, err := framework.AgonesClient.AgonesV1().GameServers(fleet.ObjectMeta.Namespace).Get(ctx, result.Status.GameServerName, metav1.GetOptions{}) + require.NoError(t, err) + log.WithField("labels", gs.ObjectMeta.Labels).Info("checking labels") + return gs.ObjectMeta.Labels[lockLabel] == "true" + }, time.Minute, time.Second, "GameServer did not unlock") + + // Run the same allocation again, we should get back the preferred item. + expected := result.Status.GameServerName + + // we will run this as an Eventually, as caches are eventually consistent + require.Eventuallyf(t, func() bool { + result, err = framework.AgonesClient.AllocationV1().GameServerAllocations(fleet.ObjectMeta.Namespace).Create(ctx, gsa, metav1.CreateOptions{}) + require.NoError(t, err) + require.Equal(t, string(allocationv1.GameServerAllocationAllocated), string(result.Status.State)) + + if expected != result.Status.GameServerName { + log.WithField("expected", expected).WithField("gsa", result).Info("Re-allocation attempt failed. Retrying.") + return false + } + + return true + }, time.Minute, time.Second, "Could not re-allocation") +} + func TestCreateFleetAndGameServerPlayerCapacityAllocation(t *testing.T) { if !(runtime.FeatureEnabled(runtime.FeatureStateAllocationFilter) && runtime.FeatureEnabled(runtime.FeaturePlayerAllocationFilter)) { t.SkipNow()