Counter and List Aggregate Fleet Metrics

pkg/metrics/controller.go

@@ -241,6 +241,15 @@ func (c *Controller) recordFleetChanges(obj interface{}) {
 
 	c.recordFleetReplicas(f.Name, f.Namespace, f.Status.Replicas, f.Status.AllocatedReplicas,
 		f.Status.ReadyReplicas, f.Spec.Replicas, f.Status.ReservedReplicas)
+
+	if runtime.FeatureEnabled(runtime.FeatureCountsAndLists) {
+		if f.Status.Counters != nil {
+			c.recordCounters(f.Name, f.Namespace, f.Status.Counters)
+		}
+		if f.Status.Lists != nil {
+			c.recordLists(f.Name, f.Namespace, f.Status.Lists)
+		}
+	}
 }
 
 func (c *Controller) recordFleetDeletion(obj interface{}) {
@@ -317,6 +326,40 @@ func (c *Controller) recordFleetReplicas(fleetName, fleetNamespace string, total
 		fleetsReplicasCountStats.M(int64(reserved)))
 }
 
+// nolint:dupl // Linter errors on lines are duplicate of recordLists
+func (c *Controller) recordCounters(fleetName, fleetNamespace string, counters map[string]agonesv1.AggregatedCounterStatus) {
+
+	ctx, _ := tag.New(context.Background(), tag.Upsert(keyFleetName, fleetName), tag.Upsert(keyNamespace, fleetNamespace))
+
+	for counter, counterStatus := range counters {
+		recordWithTags(ctx, []tag.Mutator{tag.Upsert(keyType, "allocated_count"), tag.Upsert(keyCounter, counter)},
+			fleetCountersStats.M(counterStatus.AllocatedCount))
+		recordWithTags(ctx, []tag.Mutator{tag.Upsert(keyType, "allocated_capacity"), tag.Upsert(keyCounter, counter)},
+			fleetCountersStats.M(counterStatus.AllocatedCapacity))
+		recordWithTags(ctx, []tag.Mutator{tag.Upsert(keyType, "total_count"), tag.Upsert(keyCounter, counter)},
+			fleetCountersStats.M(counterStatus.Count))
+		recordWithTags(ctx, []tag.Mutator{tag.Upsert(keyType, "total_capacity"), tag.Upsert(keyCounter, counter)},
+			fleetCountersStats.M(counterStatus.Capacity))
+	}
+}
+
+// nolint:dupl // Linter errors on lines are duplicate of recordCounters
+func (c *Controller) recordLists(fleetName, fleetNamespace string, lists map[string]agonesv1.AggregatedListStatus) {
+
+	ctx, _ := tag.New(context.Background(), tag.Upsert(keyFleetName, fleetName), tag.Upsert(keyNamespace, fleetNamespace))
+
+	for list, listStatus := range lists {
+		recordWithTags(ctx, []tag.Mutator{tag.Upsert(keyType, "allocated_count"), tag.Upsert(keyList, list)},
+			fleetListsStats.M(listStatus.AllocatedCount))
+		recordWithTags(ctx, []tag.Mutator{tag.Upsert(keyType, "allocated_capacity"), tag.Upsert(keyList, list)},
+			fleetListsStats.M(listStatus.AllocatedCapacity))
+		recordWithTags(ctx, []tag.Mutator{tag.Upsert(keyType, "total_count"), tag.Upsert(keyList, list)},
+			fleetListsStats.M(listStatus.Count))
+		recordWithTags(ctx, []tag.Mutator{tag.Upsert(keyType, "total_capacity"), tag.Upsert(keyList, list)},
+			fleetListsStats.M(listStatus.Capacity))
+	}
+}
+
 // recordGameServerStatusChanged records gameserver status changes, however since it's based
 // on cache events some events might collapsed and not appear, for example transition state
 // like creating, port allocation, could be skipped.

pkg/metrics/controller_metrics.go

@@ -28,6 +28,8 @@ const (
 	fleetAutoscalersDesiredReplicaCountName = "fleet_autoscalers_desired_replicas_count"
 	fleetAutoscalersAbleToScaleName         = "fleet_autoscalers_able_to_scale"
 	fleetAutoscalersLimitedName             = "fleet_autoscalers_limited"
+	fleetCountersName                       = "fleet_counters"
+	fleetListsName                          = "fleet_lists"
 	gameServersCountName                    = "gameservers_count"
 	gameServersTotalName                    = "gameservers_total"
 	gameServersPlayerConnectedTotalName     = "gameserver_player_connected_total"
@@ -42,7 +44,7 @@ var (
 	fleetAutoscalerViews = []string{fleetAutoscalerBufferLimitName, fleetAutoscalterBufferSizeName, fleetAutoscalerCurrentReplicaCountName,
 		fleetAutoscalersDesiredReplicaCountName, fleetAutoscalersAbleToScaleName, fleetAutoscalersLimitedName}
 	// fleetViews are metric views associated with Fleets
-	fleetViews = append([]string{fleetReplicaCountName, gameServersCountName, gameServersTotalName, gameServersPlayerConnectedTotalName, gameServersPlayerCapacityTotalName, gameServerStateDurationName}, fleetAutoscalerViews...)
+	fleetViews = append([]string{fleetReplicaCountName, gameServersCountName, gameServersTotalName, gameServersPlayerConnectedTotalName, gameServersPlayerCapacityTotalName, gameServerStateDurationName, fleetCountersName, fleetListsName}, fleetAutoscalerViews...)
 
 	stateDurationSeconds           = []float64{0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384}
 	fleetsReplicasCountStats       = stats.Int64("fleets/replicas_count", "The count of replicas per fleet", "1")
@@ -52,6 +54,8 @@ var (
 	fasDesiredReplicasStats        = stats.Int64("fas/desired_replicas_count", "The desired replicas cout as seen by autoscalers", "1")
 	fasAbleToScaleStats            = stats.Int64("fas/able_to_scale", "The fleet autoscaler can access the fleet to scale (0 indicates false, 1 indicates true)", "1")
 	fasLimitedStats                = stats.Int64("fas/limited", "The fleet autoscaler is capped (0 indicates false, 1 indicates true)", "1")
+	fleetCountersStats             = stats.Int64("fleets/counters", "Aggregated Counters counts and capacity across GameServers in the Fleet", "1")
+	fleetListsStats                = stats.Int64("fleets/lists", "Aggregated Lists counts and capacity across GameServers in the Fleet", "1")
 	gameServerCountStats           = stats.Int64("gameservers/count", "The count of gameservers", "1")
 	gameServerTotalStats           = stats.Int64("gameservers/total", "The total of gameservers", "1")
 	gameServerPlayerConnectedTotal = stats.Int64("gameservers/player_connected", "The total number of players connected to gameservers", "1")
@@ -110,6 +114,20 @@ var (
 			Aggregation: view.LastValue(),
 			TagKeys:     []tag.Key{keyName, keyFleetName, keyNamespace},
 		},
+		{
+			Name:        fleetCountersName,
+			Measure:     fleetCountersStats,
+			Description: "Aggregated Counters counts and capacity across GameServers in the Fleet",
+			Aggregation: view.LastValue(),
+			TagKeys:     []tag.Key{keyFleetName, keyNamespace, keyType, keyCounter},
+		},
+		{
+			Name:        fleetListsName,
+			Measure:     fleetListsStats,
+			Description: "Aggregated Lists counts and capacity across GameServers in the Fleet",
+			Aggregation: view.LastValue(),
+			TagKeys:     []tag.Key{keyFleetName, keyNamespace, keyType, keyList},
+		},
 		{
 			Name:        gameServersCountName,
 			Measure:     gameServerCountStats,

pkg/metrics/controller_test.go

@@ -23,6 +23,7 @@ import (
 	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
 	agtesting "agones.dev/agones/pkg/testing"
 	"agones.dev/agones/pkg/util/runtime"
+	"github.com/google/go-cmp/cmp"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
@@ -470,6 +471,70 @@ func TestControllerGameServersNodeState(t *testing.T) {
 	})
 }
 
+func TestFleetCountersAndListsMetrics(t *testing.T) {
+	runtime.FeatureTestMutex.Lock()
+	defer runtime.FeatureTestMutex.Unlock()
+	assert.NoError(t, runtime.ParseFeatures(string(runtime.FeatureCountsAndLists)+"=true"))
+
+	resetMetrics()
+	exporter := &metricExporter{}
+	reader := metricexport.NewReader()
+	c := newFakeController()
+	defer c.close()
+
+	fleetName := "cl-fleet-test"
+	counterName := "players"
+	counters := map[string]agonesv1.AggregatedCounterStatus{
+		counterName: {
+			AllocatedCount:    24,
+			AllocatedCapacity: 30,
+			Count:             28,
+			Capacity:          50,
+		},
+	}
+	listName := "rooms"
+	lists := map[string]agonesv1.AggregatedListStatus{
+		listName: {
+			AllocatedCount:    4,
+			AllocatedCapacity: 6,
+			Count:             1,
+			Capacity:          100,
+		},
+	}
+
+	f := fleet(fleetName, 8, 3, 5, 8, 0)
+	c.fleetWatch.Add(f)
+	f = f.DeepCopy()
+	f.Status.Lists = lists
+	f.Status.Counters = counters
+	c.fleetWatch.Modify(f)
+
+	c.run(t)
+	require.True(t, c.sync())
+	require.Eventually(t, func() bool {
+		fl, err := c.fleetLister.Fleets(f.GetObjectMeta().GetNamespace()).Get(fleetName)
+		assert.NoError(t, err)
+		return cmp.Equal(counters, fl.Status.Counters) && cmp.Equal(lists, fl.Status.Lists)
+	}, 5*time.Second, time.Second)
+	c.collect()
+
+	reader.ReadAndExport(exporter)
+	assertMetricData(t, exporter, fleetCountersName, []expectedMetricData{
+		// keyCounter, keyFleetName, keyNamespace, keyType
+		{labels: []string{counterName, fleetName, defaultNs, "allocated_count"}, val: int64(24)},
+		{labels: []string{counterName, fleetName, defaultNs, "allocated_capacity"}, val: int64(30)},
+		{labels: []string{counterName, fleetName, defaultNs, "total_count"}, val: int64(28)},
+		{labels: []string{counterName, fleetName, defaultNs, "total_capacity"}, val: int64(50)},
+	})
+	assertMetricData(t, exporter, fleetListsName, []expectedMetricData{
+		// keyList, keyFleetName, keyNamespace, keyType
+		{labels: []string{fleetName, listName, defaultNs, "allocated_count"}, val: int64(4)},
+		{labels: []string{fleetName, listName, defaultNs, "allocated_capacity"}, val: int64(6)},
+		{labels: []string{fleetName, listName, defaultNs, "total_count"}, val: int64(1)},
+		{labels: []string{fleetName, listName, defaultNs, "total_capacity"}, val: int64(100)},
+	})
+}
+
 func TestCalcDuration(t *testing.T) {
 	m := agtesting.NewMocks()
 	c := NewController(

pkg/metrics/util.go

@@ -38,6 +38,8 @@ var (
 	keyVerb       = MustTagKey("verb")
 	keyEndpoint   = MustTagKey("endpoint")
 	keyEmpty      = MustTagKey("empty")
+	keyCounter    = MustTagKey("counter")
+	keyList       = MustTagKey("list")
 )
 
 func recordWithTags(ctx context.Context, mutators []tag.Mutator, ms ...stats.Measurement) {