diff --git a/install/helm/agones/templates/crds/fleet.yaml b/install/helm/agones/templates/crds/fleet.yaml index 5ddefb8cbd..f5cf4cae98 100644 --- a/install/helm/agones/templates/crds/fleet.yaml +++ b/install/helm/agones/templates/crds/fleet.yaml @@ -72,6 +72,18 @@ spec: replicas: type: integer minimum: 0 + allocationOverflow: + type: object + nullable: true + properties: + labels: + type: object + additionalProperties: + type: string + annotations: + type: object + additionalProperties: + type: string scheduling: type: string enum: diff --git a/install/helm/agones/templates/crds/gameserverset.yaml b/install/helm/agones/templates/crds/gameserverset.yaml index 476ab1bd25..8d153ec520 100644 --- a/install/helm/agones/templates/crds/gameserverset.yaml +++ b/install/helm/agones/templates/crds/gameserverset.yaml @@ -74,6 +74,18 @@ spec: replicas: type: integer minimum: 0 + allocationOverflow: + type: object + nullable: true + properties: + labels: + type: object + additionalProperties: + type: string + annotations: + type: object + additionalProperties: + type: string scheduling: type: string enum: diff --git a/install/yaml/install.yaml b/install/yaml/install.yaml index e769b9a93c..6819049fb2 100644 --- a/install/yaml/install.yaml +++ b/install/yaml/install.yaml @@ -211,6 +211,18 @@ spec: replicas: type: integer minimum: 0 + allocationOverflow: + type: object + nullable: true + properties: + labels: + type: object + additionalProperties: + type: string + annotations: + type: object + additionalProperties: + type: string scheduling: type: string enum: @@ -10441,6 +10453,18 @@ spec: replicas: type: integer minimum: 0 + allocationOverflow: + type: object + nullable: true + properties: + labels: + type: object + additionalProperties: + type: string + annotations: + type: object + additionalProperties: + type: string scheduling: type: string enum: diff --git a/pkg/apis/agones/v1/common.go b/pkg/apis/agones/v1/common.go index ea58862a78..f1a5539dc4 100644 --- a/pkg/apis/agones/v1/common.go +++ b/pkg/apis/agones/v1/common.go @@ -20,6 +20,7 @@ import ( apivalidation "k8s.io/apimachinery/pkg/api/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" @@ -107,3 +108,91 @@ func validateObjectMeta(objMeta *metav1.ObjectMeta) []metav1.StatusCause { } return causes } + +// AllocationOverflow specifies what labels and/or annotations to apply on Allocated GameServers +// if the desired number of the underlying `GameServerSet` drops below the number of Allocated GameServers +// attached to it. +type AllocationOverflow struct { + // Labels to be applied to the `GameServer` + // +optional + Labels map[string]string `json:"labels,omitempty"` + // Annotations to be applied to the `GameServer` + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} + +// Validate validates the label and annotation values +func (ao *AllocationOverflow) Validate() ([]metav1.StatusCause, bool) { + var causes []metav1.StatusCause + parentField := "Spec.AllocationOverflow" + + errs := metav1validation.ValidateLabels(ao.Labels, field.NewPath(parentField)) + if len(errs) != 0 { + for _, v := range errs { + causes = append(causes, metav1.StatusCause{ + Type: metav1.CauseTypeFieldValueInvalid, + Field: "labels", + Message: v.Error(), + }) + } + } + errs = apivalidation.ValidateAnnotations(ao.Annotations, + field.NewPath(parentField)) + if len(errs) != 0 { + for _, v := range errs { + causes = append(causes, metav1.StatusCause{ + Type: metav1.CauseTypeFieldValueInvalid, + Field: "annotations", + Message: v.Error(), + }) + } + } + + return causes, len(causes) == 0 +} + +// CountMatches returns the number of Allocated GameServers that match the labels and annotations, and +// the set of GameServers left over. +func (ao *AllocationOverflow) CountMatches(list []*GameServer) (int32, []*GameServer) { + count := int32(0) + var rest []*GameServer + labelSelector := labels.Set(ao.Labels).AsSelector() + annotationSelector := labels.Set(ao.Annotations).AsSelector() + + for _, gs := range list { + if gs.Status.State != GameServerStateAllocated { + continue + } + if !labelSelector.Matches(labels.Set(gs.ObjectMeta.Labels)) { + rest = append(rest, gs) + continue + } + if !annotationSelector.Matches(labels.Set(gs.ObjectMeta.Annotations)) { + rest = append(rest, gs) + continue + } + count++ + } + + return count, rest +} + +// Apply applies the labels and annotations to the passed in GameServer +func (ao *AllocationOverflow) Apply(gs *GameServer) { + if ao.Annotations != nil { + if gs.ObjectMeta.Annotations == nil { + gs.ObjectMeta.Annotations = map[string]string{} + } + for k, v := range ao.Annotations { + gs.ObjectMeta.Annotations[k] = v + } + } + if ao.Labels != nil { + if gs.ObjectMeta.Labels == nil { + gs.ObjectMeta.Labels = map[string]string{} + } + for k, v := range ao.Labels { + gs.ObjectMeta.Labels[k] = v + } + } +} diff --git a/pkg/apis/agones/v1/common_test.go b/pkg/apis/agones/v1/common_test.go new file mode 100644 index 0000000000..d8d82f2293 --- /dev/null +++ b/pkg/apis/agones/v1/common_test.go @@ -0,0 +1,208 @@ +// Copyright 2023 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestAllocationOverflowValidate(t *testing.T) { + // valid + type expected struct { + valid bool + fields []string + } + + fixtures := map[string]struct { + ao AllocationOverflow + expected + }{ + "empty": { + ao: AllocationOverflow{}, + expected: expected{ + valid: true, + fields: nil, + }, + }, + "bad label name": { + ao: AllocationOverflow{ + Labels: map[string]string{"$$$foobar": "stuff"}, + Annotations: nil, + }, + expected: expected{ + valid: false, + fields: []string{"labels"}, + }, + }, + "bad label value": { + ao: AllocationOverflow{ + Labels: map[string]string{"valid": "$$$NOPE"}, + Annotations: nil, + }, + expected: expected{ + valid: false, + fields: []string{"labels"}, + }, + }, + "bad annotation name": { + ao: AllocationOverflow{ + Labels: nil, + Annotations: map[string]string{"$$$foobar": "stuff"}, + }, + expected: expected{ + valid: false, + fields: []string{"annotations"}, + }, + }, + "valid full": { + ao: AllocationOverflow{ + Labels: map[string]string{"valid": "yes", "still.valid": "check-me-out"}, + Annotations: map[string]string{"icando-this": "yes, I can do all kinds of things here $$$"}, + }, + expected: expected{ + valid: true, + fields: nil, + }, + }, + } + + for k, v := range fixtures { + t.Run(k, func(t *testing.T) { + causes, valid := v.ao.Validate() + assert.Equal(t, v.expected.valid, valid, "valid") + if v.expected.fields == nil { + assert.Empty(t, causes) + } else { + for i, cause := range causes { + assert.Equal(t, metav1.CauseTypeFieldValueInvalid, cause.Type) + // messages come from K8s validation libraries, so testing exact matches would be brittle. + assert.Contains(t, cause.Message, "Invalid value:") + assert.Equal(t, v.expected.fields[i], cause.Field) + } + } + }) + } +} + +func TestAllocationOverflowCountMatches(t *testing.T) { + type expected struct { + count int32 + rest int + } + + fixtures := map[string]struct { + list func([]*GameServer) + ao func(*AllocationOverflow) + expected expected + }{ + "simple": { + list: func(_ []*GameServer) {}, + ao: func(_ *AllocationOverflow) {}, + expected: expected{ + count: 2, + rest: 0, + }, + }, + "label selector": { + list: func(list []*GameServer) { + list[0].ObjectMeta.Labels = map[string]string{"colour": "blue"} + }, + ao: func(ao *AllocationOverflow) { + ao.Labels = map[string]string{"colour": "blue"} + }, + expected: expected{ + count: 1, + rest: 1, + }, + }, + "annotation selector": { + list: func(list []*GameServer) { + list[0].ObjectMeta.Annotations = map[string]string{"colour": "green"} + }, + ao: func(ao *AllocationOverflow) { + ao.Annotations = map[string]string{"colour": "green"} + }, + expected: expected{ + count: 1, + rest: 1, + }, + }, + "both": { + list: func(list []*GameServer) { + list[0].ObjectMeta.Labels = map[string]string{"colour": "blue"} + list[0].ObjectMeta.Annotations = map[string]string{"colour": "green"} + }, + ao: func(ao *AllocationOverflow) { + ao.Labels = map[string]string{"colour": "blue"} + ao.Annotations = map[string]string{"colour": "green"} + }, + expected: expected{ + count: 1, + rest: 1, + }, + }, + } + + for k, v := range fixtures { + t.Run(k, func(t *testing.T) { + list := []*GameServer{ + {ObjectMeta: metav1.ObjectMeta{Name: "g1"}, Status: GameServerStatus{State: GameServerStateAllocated}}, + {ObjectMeta: metav1.ObjectMeta{Name: "g2"}, Status: GameServerStatus{State: GameServerStateAllocated}}, + {ObjectMeta: metav1.ObjectMeta{Name: "g3"}, Status: GameServerStatus{State: GameServerStateReady}}, + } + v.list(list) + ao := &AllocationOverflow{ + Labels: nil, + Annotations: nil, + } + v.ao(ao) + + count, rest := ao.CountMatches(list) + assert.Equal(t, v.expected.count, count, "count") + assert.Equal(t, v.expected.rest, len(rest), "rest") + for _, gs := range rest { + assert.Equal(t, GameServerStateAllocated, gs.Status.State) + } + }) + } +} + +func TestAllocationOverflowApply(t *testing.T) { + // check empty + gs := &GameServer{} + ao := AllocationOverflow{Labels: map[string]string{"colour": "green"}, Annotations: map[string]string{"colour": "blue", "map": "ice cream"}} + + ao.Apply(gs) + + require.Equal(t, ao.Annotations, gs.ObjectMeta.Annotations) + require.Equal(t, ao.Labels, gs.ObjectMeta.Labels) + + // check append + ao = AllocationOverflow{Labels: map[string]string{"version": "1.0"}, Annotations: map[string]string{"version": "1.0"}} + ao.Apply(gs) + + require.Equal(t, map[string]string{"colour": "green", "version": "1.0"}, gs.ObjectMeta.Labels) + require.Equal(t, map[string]string{"colour": "blue", "map": "ice cream", "version": "1.0"}, gs.ObjectMeta.Annotations) + + // check overwrite + ao = AllocationOverflow{Labels: map[string]string{"colour": "red"}, Annotations: map[string]string{"colour": "green"}} + ao.Apply(gs) + require.Equal(t, map[string]string{"colour": "red", "version": "1.0"}, gs.ObjectMeta.Labels) + require.Equal(t, map[string]string{"colour": "green", "map": "ice cream", "version": "1.0"}, gs.ObjectMeta.Annotations) +} diff --git a/pkg/apis/agones/v1/fleet.go b/pkg/apis/agones/v1/fleet.go index 8701fb61b8..0131fcf967 100644 --- a/pkg/apis/agones/v1/fleet.go +++ b/pkg/apis/agones/v1/fleet.go @@ -15,9 +15,12 @@ package v1 import ( + "fmt" + "agones.dev/agones/pkg" "agones.dev/agones/pkg/apis" "agones.dev/agones/pkg/apis/agones" + "agones.dev/agones/pkg/util/runtime" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -57,6 +60,12 @@ type FleetList struct { type FleetSpec struct { // Replicas are the number of GameServers that should be in this set. Defaults to 0. Replicas int32 `json:"replicas"` + // [Stage: Alpha] + // [FeatureFlag:FleetAllocationOverflow] + // Labels and Annotations to apply to GameServers when the number of Allocated GameServers drops below + // the desired replicas on the underlying `GameServerSet` + // +optional + AllocationOverflow *AllocationOverflow `json:"allocationOverflow,omitempty"` // Deployment strategy Strategy appsv1.DeploymentStrategy `json:"strategy"` // Scheduling strategy. Defaults to "Packed". @@ -110,6 +119,10 @@ func (f *Fleet) GameServerSet() *GameServerSet { gsSet.ObjectMeta.Labels[FleetNameLabel] = f.ObjectMeta.Name + if runtime.FeatureEnabled(runtime.FeatureFleetAllocateOverflow) && f.Spec.AllocationOverflow != nil { + gsSet.Spec.AllocationOverflow = f.Spec.AllocationOverflow.DeepCopy() + } + return gsSet } @@ -185,6 +198,7 @@ func (f *Fleet) Validate(apiHooks APIHooks) ([]metav1.StatusCause, bool) { Message: "Strategy Type should be one of: RollingUpdate, Recreate.", }) } + // check Gameserver specification in a Fleet gsCauses := validateGSSpec(apiHooks, f) if len(gsCauses) > 0 { @@ -198,6 +212,21 @@ func (f *Fleet) Validate(apiHooks APIHooks) ([]metav1.StatusCause, bool) { causes = append(causes, objMetaCauses...) } + if f.Spec.AllocationOverflow != nil { + if runtime.FeatureEnabled(runtime.FeatureFleetAllocateOverflow) { + aoCauses, valid := f.Spec.AllocationOverflow.Validate() + if !valid { + causes = append(causes, aoCauses...) + } + } else { + causes = append(causes, metav1.StatusCause{ + Type: metav1.CauseTypeFieldValueNotSupported, + Field: "allocationOverflow", + Message: fmt.Sprintf("Value cannot be set unless feature flag %s is enabled", runtime.FeatureFleetAllocateOverflow), + }) + } + } + return causes, len(causes) == 0 } diff --git a/pkg/apis/agones/v1/fleet_test.go b/pkg/apis/agones/v1/fleet_test.go index cc7cc4634d..bc6a7d62ec 100644 --- a/pkg/apis/agones/v1/fleet_test.go +++ b/pkg/apis/agones/v1/fleet_test.go @@ -15,11 +15,14 @@ package v1 import ( + "fmt" "strings" "testing" "agones.dev/agones/pkg/apis" + "agones.dev/agones/pkg/util/runtime" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,6 +31,8 @@ import ( ) func TestFleetGameServerSetGameServer(t *testing.T) { + t.Parallel() + f := Fleet{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -59,6 +64,22 @@ func TestFleetGameServerSetGameServer(t *testing.T) { assert.Equal(t, f.Spec.Scheduling, gsSet.Spec.Scheduling) assert.Equal(t, f.Spec.Template, gsSet.Spec.Template) assert.True(t, metav1.IsControlledBy(gsSet, &f)) + + runtime.FeatureTestMutex.Lock() + defer runtime.FeatureTestMutex.Unlock() + + runtime.Must(runtime.ParseFeatures(fmt.Sprintf("%s=true", runtime.FeatureFleetAllocateOverflow))) + gsSet = f.GameServerSet() + assert.Nil(t, gsSet.Spec.AllocationOverflow) + + f.Spec.AllocationOverflow = &AllocationOverflow{ + Labels: map[string]string{"stuff": "things"}, + Annotations: nil, + } + + gsSet = f.GameServerSet() + assert.NotNil(t, gsSet.Spec.AllocationOverflow) + assert.Equal(t, "things", gsSet.Spec.AllocationOverflow.Labels["stuff"]) } func TestFleetApplyDefaults(t *testing.T) { @@ -192,6 +213,37 @@ func TestFleetGameserverSpec(t *testing.T) { assert.Len(t, causes, 1) } +func TestFleetAllocationOverflow(t *testing.T) { + t.Parallel() + runtime.FeatureTestMutex.Lock() + defer runtime.FeatureTestMutex.Unlock() + + runtime.Must(runtime.ParseFeatures(fmt.Sprintf("%s=true", runtime.FeatureFleetAllocateOverflow))) + + f := defaultFleet() + f.ApplyDefaults() + + causes, valid := f.Validate(fakeAPIHooks{}) + require.True(t, valid) + require.Empty(t, causes) + + f.Spec.AllocationOverflow = &AllocationOverflow{ + Labels: map[string]string{"$$$nope": "value"}, + Annotations: nil, + } + + causes, valid = f.Validate(fakeAPIHooks{}) + require.False(t, valid) + require.Len(t, causes, 1) + require.Equal(t, metav1.CauseTypeFieldValueInvalid, causes[0].Type) + + runtime.Must(runtime.ParseFeatures(fmt.Sprintf("%s=false", runtime.FeatureFleetAllocateOverflow))) + causes, valid = f.Validate(fakeAPIHooks{}) + require.False(t, valid) + require.Len(t, causes, 1) + require.Equal(t, metav1.CauseTypeFieldValueNotSupported, causes[0].Type) +} + func TestFleetName(t *testing.T) { f := defaultFleet() f.ApplyDefaults() diff --git a/pkg/apis/agones/v1/gameserverset.go b/pkg/apis/agones/v1/gameserverset.go index 79f28f6998..7399e76dc3 100644 --- a/pkg/apis/agones/v1/gameserverset.go +++ b/pkg/apis/agones/v1/gameserverset.go @@ -57,6 +57,12 @@ type GameServerSetList struct { type GameServerSetSpec struct { // Replicas are the number of GameServers that should be in this set Replicas int32 `json:"replicas"` + // [Stage: Alpha] + // [FeatureFlag:FleetAllocationOverflow] + // Labels and Annotations to apply to GameServers when the number of Allocated GameServers drops below + // the desired replicas on the underlying `GameServerSet` + // +optional + AllocationOverflow *AllocationOverflow `json:"allocationOverflow,omitempty"` // Scheduling strategy. Defaults to "Packed". Scheduling apis.SchedulingStrategy `json:"scheduling,omitempty"` // Template the GameServer template to apply for this GameServerSet diff --git a/pkg/apis/agones/v1/zz_generated.deepcopy.go b/pkg/apis/agones/v1/zz_generated.deepcopy.go index ca32e8d69b..99bc073c09 100644 --- a/pkg/apis/agones/v1/zz_generated.deepcopy.go +++ b/pkg/apis/agones/v1/zz_generated.deepcopy.go @@ -41,6 +41,36 @@ func (in *AggregatedPlayerStatus) DeepCopy() *AggregatedPlayerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AllocationOverflow) DeepCopyInto(out *AllocationOverflow) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllocationOverflow. +func (in *AllocationOverflow) DeepCopy() *AllocationOverflow { + if in == nil { + return nil + } + out := new(AllocationOverflow) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CounterSpec) DeepCopyInto(out *CounterSpec) { *out = *in @@ -137,6 +167,11 @@ func (in *FleetList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FleetSpec) DeepCopyInto(out *FleetSpec) { *out = *in + if in.AllocationOverflow != nil { + in, out := &in.AllocationOverflow, &out.AllocationOverflow + *out = new(AllocationOverflow) + (*in).DeepCopyInto(*out) + } in.Strategy.DeepCopyInto(&out.Strategy) in.Template.DeepCopyInto(&out.Template) return @@ -319,6 +354,11 @@ func (in *GameServerSetList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GameServerSetSpec) DeepCopyInto(out *GameServerSetSpec) { *out = *in + if in.AllocationOverflow != nil { + in, out := &in.AllocationOverflow, &out.AllocationOverflow + *out = new(AllocationOverflow) + (*in).DeepCopyInto(*out) + } in.Template.DeepCopyInto(&out.Template) return } diff --git a/site/content/en/docs/Reference/agones_crd_api_reference.html b/site/content/en/docs/Reference/agones_crd_api_reference.html index 74a3ec271a..861fc679c1 100644 --- a/site/content/en/docs/Reference/agones_crd_api_reference.html +++ b/site/content/en/docs/Reference/agones_crd_api_reference.html @@ -3239,6 +3239,23 @@

Fleet +allocationOverflow
+ + +AllocationOverflow + + + + +(Optional) +

[Stage: Alpha] +[FeatureFlag:FleetAllocationOverflow] +Labels and Annotations to apply to GameServers when the number of Allocated GameServers drops below +the desired replicas on the underlying GameServerSet

+ + + + strategy
@@ -3570,6 +3587,23 @@

GameServerSet +allocationOverflow
+ +
+AllocationOverflow + + + + +(Optional) +

[Stage: Alpha] +[FeatureFlag:FleetAllocationOverflow] +Labels and Annotations to apply to GameServers when the number of Allocated GameServers drops below +the desired replicas on the underlying GameServerSet

+ + + + scheduling
agones.dev/agones/pkg/apis.SchedulingStrategy @@ -3649,6 +3683,52 @@

AggregatedPlayerStatus +

AllocationOverflow +

+

+(Appears on: +FleetSpec, +GameServerSetSpec) +

+

+

AllocationOverflow specifies what labels and/or annotations to apply on Allocated GameServers +if the desired number of the underlying GameServerSet drops below the number of Allocated GameServers +attached to it.

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+labels
+ +map[string]string + +
+(Optional) +

Labels to be applied to the GameServer

+
+annotations
+ +map[string]string + +
+(Optional) +

Annotations to be applied to the GameServer

+

CounterSpec

@@ -3764,6 +3844,23 @@

FleetSpec +allocationOverflow
+ + +AllocationOverflow + + + + +(Optional) +

[Stage: Alpha] +[FeatureFlag:FleetAllocationOverflow] +Labels and Annotations to apply to GameServers when the number of Allocated GameServers drops below +the desired replicas on the underlying GameServerSet

+ + + + strategy
@@ -4007,6 +4104,23 @@

GameServerSetSpec +allocationOverflow
+ +
+AllocationOverflow + + + + +(Optional) +

[Stage: Alpha] +[FeatureFlag:FleetAllocationOverflow] +Labels and Annotations to apply to GameServers when the number of Allocated GameServers drops below +the desired replicas on the underlying GameServerSet

+ + + + scheduling
agones.dev/agones/pkg/apis.SchedulingStrategy