Skip to content

Commit

Permalink
Delete ClusterQueue.Lendable (#2563)
Browse files Browse the repository at this point in the history
  • Loading branch information
@gabesaba
gabesaba committed Jul 9, 2024
1 parent 1019324 commit 408f2cb
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 78 deletions.
52 changes: 0 additions & 52 deletions pkg/cache/cache_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "default", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 10_000,
},
Status: active,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand All @@ -134,9 +131,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "default", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 15_000,
},
Status: active,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand Down Expand Up @@ -172,9 +166,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 15_000,
},
Status: pending,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand Down Expand Up @@ -272,9 +263,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "default", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 10_000,
},
Status: active,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand All @@ -290,9 +278,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "default", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 15_000,
},
Status: active,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand Down Expand Up @@ -328,9 +313,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 15_000,
},
Status: pending,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand Down Expand Up @@ -403,9 +385,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "default", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 5_000,
},
Status: active,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand Down Expand Up @@ -454,9 +433,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "default", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 4_000,
},
Status: active,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand Down Expand Up @@ -509,9 +485,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "default", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 15_000,
},
Status: active,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand All @@ -537,9 +510,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 15_000,
},
Status: pending,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand Down Expand Up @@ -587,9 +557,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "default", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 10_000,
},
Status: active,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand All @@ -605,9 +572,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "default", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 15_000,
},
Status: active,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand Down Expand Up @@ -643,9 +607,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 15_000,
},
Status: active,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand Down Expand Up @@ -716,11 +677,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
{Flavor: "theta", Resource: "example.com/gpu"}: 0,
{Flavor: "gamma", Resource: "example.com/gpu"}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 0,
corev1.ResourceMemory: 0,
"example.com/gpu": 0,
},
Status: pending,
Preemption: defaultPreemption,
FairWeight: oneQuantity,
Expand Down Expand Up @@ -902,9 +858,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
AdmittedUsage: resources.FlavorResourceQuantitiesFlat{
{Flavor: "f1", Resource: corev1.ResourceCPU}: 1000,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 10_000,
},
FairWeight: oneQuantity,
Workloads: map[string]*workload.Info{
"ns/reserving": {
Expand Down Expand Up @@ -1020,11 +973,6 @@ func TestCacheClusterQueueOperations(t *testing.T) {
{Flavor: "spot", Resource: corev1.ResourceMemory}: 0,
{Flavor: "license", Resource: "license"}: 0,
}.Unflatten(),
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 28_000,
corev1.ResourceMemory: 28 * utiltesting.Gi,
"license": 4,
},
},
},
},
Expand Down
15 changes: 6 additions & 9 deletions pkg/cache/clusterqueue.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,6 @@ type clusterQueue struct {
// deleted, or the resource groups are changed.
AllocatableResourceGeneration int64

// Lendable holds the total lendable quota for the resources of the ClusterQueue, independent of the flavor.
Lendable map[corev1.ResourceName]int64

AdmittedUsage resources.FlavorResourceQuantities
// localQueues by (namespace/name).
localQueues map[string]*queue
Expand Down Expand Up @@ -130,8 +127,12 @@ func newCohort(name string, size int) *cohort {
func (c *cohort) CalculateLendable() map[corev1.ResourceName]int64 {
lendable := make(map[corev1.ResourceName]int64)
for member := range c.Members {
for res, v := range member.Lendable {
lendable[res] += v
for _, frq := range flavorResourceQuotas(member) {
if features.Enabled(features.LendingLimit) && frq.quota.LendingLimit != nil {
lendable[frq.fr.Resource] += *frq.quota.LendingLimit
} else {
lendable[frq.fr.Resource] += frq.quota.Nominal
}
}
}
return lendable
Expand Down Expand Up @@ -245,7 +246,6 @@ func filterFlavorQuantities(orig resources.FlavorResourceQuantities, resourceGro
func (c *clusterQueue) updateResourceGroups(in []kueue.ResourceGroup) {
oldRG := c.ResourceGroups
c.ResourceGroups = make([]ResourceGroup, len(in))
c.Lendable = make(map[corev1.ResourceName]int64)
for i, rgIn := range in {
rg := &c.ResourceGroups[i]
*rg = ResourceGroup{
Expand All @@ -268,9 +268,6 @@ func (c *clusterQueue) updateResourceGroups(in []kueue.ResourceGroup) {
}
if features.Enabled(features.LendingLimit) && rIn.LendingLimit != nil {
rQuota.LendingLimit = ptr.To(resources.ResourceValue(rIn.Name, *rIn.LendingLimit))
c.Lendable[rIn.Name] += *rQuota.LendingLimit
} else {
c.Lendable[rIn.Name] += nominal
}
fQuotas.Resources[rIn.Name] = &rQuota
}
Expand Down
43 changes: 26 additions & 17 deletions pkg/cache/clusterqueue_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package cache

import (
"context"
"math"
"testing"

Expand Down Expand Up @@ -1074,30 +1075,38 @@ func TestDominantResourceShare(t *testing.T) {
}

func TestCohortLendable(t *testing.T) {
cq := clusterQueue{
Cohort: &cohort{
Members: sets.New(
&clusterQueue{
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 8_000,
"example.com/gpu": 3,
},
},
&clusterQueue{
Lendable: map[corev1.ResourceName]int64{
corev1.ResourceCPU: 2_000,
},
},
),
},
cache := New(utiltesting.NewFakeClient())

cq1 := utiltesting.MakeClusterQueue("cq1").
ResourceGroup(
utiltesting.MakeFlavorQuotas("default").
ResourceQuotaWrapper("cpu").NominalQuota("8").LendingLimit("8").Append().
ResourceQuotaWrapper("example.com/gpu").NominalQuota("3").LendingLimit("3").Append().
FlavorQuotas,
).Cohort("test-cohort").
ClusterQueue

cq2 := utiltesting.MakeClusterQueue("cq2").
ResourceGroup(
utiltesting.MakeFlavorQuotas("default").
ResourceQuotaWrapper("cpu").NominalQuota("2").LendingLimit("2").Append().
FlavorQuotas,
).Cohort("test-cohort").
ClusterQueue

if err := cache.AddClusterQueue(context.Background(), &cq1); err != nil {
t.Fatal("Failed to add CQ to cache", err)
}
if err := cache.AddClusterQueue(context.Background(), &cq2); err != nil {
t.Fatal("Failed to add CQ to cache", err)
}

wantLendable := map[corev1.ResourceName]int64{
corev1.ResourceCPU: 10_000,
"example.com/gpu": 3,
}

lendable := cq.Cohort.CalculateLendable()
lendable := cache.clusterQueues["cq1"].Cohort.CalculateLendable()
if diff := cmp.Diff(wantLendable, lendable); diff != "" {
t.Errorf("Unexpected cohort lendable (-want,+got):\n%s", diff)
}
Expand Down

0 comments on commit 408f2cb

Please sign in to comment.