From e0f3f005717055b070f68c2cf68c77804342e006 Mon Sep 17 00:00:00 2001 From: Frame Date: Wed, 30 Aug 2023 15:20:47 +0800 Subject: [PATCH] release v1.3.0 (#7) Signed-off-by: saintube --- .../cluster_colocation_profile_types.go | 6 +- config/v1alpha1/zz_generated.deepcopy.go | 6 + .../slo_controller_config.go | 76 +++++++-- .../zz_generated.deepcopy.go | 12 +- extension/cluster_colocation_profile.go | 33 ++++ extension/constants.go | 15 ++ extension/coscheduling.go | 92 +++++++++++ extension/deprecated.go | 20 +-- extension/node_reservation.go | 65 +++++--- extension/operating_pod.go | 93 +++++++++++ extension/priority.go | 32 +++- extension/priority_utils.go | 47 ++++++ extension/qos_utils.go | 46 +++++- extension/reservation.go | 47 +++++- extension/resource.go | 21 +++ extension/scheduling.go | 56 +------ extension/system_qos.go | 53 +++++++ go.mod | 3 +- go.sum | 34 +--- hack/clone-api-files.sh | 2 + scheduling/v1alpha1/device_types.go | 29 +++- scheduling/v1alpha1/reservation_types.go | 28 +++- scheduling/v1alpha1/zz_generated.deepcopy.go | 86 ++++++++++ slo/v1alpha1/nodemetric_types.go | 31 ++-- slo/v1alpha1/nodeslo_types.go | 110 +++++++++---- {extension => slo/v1alpha1}/pod.go | 18 ++- slo/v1alpha1/zz_generated.deepcopy.go | 150 +++++++++++++++++- 27 files changed, 1011 insertions(+), 200 deletions(-) rename {extension => configuration}/slo_controller_config.go (79%) rename {extension => configuration}/zz_generated.deepcopy.go (97%) create mode 100644 extension/cluster_colocation_profile.go create mode 100644 extension/coscheduling.go create mode 100644 extension/operating_pod.go create mode 100644 extension/priority_utils.go create mode 100644 extension/system_qos.go rename {extension => slo/v1alpha1}/pod.go (72%) diff --git a/config/v1alpha1/cluster_colocation_profile_types.go b/config/v1alpha1/cluster_colocation_profile_types.go index 6d46597..286f5b0 100644 --- a/config/v1alpha1/cluster_colocation_profile_types.go +++ b/config/v1alpha1/cluster_colocation_profile_types.go @@ -19,6 +19,7 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" ) // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! @@ -38,6 +39,10 @@ type ClusterColocationProfileSpec struct { // +optional Selector *metav1.LabelSelector `json:"selector,omitempty"` + // Probability indicates profile will make effect with a probability. + // +optional + Probability *intstr.IntOrString `json:"probability,omitempty"` + // QoSClass describes the type of Koordinator QoS that the Pod is running. // The value will be injected into Pod as label koordinator.sh/qosClass. // Options are LSE/LSR/LS/BE/SYSTEM. @@ -50,7 +55,6 @@ type ClusterColocationProfileSpec struct { // The PriorityClassName, priority value in PriorityClassName and // KoordinatorPriority will affect the scheduling, preemption and // other behaviors of Koordinator system. - // +kubebuilder:validation:Enum=koord-prod;koord-mid;koord-batch;koord-free // +optional PriorityClassName string `json:"priorityClassName"` diff --git a/config/v1alpha1/zz_generated.deepcopy.go b/config/v1alpha1/zz_generated.deepcopy.go index 7cea407..91de457 100644 --- a/config/v1alpha1/zz_generated.deepcopy.go +++ b/config/v1alpha1/zz_generated.deepcopy.go @@ -24,6 +24,7 @@ package v1alpha1 import ( "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -98,6 +99,11 @@ func (in *ClusterColocationProfileSpec) DeepCopyInto(out *ClusterColocationProfi *out = new(v1.LabelSelector) (*in).DeepCopyInto(*out) } + if in.Probability != nil { + in, out := &in.Probability, &out.Probability + *out = new(intstr.IntOrString) + **out = **in + } if in.KoordinatorPriority != nil { in, out := &in.KoordinatorPriority, &out.KoordinatorPriority *out = new(int32) diff --git a/extension/slo_controller_config.go b/configuration/slo_controller_config.go similarity index 79% rename from extension/slo_controller_config.go rename to configuration/slo_controller_config.go index 64f1786..174abbd 100644 --- a/extension/slo_controller_config.go +++ b/configuration/slo_controller_config.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package extension +package configuration import ( "github.com/mohae/deepcopy" @@ -43,7 +43,7 @@ type NodeCfgProfile struct { // +k8s:deepcopy-gen=true type ColocationCfg struct { ColocationStrategy `json:",inline"` - NodeConfigs []NodeColocationCfg `json:"nodeConfigs,omitempty"` + NodeConfigs []NodeColocationCfg `json:"nodeConfigs,omitempty" validate:"dive"` } // +k8s:deepcopy-gen=true @@ -55,7 +55,7 @@ type NodeColocationCfg struct { // +k8s:deepcopy-gen=true type ResourceThresholdCfg struct { ClusterStrategy *slov1alpha1.ResourceThresholdStrategy `json:"clusterStrategy,omitempty"` - NodeStrategies []NodeResourceThresholdStrategy `json:"nodeStrategies,omitempty"` + NodeStrategies []NodeResourceThresholdStrategy `json:"nodeStrategies,omitempty" validate:"dive"` } // +k8s:deepcopy-gen=true @@ -73,7 +73,7 @@ type NodeCPUBurstCfg struct { // +k8s:deepcopy-gen=true type CPUBurstCfg struct { ClusterStrategy *slov1alpha1.CPUBurstStrategy `json:"clusterStrategy,omitempty"` - NodeStrategies []NodeCPUBurstCfg `json:"nodeStrategies,omitempty"` + NodeStrategies []NodeCPUBurstCfg `json:"nodeStrategies,omitempty" validate:"dive"` } // +k8s:deepcopy-gen=true @@ -85,13 +85,13 @@ type NodeSystemStrategy struct { // +k8s:deepcopy-gen=true type SystemCfg struct { ClusterStrategy *slov1alpha1.SystemStrategy `json:"clusterStrategy,omitempty"` - NodeStrategies []NodeSystemStrategy `json:"nodeStrategies,omitempty"` + NodeStrategies []NodeSystemStrategy `json:"nodeStrategies,omitempty" validate:"dive"` } // +k8s:deepcopy-gen=true type ResourceQOSCfg struct { ClusterStrategy *slov1alpha1.ResourceQOSStrategy `json:"clusterStrategy,omitempty"` - NodeStrategies []NodeResourceQOSStrategy `json:"nodeStrategies,omitempty"` + NodeStrategies []NodeResourceQOSStrategy `json:"nodeStrategies,omitempty" validate:"dive"` } // +k8s:deepcopy-gen=true @@ -196,16 +196,25 @@ func (in *ExtraFields) DeepCopy() *ExtraFields { // +k8s:deepcopy-gen=true type ColocationStrategy struct { Enable *bool `json:"enable,omitempty"` - MetricAggregateDurationSeconds *int64 `json:"metricAggregateDurationSeconds,omitempty"` - MetricReportIntervalSeconds *int64 `json:"metricReportIntervalSeconds,omitempty"` + MetricAggregateDurationSeconds *int64 `json:"metricAggregateDurationSeconds,omitempty" validate:"omitempty,min=1"` + MetricReportIntervalSeconds *int64 `json:"metricReportIntervalSeconds,omitempty" validate:"omitempty,min=1"` MetricAggregatePolicy *slov1alpha1.AggregatePolicy `json:"metricAggregatePolicy,omitempty"` - CPUReclaimThresholdPercent *int64 `json:"cpuReclaimThresholdPercent,omitempty"` - MemoryReclaimThresholdPercent *int64 `json:"memoryReclaimThresholdPercent,omitempty"` - MemoryCalculatePolicy *CalculatePolicy `json:"memoryCalculatePolicy,omitempty"` - DegradeTimeMinutes *int64 `json:"degradeTimeMinutes,omitempty"` - UpdateTimeThresholdSeconds *int64 `json:"updateTimeThresholdSeconds,omitempty"` - ResourceDiffThreshold *float64 `json:"resourceDiffThreshold,omitempty"` - ColocationStrategyExtender `json:",inline"` // for third-party extension + + CPUReclaimThresholdPercent *int64 `json:"cpuReclaimThresholdPercent,omitempty" validate:"omitempty,min=0,max=100"` + MemoryReclaimThresholdPercent *int64 `json:"memoryReclaimThresholdPercent,omitempty" validate:"omitempty,min=0,max=100"` + MemoryCalculatePolicy *CalculatePolicy `json:"memoryCalculatePolicy,omitempty"` + DegradeTimeMinutes *int64 `json:"degradeTimeMinutes,omitempty" validate:"omitempty,min=1"` + UpdateTimeThresholdSeconds *int64 `json:"updateTimeThresholdSeconds,omitempty" validate:"omitempty,min=1"` + ResourceDiffThreshold *float64 `json:"resourceDiffThreshold,omitempty" validate:"omitempty,gt=0,max=1"` + + // MidCPUThresholdPercent defines the maximum percentage of the Mid-tier cpu resource dividing the node allocatable. + // MidCPUAllocatable <= NodeCPUAllocatable * MidCPUThresholdPercent / 100. + MidCPUThresholdPercent *int64 `json:"midCPUThresholdPercent,omitempty" validate:"omitempty,min=0,max=100"` + // MidMemoryThresholdPercent defines the maximum percentage of the Mid-tier memory resource dividing the node allocatable. + // MidMemoryAllocatable <= NodeMemoryAllocatable * MidMemoryThresholdPercent / 100. + MidMemoryThresholdPercent *int64 `json:"midMemoryThresholdPercent,omitempty" validate:"omitempty,min=0,max=100"` + + ColocationStrategyExtender `json:",inline"` // for third-party extension } /* @@ -363,12 +372,49 @@ data: "priority": 0, "oomKillGroup": 0 }, + "blkioQOS": { + "blocks": [ + { + "ioCfg": { + "ioWeightPercent": 60 + }, + "name": "ackdistro-pool", + "type": "volumegroup" + }, + { + "ioCfg": { + "readBPS": 102400000, + "readIOPS": 20480, + "writeBPS": 204800004, + "writeIOPS": 10240 + }, + "name": "/dev/sdb", + "type": "device" + }, + ], + "enable": true + }, "resctrlQOS": { "enable": false, "catRangeStartPercent": 0, "catRangeEndPercent": 30, "mbaPercent": 100 } + }, + "cgroupRoot": { + "blkioQOS": { + "blocks": [ + { + "ioCfg": { + "readLatency": 3000, + "writeLatency": 3000 + }, + "name": "ackdistro-pool", + "type": "volumegroup" + } + ], + "enable": true + } } }, "nodeStrategies": [ diff --git a/extension/zz_generated.deepcopy.go b/configuration/zz_generated.deepcopy.go similarity index 97% rename from extension/zz_generated.deepcopy.go rename to configuration/zz_generated.deepcopy.go index 3c35037..bcb0b99 100644 --- a/extension/zz_generated.deepcopy.go +++ b/configuration/zz_generated.deepcopy.go @@ -19,7 +19,7 @@ limitations under the License. // Code generated by controller-gen. DO NOT EDIT. -package extension +package configuration import ( "github.com/koordinator-sh/apis/slo/v1alpha1" @@ -129,6 +129,16 @@ func (in *ColocationStrategy) DeepCopyInto(out *ColocationStrategy) { *out = new(float64) **out = **in } + if in.MidCPUThresholdPercent != nil { + in, out := &in.MidCPUThresholdPercent, &out.MidCPUThresholdPercent + *out = new(int64) + **out = **in + } + if in.MidMemoryThresholdPercent != nil { + in, out := &in.MidMemoryThresholdPercent, &out.MidMemoryThresholdPercent + *out = new(int64) + **out = **in + } in.ColocationStrategyExtender.DeepCopyInto(&out.ColocationStrategyExtender) } diff --git a/extension/cluster_colocation_profile.go b/extension/cluster_colocation_profile.go new file mode 100644 index 0000000..f5ba81a --- /dev/null +++ b/extension/cluster_colocation_profile.go @@ -0,0 +1,33 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package extension + +import ( + configv1alpha1 "github.com/koordinator-sh/apis/config/v1alpha1" +) + +const ( + AnnotationSkipUpdateResource = "config.koordinator.sh/skip-update-resources" +) + +func ShouldSkipUpdateResource(profile *configv1alpha1.ClusterColocationProfile) bool { + if profile == nil || profile.Annotations == nil { + return false + } + _, ok := profile.Annotations[AnnotationSkipUpdateResource] + return ok +} diff --git a/extension/constants.go b/extension/constants.go index 1447141..d5d1ca3 100644 --- a/extension/constants.go +++ b/extension/constants.go @@ -29,6 +29,21 @@ const ( LabelPodQoS = DomainPrefix + "qosClass" LabelPodPriority = DomainPrefix + "priority" + // LabelPodPriorityClass is used to revise those Pods that are already running and have Priority set, so that + // Koordinator can be smoothly deployed to the running cluster. If you don't have a running Pod with + // PriorityClass set, don't set this field specifically. + LabelPodPriorityClass = DomainPrefix + "priority-class" LabelManagedBy = "app.kubernetes.io/managed-by" ) + +type AggregationType string + +const ( + // max is not welcomed since it may import outliers + AVG AggregationType = "avg" + P99 AggregationType = "p99" + P95 AggregationType = "p95" + P90 AggregationType = "p90" + P50 AggregationType = "p50" +) diff --git a/extension/coscheduling.go b/extension/coscheduling.go new file mode 100644 index 0000000..51cc7b6 --- /dev/null +++ b/extension/coscheduling.go @@ -0,0 +1,92 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package extension + +import ( + "strconv" + + corev1 "k8s.io/api/core/v1" +) + +const ( + AnnotationGangPrefix = "gang.scheduling.koordinator.sh" + // AnnotationGangName specifies the name of the gang + AnnotationGangName = AnnotationGangPrefix + "/name" + + // AnnotationGangMinNum specifies the minimum number of the gang that can be executed + AnnotationGangMinNum = AnnotationGangPrefix + "/min-available" + + // AnnotationGangWaitTime specifies gang's max wait time in Permit Stage + AnnotationGangWaitTime = AnnotationGangPrefix + "/waiting-time" + + // AnnotationGangTotalNum specifies the total children number of the gang + // If not specified,it will be set with the AnnotationGangMinNum + AnnotationGangTotalNum = AnnotationGangPrefix + "/total-number" + + // AnnotationGangMode defines the Gang Scheduling operation when failed scheduling + // Support GangModeStrict and GangModeNonStrict, default is GangModeStrict + AnnotationGangMode = AnnotationGangPrefix + "/mode" + + // AnnotationGangGroups defines which gangs are bundled as a group + // The gang will go to bind only all gangs in one group meet the conditions + AnnotationGangGroups = AnnotationGangPrefix + "/groups" + + // AnnotationGangTimeout means that the entire gang cannot be scheduled due to timeout + // The annotation is added by the scheduler when the gang times out + AnnotationGangTimeout = AnnotationGangPrefix + "/timeout" + + GangModeStrict = "Strict" + GangModeNonStrict = "NonStrict" + + // AnnotationGangMatchPolicy defines the Gang Scheduling operation of taking which status pod into account + // Support GangMatchPolicyOnlyWaiting, GangMatchPolicyWaitingAndRunning, GangMatchPolicyOnceSatisfied, default is GangMatchPolicyOnceSatisfied + AnnotationGangMatchPolicy = AnnotationGangPrefix + "/match-policy" + GangMatchPolicyOnlyWaiting = "only-waiting" + GangMatchPolicyWaitingAndRunning = "waiting-and-running" + GangMatchPolicyOnceSatisfied = "once-satisfied" + + // AnnotationAliasGangMatchPolicy defines same match policy but different prefix. + // Duplicate definitions here are only for compatibility considerations + AnnotationAliasGangMatchPolicy = "pod-group.scheduling.sigs.k8s.io/match-policy" +) + +const ( + // Deprecated: kubernetes-sigs/scheduler-plugins/lightweight-coscheduling + LabelLightweightCoschedulingPodGroupName = "pod-group.scheduling.sigs.k8s.io/name" + // Deprecated: kubernetes-sigs/scheduler-plugins/lightweight-coscheduling + LabelLightweightCoschedulingPodGroupMinAvailable = "pod-group.scheduling.sigs.k8s.io/min-available" +) + +func GetMinNum(pod *corev1.Pod) (int, error) { + minRequiredNum, err := strconv.ParseInt(pod.Annotations[AnnotationGangMinNum], 10, 32) + if err != nil { + return 0, err + } + return int(minRequiredNum), nil +} + +func GetGangName(pod *corev1.Pod) string { + return pod.Annotations[AnnotationGangName] +} + +func GetGangMatchPolicy(pod *corev1.Pod) string { + policy := pod.Annotations[AnnotationGangMatchPolicy] + if policy != "" { + return policy + } + return pod.Annotations[AnnotationAliasGangMatchPolicy] +} diff --git a/extension/deprecated.go b/extension/deprecated.go index 3ab69e9..25064d7 100644 --- a/extension/deprecated.go +++ b/extension/deprecated.go @@ -45,7 +45,12 @@ const ( DeprecatedGPUModel string = ResourceDomainPrefix + "gpu-model" ) -var deprecatedDeviceResourceNameMapper = map[corev1.ResourceName]corev1.ResourceName{ +var DeprecatedBatchResourcesMapper = map[corev1.ResourceName]corev1.ResourceName{ + KoordBatchCPU: BatchCPU, + KoordBatchMemory: BatchMemory, +} + +var DeprecatedDeviceResourcesMapper = map[corev1.ResourceName]corev1.ResourceName{ DeprecatedKoordRDMA: ResourceRDMA, DeprecatedKoordFPGA: ResourceFPGA, DeprecatedKoordGPU: ResourceGPU, @@ -53,16 +58,3 @@ var deprecatedDeviceResourceNameMapper = map[corev1.ResourceName]corev1.Resource DeprecatedGPUMemory: ResourceGPUMemory, DeprecatedGPUMemoryRatio: ResourceGPUMemoryRatio, } - -func TransformDeprecatedDeviceResources(resList corev1.ResourceList) corev1.ResourceList { - r := make(corev1.ResourceList, len(resList)) - for k, v := range resList { - newResName := deprecatedDeviceResourceNameMapper[k] - if newResName != "" { - r[newResName] = v - } else { - r[k] = v - } - } - return r -} diff --git a/extension/node_reservation.go b/extension/node_reservation.go index 154ebd6..c4a1a5c 100644 --- a/extension/node_reservation.go +++ b/extension/node_reservation.go @@ -30,44 +30,61 @@ const ( // NodeReservation resource reserved by node.annotation, // If node.annotation declares the resources to be reserved, like this: -// annotations: -// node.koordinator.sh/reservation: >- -// {"reservedCPUs":"0-5"} - -// In the filter phase it needs to satisfy: node.alloc - node.req - reserved(6c) > pod.req -// if qos==LSE/LSR: the cores 0-5 are not used in the reserve phase +// +// annotations: +// node.koordinator.sh/reservation: >- +// {"reservedCPUs":"0-5"} type NodeReservation struct { // resources need to be reserved. like, {"cpu":"1C", "memory":"2Gi"} Resources corev1.ResourceList `json:"resources,omitempty"` // reserved cpus need to be reserved, such as 1-6, or 2,4,6,8 ReservedCPUs string `json:"reservedCPUs,omitempty"` + // ApplyPolicy indicates how the reserved resources take effect. + ApplyPolicy NodeReservationApplyPolicy `json:"applyPolicy,omitempty"` } -func GetReservedCPUs(anno map[string]string) (string, int) { - specificCPUsReservedStr := "" - numReservedCPUs := 0 +type NodeReservationApplyPolicy string + +const ( + // NodeReservationApplyPolicyDefault will affect the total amount of schedulable resources of the node and reserve CPU Cores. + // For example, NodeInfo.Allocatable will be modified in the scheduler to deduct the amount of reserved resources + NodeReservationApplyPolicyDefault NodeReservationApplyPolicy = "Default" + // NodeReservationApplyPolicyReservedCPUsOnly means that only CPU Cores are reserved, but it will + // not affect the total amount of schedulable resources of the node. + // The total amount of schedulable resources is taken into effect by the kubelet's reservation mechanism. + // But koordinator need to exclude reserved CPUs when allocating CPU Cores + NodeReservationApplyPolicyReservedCPUsOnly NodeReservationApplyPolicy = "ReservedCPUsOnly" +) - val, ok := anno[AnnotationNodeReservation] - if !ok || val == "" { - return specificCPUsReservedStr, numReservedCPUs +func GetNodeReservation(annotations map[string]string) (*NodeReservation, error) { + if s := annotations[AnnotationNodeReservation]; s != "" { + reservation := &NodeReservation{} + if err := json.Unmarshal([]byte(s), &reservation); err != nil { + return nil, err + } + return reservation, nil } + return nil, nil +} - reserved := NodeReservation{} - if err := json.Unmarshal([]byte(val), &reserved); err != nil { - klog.Errorf("failed to unmarshal reserved resources from node.annotation in nodenumaresource scheduler plugin.err:%v", err) - return specificCPUsReservedStr, numReservedCPUs +func GetReservedCPUs(annotations map[string]string) (reservedCPUs string, numReservedCPUs int) { + reservation, err := GetNodeReservation(annotations) + if err != nil { + klog.ErrorS(err, "failed to GetNodeReservation") + return + } + if reservation == nil { + return } - CPUsQuantityReserved, ok := reserved.Resources[corev1.ResourceCPU] - if ok && CPUsQuantityReserved.MilliValue() > 0 { - reservedCPUsFloat := float64(CPUsQuantityReserved.MilliValue()) / 1000 - numReservedCPUs = int(math.Ceil(reservedCPUsFloat)) + quantity := reservation.Resources[corev1.ResourceCPU] + if quantity.MilliValue() > 0 { + numReservedCPUs = int(math.Ceil(float64(quantity.MilliValue()) / 1000)) } - if reserved.ReservedCPUs != "" { + if reservation.ReservedCPUs != "" { numReservedCPUs = 0 } - specificCPUsReservedStr = reserved.ReservedCPUs - - return specificCPUsReservedStr, numReservedCPUs + reservedCPUs = reservation.ReservedCPUs + return } diff --git a/extension/operating_pod.go b/extension/operating_pod.go new file mode 100644 index 0000000..dd911a8 --- /dev/null +++ b/extension/operating_pod.go @@ -0,0 +1,93 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package extension + +import ( + "encoding/json" + + corev1 "k8s.io/api/core/v1" + + schedulingv1alpha1 "github.com/koordinator-sh/apis/scheduling/v1alpha1" +) + +const ( + // LabelPodOperatingMode describes the mode of operation for Pod. + LabelPodOperatingMode = SchedulingDomainPrefix + "/operating-mode" + + // AnnotationReservationOwners indicates the owner specification which can allocate reserved resources + AnnotationReservationOwners = SchedulingDomainPrefix + "/reservation-owners" + + // AnnotationReservationCurrentOwner indicates current resource owners which allocated the reservation resources. + AnnotationReservationCurrentOwner = SchedulingDomainPrefix + "/reservation-current-owner" +) + +// The concept of PodOperatingMode refers to the design document https://docs.google.com/document/d/1sbFUA_9qWtorJkcukNULr12FKX6lMvISiINxAURHNFo/edit#heading=h.xgjl2srtytjt + +type PodOperatingMode string + +const ( + // RunnablePodOperatingMode represents the original pod behavior, it is the default mode where the + // pod’s containers are executed by Kubelet when the pod is assigned a node. + RunnablePodOperatingMode PodOperatingMode = "Runnable" + + // ReservationPodOperatingMode means the pod represents a scheduling and resource reservation unit + ReservationPodOperatingMode PodOperatingMode = "Reservation" +) + +func IsReservationOperatingMode(pod *corev1.Pod) bool { + return pod.Labels[LabelPodOperatingMode] == string(ReservationPodOperatingMode) +} + +func GetReservationOwners(annotations map[string]string) ([]schedulingv1alpha1.ReservationOwner, error) { + var owners []schedulingv1alpha1.ReservationOwner + if s := annotations[AnnotationReservationOwners]; s != "" { + err := json.Unmarshal([]byte(s), &owners) + if err != nil { + return nil, err + } + } + return owners, nil +} + +func GetReservationCurrentOwner(annotations map[string]string) (*corev1.ObjectReference, error) { + var owner corev1.ObjectReference + s := annotations[AnnotationReservationCurrentOwner] + if s == "" { + return nil, nil + } + err := json.Unmarshal([]byte(s), &owner) + if err != nil { + return nil, err + } + return &owner, nil +} + +func SetReservationCurrentOwner(annotations map[string]string, owner *corev1.ObjectReference) error { + if owner == nil { + return nil + } + data, err := json.Marshal(owner) + if err != nil { + return err + } + annotations[AnnotationReservationCurrentOwner] = string(data) + return nil +} + +func RemoveReservationCurrentOwner(annotations map[string]string) { + delete(annotations, AnnotationReservationCurrentOwner) +} diff --git a/extension/priority.go b/extension/priority.go index bb65a9e..bc62fda 100644 --- a/extension/priority.go +++ b/extension/priority.go @@ -48,8 +48,34 @@ var ( PriorityFreeValueMin int32 = 3000 ) -func GetPriorityClass(pod *corev1.Pod) PriorityClass { - if pod == nil || pod.Spec.Priority == nil { +// KnownPriorityClasses is the list of known priority classes in koordinator. +var KnownPriorityClasses = []PriorityClass{ + PriorityProd, + PriorityMid, + PriorityBatch, + PriorityFree, + PriorityNone, +} + +func GetPodPriorityClassByName(priorityClass string) PriorityClass { + p := PriorityClass(priorityClass) + + switch p { + case PriorityProd, PriorityMid, PriorityBatch, PriorityFree: + return p + } + + return PriorityNone +} + +func GetPodPriorityClassRaw(pod *corev1.Pod) PriorityClass { + if pod == nil { + return PriorityNone + } + if p, ok := pod.Labels[LabelPodPriorityClass]; ok { + return GetPodPriorityClassByName(p) + } + if pod.Spec.Priority == nil { return PriorityNone } return getPriorityClassByPriority(pod.Spec.Priority) @@ -71,7 +97,7 @@ func getPriorityClassByPriority(priority *int32) PriorityClass { return PriorityFree } - return PriorityNone + return DefaultPriorityClass } // GetPodSubPriority get pod's sub-priority in Koordinator from label diff --git a/extension/priority_utils.go b/extension/priority_utils.go new file mode 100644 index 0000000..3035aee --- /dev/null +++ b/extension/priority_utils.go @@ -0,0 +1,47 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package extension + +import corev1 "k8s.io/api/core/v1" + +// NOTE: functions in this file can be overwritten for extension + +var DefaultPriorityClass = PriorityNone + +// GetPodPriorityClassWithDefault gets the pod's PriorityClass with the default config. +func GetPodPriorityClassWithDefault(pod *corev1.Pod) PriorityClass { + priorityClass := GetPodPriorityClassRaw(pod) + if priorityClass != PriorityNone { + return priorityClass + } + + return GetPodPriorityClassWithQoS(GetPodQoSClassWithDefault(pod)) +} + +// GetPodPriorityClassWithQoS returns the default PriorityClass according to its QoSClass when the pod does not specify +// a PriorityClass explicitly. +// Note that this is only a derivation of the default value, and the reverse is not true. For example, PriorityMid +// can also be combined with QoSLS. +func GetPodPriorityClassWithQoS(qos QoSClass) PriorityClass { + switch qos { + case QoSSystem, QoSLSE, QoSLSR, QoSLS: + return PriorityProd + case QoSBE: + return PriorityBatch + } + return DefaultPriorityClass +} diff --git a/extension/qos_utils.go b/extension/qos_utils.go index 5577145..63ae562 100644 --- a/extension/qos_utils.go +++ b/extension/qos_utils.go @@ -16,11 +16,45 @@ limitations under the License. package extension -import corev1 "k8s.io/api/core/v1" +import ( + corev1 "k8s.io/api/core/v1" + v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" +) // NOTE: functions in this file can be overwritten for extension -func GetPodQoSClass(pod *corev1.Pod) QoSClass { +// QoSClassForGuaranteed indicates the QoSClass which a Guaranteed Pod without a koordinator QoSClass specified should +// be regarded by default. +// TODO: add component options to customize it. +var QoSClassForGuaranteed = QoSLSR + +// GetPodQoSClassWithDefault gets the pod's QoSClass with the default config. +func GetPodQoSClassWithDefault(pod *corev1.Pod) QoSClass { + qosClass := GetPodQoSClassRaw(pod) + if qosClass != QoSNone { + return qosClass + } + + return GetPodQoSClassWithKubeQoS(GetKubeQosClass(pod)) +} + +// GetPodQoSClassWithKubeQoS returns the default QoSClass according to its kubernetes QoSClass when the pod does not +// specify a koordinator QoSClass explicitly. +// https://koordinator.sh/docs/architecture/qos#koordinator-qos-vs-kubernetes-qos +func GetPodQoSClassWithKubeQoS(kubeQOS corev1.PodQOSClass) QoSClass { + switch kubeQOS { + case corev1.PodQOSGuaranteed: + return QoSClassForGuaranteed + case corev1.PodQOSBurstable: + return QoSLS + case corev1.PodQOSBestEffort: + return QoSBE + } + // should never reach here + return QoSNone +} + +func GetPodQoSClassRaw(pod *corev1.Pod) QoSClass { if pod == nil || pod.Labels == nil { return QoSNone } @@ -34,3 +68,11 @@ func GetQoSClassByAttrs(labels, annotations map[string]string) QoSClass { } return QoSNone } + +func GetKubeQosClass(pod *corev1.Pod) corev1.PodQOSClass { + qosClass := pod.Status.QOSClass + if len(qosClass) > 0 { + return qosClass + } + return v1qos.GetPodQOS(pod) +} diff --git a/extension/reservation.go b/extension/reservation.go index 33057c0..653e30a 100644 --- a/extension/reservation.go +++ b/extension/reservation.go @@ -20,7 +20,9 @@ import ( "encoding/json" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" schedulingv1alpha1 "github.com/koordinator-sh/apis/scheduling/v1alpha1" ) @@ -33,6 +35,9 @@ const ( // AnnotationReservationAllocated represents the reservation allocated by the pod. AnnotationReservationAllocated = SchedulingDomainPrefix + "/reservation-allocated" + + // AnnotationReservationAffinity represents the constraints of Pod selection Reservation + AnnotationReservationAffinity = SchedulingDomainPrefix + "/reservation-affinity" ) type ReservationAllocated struct { @@ -40,6 +45,28 @@ type ReservationAllocated struct { UID types.UID `json:"uid,omitempty"` } +// ReservationAffinity represents the constraints of Pod selection Reservation +type ReservationAffinity struct { + // If the affinity requirements specified by this field are not met at + // scheduling time, the pod will not be scheduled onto the node. + // If the affinity requirements specified by this field cease to be met + // at some point during pod execution (e.g. due to an update), the system + // may or may not try to eventually evict the pod from its node. + RequiredDuringSchedulingIgnoredDuringExecution *ReservationAffinitySelector `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty"` + // ReservationSelector is a selector which must be true for the pod to fit on a reservation. + // Selector which must match a reservation's labels for the pod to be scheduled on that node. + ReservationSelector map[string]string `json:"reservationSelector,omitempty"` +} + +// ReservationAffinitySelector represents the union of the results of one or more label queries +// over a set of reservations; that is, it represents the OR of the selectors represented +// by the reservation selector terms. +type ReservationAffinitySelector struct { + // Required. A list of reservation selector terms. The terms are ORed. + // Reuse corev1.NodeSelectorTerm to avoid defining too many repeated definitions. + ReservationSelectorTerms []corev1.NodeSelectorTerm `json:"reservationSelectorTerms,omitempty"` +} + func GetReservationAllocated(pod *corev1.Pod) (*ReservationAllocated, error) { if pod.Annotations == nil { return nil, nil @@ -56,14 +83,28 @@ func GetReservationAllocated(pod *corev1.Pod) (*ReservationAllocated, error) { return reservationAllocated, nil } -func SetReservationAllocated(pod *corev1.Pod, r *schedulingv1alpha1.Reservation) { +func SetReservationAllocated(pod *corev1.Pod, r metav1.Object) { if pod.Annotations == nil { pod.Annotations = map[string]string{} } reservationAllocated := &ReservationAllocated{ - Name: r.Name, - UID: r.UID, + Name: r.GetName(), + UID: r.GetUID(), } data, _ := json.Marshal(reservationAllocated) // assert no error pod.Annotations[AnnotationReservationAllocated] = string(data) } + +func IsReservationAllocateOnce(r *schedulingv1alpha1.Reservation) bool { + return pointer.BoolDeref(r.Spec.AllocateOnce, true) +} + +func GetReservationAffinity(annotations map[string]string) (*ReservationAffinity, error) { + var affinity ReservationAffinity + if s := annotations[AnnotationReservationAffinity]; s != "" { + if err := json.Unmarshal([]byte(s), &affinity); err != nil { + return nil, err + } + } + return &affinity, nil +} diff --git a/extension/resource.go b/extension/resource.go index 1b215a3..7f23b4d 100644 --- a/extension/resource.go +++ b/extension/resource.go @@ -26,8 +26,11 @@ import ( const ( BatchCPU corev1.ResourceName = ResourceDomainPrefix + "batch-cpu" BatchMemory corev1.ResourceName = ResourceDomainPrefix + "batch-memory" + MidCPU corev1.ResourceName = ResourceDomainPrefix + "mid-cpu" + MidMemory corev1.ResourceName = ResourceDomainPrefix + "mid-memory" ResourceNvidiaGPU corev1.ResourceName = "nvidia.com/gpu" + ResourceHygonDCU corev1.ResourceName = "dcu.com/gpu" ResourceRDMA corev1.ResourceName = DomainPrefix + "rdma" ResourceFPGA corev1.ResourceName = DomainPrefix + "fpga" ResourceGPU corev1.ResourceName = DomainPrefix + "gpu" @@ -61,6 +64,10 @@ var ( corev1.ResourceCPU: BatchCPU, corev1.ResourceMemory: BatchMemory, }, + PriorityMid: { + corev1.ResourceCPU: MidCPU, + corev1.ResourceMemory: MidMemory, + }, } ) @@ -142,6 +149,20 @@ func GetResourceSpec(annotations map[string]string) (*ResourceSpec, error) { return resourceSpec, nil } +func SetResourceSpec(obj metav1.Object, spec *ResourceSpec) error { + data, err := json.Marshal(spec) + if err != nil { + return err + } + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[AnnotationResourceSpec] = string(data) + obj.SetAnnotations(annotations) + return nil +} + // GetResourceStatus parses ResourceStatus from annotations func GetResourceStatus(annotations map[string]string) (*ResourceStatus, error) { resourceStatus := &ResourceStatus{} diff --git a/extension/scheduling.go b/extension/scheduling.go index 27a6cb6..f569c4d 100644 --- a/extension/scheduling.go +++ b/extension/scheduling.go @@ -18,13 +18,11 @@ package extension import ( "encoding/json" - "strconv" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" schedulingv1alpha1 "github.com/koordinator-sh/apis/scheduling/v1alpha1" - slov1alpha1 "github.com/koordinator-sh/apis/slo/v1alpha1" ) const ( @@ -36,44 +34,6 @@ const ( AnnotationDeviceAllocated = SchedulingDomainPrefix + "/device-allocated" ) -const ( - AnnotationGangPrefix = "gang.scheduling.koordinator.sh" - // AnnotationGangName specifies the name of the gang - AnnotationGangName = AnnotationGangPrefix + "/name" - - // AnnotationGangMinNum specifies the minimum number of the gang that can be executed - AnnotationGangMinNum = AnnotationGangPrefix + "/min-available" - - // AnnotationGangWaitTime specifies gang's max wait time in Permit Stage - AnnotationGangWaitTime = AnnotationGangPrefix + "/waiting-time" - - // AnnotationGangTotalNum specifies the total children number of the gang - // If not specified,it will be set with the AnnotationGangMinNum - AnnotationGangTotalNum = AnnotationGangPrefix + "/total-number" - - // AnnotationGangMode defines the Gang Scheduling operation when failed scheduling - // Support GangModeStrict and GangModeNonStrict, default is GangModeStrict - AnnotationGangMode = AnnotationGangPrefix + "/mode" - - // AnnotationGangGroups defines which gangs are bundled as a group - // The gang will go to bind only all gangs in one group meet the conditions - AnnotationGangGroups = AnnotationGangPrefix + "/groups" - - // AnnotationGangTimeout means that the entire gang cannot be scheduled due to timeout - // The annotation is added by the scheduler when the gang times out - AnnotationGangTimeout = AnnotationGangPrefix + "/timeout" - - GangModeStrict = "Strict" - GangModeNonStrict = "NonStrict" -) - -const ( - // Deprecated: kubernetes-sigs/scheduler-plugins/lightweight-coscheduling - LabelLightweightCoschedulingPodGroupName = "pod-group.scheduling.sigs.k8s.io/name" - // Deprecated: kubernetes-sigs/scheduler-plugins/lightweight-coscheduling - LabelLightweightCoschedulingPodGroupMinAvailable = "pod-group.scheduling.sigs.k8s.io/min-available" -) - // CustomUsageThresholds supports user-defined node resource utilization thresholds. type CustomUsageThresholds struct { // UsageThresholds indicates the resource utilization threshold of the whole machine. @@ -88,7 +48,7 @@ type CustomAggregatedUsage struct { // UsageThresholds indicates the resource utilization threshold of the machine based on percentile statistics UsageThresholds map[corev1.ResourceName]int64 `json:"usageThresholds,omitempty"` // UsageAggregationType indicates the percentile type of the machine's utilization when filtering - UsageAggregationType slov1alpha1.AggregationType `json:"usageAggregationType,omitempty"` + UsageAggregationType AggregationType `json:"usageAggregationType,omitempty"` // UsageAggregatedDuration indicates the statistical period of the percentile of the machine's utilization when filtering UsageAggregatedDuration *metav1.Duration `json:"usageAggregatedDuration,omitempty"` } @@ -137,7 +97,7 @@ type DeviceAllocation struct { Extension json.RawMessage `json:"extension,omitempty"` } -var GetDeviceAllocations = func(podAnnotations map[string]string) (DeviceAllocations, error) { +func GetDeviceAllocations(podAnnotations map[string]string) (DeviceAllocations, error) { deviceAllocations := DeviceAllocations{} data, ok := podAnnotations[AnnotationDeviceAllocated] if !ok { @@ -165,15 +125,3 @@ func SetDeviceAllocations(obj metav1.Object, allocations DeviceAllocations) erro obj.SetAnnotations(annotations) return nil } - -var GetMinNum = func(pod *corev1.Pod) (int, error) { - minRequiredNum, err := strconv.ParseInt(pod.Annotations[AnnotationGangMinNum], 10, 32) - if err != nil { - return 0, err - } - return int(minRequiredNum), nil -} - -var GetGangName = func(pod *corev1.Pod) string { - return pod.Annotations[AnnotationGangName] -} diff --git a/extension/system_qos.go b/extension/system_qos.go new file mode 100644 index 0000000..a477f6c --- /dev/null +++ b/extension/system_qos.go @@ -0,0 +1,53 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package extension + +import ( + "encoding/json" +) + +const ( + AnnotationNodeSystemQOSResource = NodeDomainPrefix + "/system-qos-resource" +) + +type SystemQOSResource struct { + // CPU cores used for System QoS Pods, format should follow Linux CPU list + // See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS + CPUSet string `json:"cpuset,omitempty"` + // whether CPU cores for System QoS are exclusive(default = true), which means could not be used by other pods(LS/LSR/BE) + CPUSetExclusive *bool `json:"cpusetExclusive,omitempty"` +} + +func (r *SystemQOSResource) IsCPUSetExclusive() bool { + // CPUSetExclusive default is true + return r.CPUSetExclusive == nil || *r.CPUSetExclusive +} + +func GetSystemQOSResource(anno map[string]string) (*SystemQOSResource, error) { + if anno == nil { + return nil, nil + } + systemQOSRes := &SystemQOSResource{} + data, ok := anno[AnnotationNodeSystemQOSResource] + if !ok { + return systemQOSRes, nil + } + if err := json.Unmarshal([]byte(data), systemQOSRes); err != nil { + return nil, err + } + return systemQOSRes, nil +} diff --git a/go.mod b/go.mod index 120239e..02beda2 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,8 @@ require ( k8s.io/client-go v0.22.6 k8s.io/code-generator v0.22.6 k8s.io/klog/v2 v2.80.1 + k8s.io/kubernetes v1.22.6 + k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a sigs.k8s.io/scheduler-plugins v0.22.6 ) @@ -64,7 +66,6 @@ require ( k8s.io/component-base v0.22.6 // indirect k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect - k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 317c02c..05e6c98 100644 --- a/go.sum +++ b/go.sum @@ -93,7 +93,6 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -153,6 +152,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= @@ -161,7 +161,6 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -187,7 +186,6 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= -github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= @@ -218,7 +216,6 @@ github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5F github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -262,6 +259,7 @@ github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA// github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cadvisor v0.39.3/go.mod h1:kN93gpdevu+bpS227TyHVZyCU5bbqCzTj5T9drl34MI= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -285,7 +283,6 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -336,7 +333,6 @@ github.com/heketi/heketi v10.3.0+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8= @@ -368,7 +364,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/koordinator-sh/cadvisor v0.0.0-20220919031936-833eb74e858e/go.mod h1:Z/GP9SfqAaUTTzTuj4HBkMAwdWAoFceS6btaHI51Y0M= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -399,6 +394,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -437,7 +433,6 @@ github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -445,18 +440,13 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -608,7 +598,6 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -657,7 +646,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -740,12 +728,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -816,7 +800,6 @@ golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -825,12 +808,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -899,7 +877,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1000,7 +977,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1086,6 +1062,7 @@ k8s.io/kube-proxy v0.22.6/go.mod h1:xLxEZ3sHyz11XaRyxqI4Z4F3I/Wtt+Jlep8w5yxQPAY= k8s.io/kube-scheduler v0.22.6/go.mod h1:DcHj6ixvb0M1PvWFbg133a1pz/vv7OSCgZUDU/UUhlU= k8s.io/kubectl v0.22.6/go.mod h1:9ktAgMwUsd2w12Yhj/xhMZhNna1t9rfExJg9j9jCIYk= k8s.io/kubelet v0.22.6/go.mod h1:/nSfVw7oYzpmLn8Ua2q2Zix09Fq5gpDGnNqTbab9wts= +k8s.io/kubernetes v1.22.6 h1:OPKNO4FElcN6wHc3N3P6uW3P1oHvzNxu+HJ8vGQtBzM= k8s.io/kubernetes v1.22.6/go.mod h1:l2ikQCpfvsMAXgL7FDtzgn/AVdjt4XGUYHMXn2vuzYI= k8s.io/legacy-cloud-providers v0.22.6/go.mod h1:ZhqLzcCCT4mGgxzVXJNg9/zQDBE8S2isa053fzXSR1s= k8s.io/metrics v0.22.6/go.mod h1:4a2o3y5qe2CleeMWZ80VCKzz+N8vPboAw/pz3xKQicI= @@ -1094,7 +1071,6 @@ k8s.io/pod-security-admission v0.22.6/go.mod h1:aBlsoKgqpuixDMaCu7U4d8IywNzcmGAK k8s.io/sample-apiserver v0.22.6/go.mod h1:wiamO3alvd/dicaP8PvTPCdqq31QKDbRxm1uqOGalsI= k8s.io/system-validators v1.5.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= diff --git a/hack/clone-api-files.sh b/hack/clone-api-files.sh index 0e2181c..a34b4cd 100755 --- a/hack/clone-api-files.sh +++ b/hack/clone-api-files.sh @@ -5,8 +5,10 @@ set -e # Define the relative path of API src and dst API_PATHS_MAP=( "apis/config:config" + "apis/configuration:configuration" "apis/runtime:runtime" "apis/slo:slo" + "apis/scheme:scheme" "apis/extension:extension" "apis/scheduling:scheduling" ) diff --git a/scheduling/v1alpha1/device_types.go b/scheduling/v1alpha1/device_types.go index 496f25f..fd55f4a 100644 --- a/scheduling/v1alpha1/device_types.go +++ b/scheduling/v1alpha1/device_types.go @@ -34,16 +34,41 @@ type DeviceSpec struct { } type DeviceInfo struct { + // Type represents the type of device + Type DeviceType `json:"type,omitempty"` + // Labels represents the device properties that can be used to organize and categorize (scope and select) objects + Labels map[string]string `json:"labels,omitempty"` // UUID represents the UUID of device UUID string `json:"id,omitempty"` // Minor represents the Minor number of Device, starting from 0 Minor *int32 `json:"minor,omitempty"` - // Type represents the type of device - Type DeviceType `json:"type,omitempty"` + // ModuleID represents the physical id of Device + ModuleID *int32 `json:"moduleID,omitempty"` // Health indicates whether the device is normal Health bool `json:"health,omitempty"` // Resources is a set of (resource name, quantity) pairs Resources corev1.ResourceList `json:"resources,omitempty"` + // Topology represents the topology information about the device + Topology *DeviceTopology `json:"topology,omitempty"` + // VFGroups represents the virtual function devices + VFGroups []VirtualFunctionGroup `json:"vfGroups,omitempty"` +} + +type DeviceTopology struct { + SocketID int32 `json:"socketID"` + NodeID int32 `json:"nodeID"` + PCIEID int32 `json:"pcieID"` + BusID string `json:"busID,omitempty"` +} + +type VirtualFunctionGroup struct { + Labels map[string]string `json:"labels,omitempty"` + VFs []VirtualFunction `json:"vfs,omitempty"` +} + +type VirtualFunction struct { + Minor int32 `json:"minor"` + BusID string `json:"busID,omitempty"` } type DeviceStatus struct { diff --git a/scheduling/v1alpha1/reservation_types.go b/scheduling/v1alpha1/reservation_types.go index 9bd1b47..7bff4f2 100644 --- a/scheduling/v1alpha1/reservation_types.go +++ b/scheduling/v1alpha1/reservation_types.go @@ -61,9 +61,35 @@ type ReservationSpec struct { // and are not allocatable to other owners anymore. Defaults to true. // +kubebuilder:default=true // +optional - AllocateOnce bool `json:"allocateOnce,omitempty"` + AllocateOnce *bool `json:"allocateOnce,omitempty"` + // AllocatePolicy represents the allocation policy of reserved resources that Reservation expects. + // +kubebuilder:validation:Enum=Aligned;Restricted + // +optional + AllocatePolicy ReservationAllocatePolicy `json:"allocatePolicy,omitempty"` + // Unschedulable controls reservation schedulability of new pods. By default, reservation is schedulable. + // +optional + Unschedulable bool `json:"unschedulable,omitempty"` } +type ReservationAllocatePolicy string + +const ( + // ReservationAllocatePolicyDefault means that there is no restriction on the policy of reserved resources, + // and allocated from the Reservation first, and if it is insufficient, it is allocated from the node. + ReservationAllocatePolicyDefault ReservationAllocatePolicy = "" + // ReservationAllocatePolicyAligned indicates that the Pod allocates resources from the Reservation first. + // If the remaining resources of the Reservation are insufficient, it can be allocated from the node, + // but it is required to strictly follow the resource specifications of the Pod. + // This can be used to avoid the problem that a Pod uses multiple Reservations at the same time. + ReservationAllocatePolicyAligned ReservationAllocatePolicy = "Aligned" + // ReservationAllocatePolicyRestricted means that the resources + // requested by the Pod overlap with the resources reserved by the Reservation, + // then these intersection resources can only be allocated from the Reservation, + // but resources declared in Pods but not reserved in Reservations can be allocated from Nodes. + // ReservationAllocatePolicyRestricted includes the semantics of ReservationAllocatePolicyAligned. + ReservationAllocatePolicyRestricted ReservationAllocatePolicy = "Restricted" +) + // ReservationTemplateSpec describes the data a Reservation should have when created from a template type ReservationTemplateSpec struct { // Standard object's metadata. diff --git a/scheduling/v1alpha1/zz_generated.deepcopy.go b/scheduling/v1alpha1/zz_generated.deepcopy.go index c1a2781..877e866 100644 --- a/scheduling/v1alpha1/zz_generated.deepcopy.go +++ b/scheduling/v1alpha1/zz_generated.deepcopy.go @@ -99,11 +99,23 @@ func (in *DeviceAllocationItem) DeepCopy() *DeviceAllocationItem { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeviceInfo) DeepCopyInto(out *DeviceInfo) { *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Minor != nil { in, out := &in.Minor, &out.Minor *out = new(int32) **out = **in } + if in.ModuleID != nil { + in, out := &in.ModuleID, &out.ModuleID + *out = new(int32) + **out = **in + } if in.Resources != nil { in, out := &in.Resources, &out.Resources *out = make(v1.ResourceList, len(*in)) @@ -111,6 +123,18 @@ func (in *DeviceInfo) DeepCopyInto(out *DeviceInfo) { (*out)[key] = val.DeepCopy() } } + if in.Topology != nil { + in, out := &in.Topology, &out.Topology + *out = new(DeviceTopology) + **out = **in + } + if in.VFGroups != nil { + in, out := &in.VFGroups, &out.VFGroups + *out = make([]VirtualFunctionGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceInfo. @@ -199,6 +223,21 @@ func (in *DeviceStatus) DeepCopy() *DeviceStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceTopology) DeepCopyInto(out *DeviceTopology) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceTopology. +func (in *DeviceTopology) DeepCopy() *DeviceTopology { + if in == nil { + return nil + } + out := new(DeviceTopology) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodMigrateReservationOptions) DeepCopyInto(out *PodMigrateReservationOptions) { *out = *in @@ -565,6 +604,11 @@ func (in *ReservationSpec) DeepCopyInto(out *ReservationSpec) { in, out := &in.Expires, &out.Expires *out = (*in).DeepCopy() } + if in.AllocateOnce != nil { + in, out := &in.AllocateOnce, &out.AllocateOnce + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReservationSpec. @@ -634,3 +678,45 @@ func (in *ReservationTemplateSpec) DeepCopy() *ReservationTemplateSpec { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualFunction) DeepCopyInto(out *VirtualFunction) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualFunction. +func (in *VirtualFunction) DeepCopy() *VirtualFunction { + if in == nil { + return nil + } + out := new(VirtualFunction) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualFunctionGroup) DeepCopyInto(out *VirtualFunctionGroup) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.VFs != nil { + in, out := &in.VFs, &out.VFs + *out = make([]VirtualFunction, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualFunctionGroup. +func (in *VirtualFunctionGroup) DeepCopy() *VirtualFunctionGroup { + if in == nil { + return nil + } + out := new(VirtualFunctionGroup) + in.DeepCopyInto(out) + return out +} diff --git a/slo/v1alpha1/nodemetric_types.go b/slo/v1alpha1/nodemetric_types.go index c6071d5..bd58312 100644 --- a/slo/v1alpha1/nodemetric_types.go +++ b/slo/v1alpha1/nodemetric_types.go @@ -17,32 +17,28 @@ limitations under the License. package v1alpha1 import ( + apiext "github.com/koordinator-sh/apis/extension" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -type AggregationType string - -const ( - // max is not welcomed since it may import outliers - AVG AggregationType = "avg" - P99 AggregationType = "p99" - P95 AggregationType = "p95" - P90 AggregationType = "p90" - P50 AggregationType = "p50" -) - type NodeMetricInfo struct { + // NodeUsage is the total resource usage of node NodeUsage ResourceMap `json:"nodeUsage,omitempty"` // AggregatedNodeUsages will report only if there are enough samples AggregatedNodeUsages []AggregatedUsage `json:"aggregatedNodeUsages,omitempty"` + // SystemUsage is the resource usage of daemon processes and OS kernel, calculated by `NodeUsage - sum(podUsage)` + SystemUsage ResourceMap `json:"systemUsage,omitempty"` + // AggregatedSystemUsages will report only if there are enough samples + // Deleted pods will be excluded during aggregation + AggregatedSystemUsages []AggregatedUsage `json:"aggregatedSystemUsages,omitempty"` } type AggregatedUsage struct { - Usage map[AggregationType]ResourceMap `json:"usage,omitempty"` - Duration metav1.Duration `json:"duration,omitempty"` + Usage map[apiext.AggregationType]ResourceMap `json:"usage,omitempty"` + Duration metav1.Duration `json:"duration,omitempty"` } type PodMetricInfo struct { @@ -73,6 +69,12 @@ type AggregatePolicy struct { Durations []metav1.Duration `json:"durations,omitempty"` } +// ReclaimableMetric defines the reclaimable metric of resource priority +type ReclaimableMetric struct { + // Resource is the resource usage of the prediction + Resource ResourceMap `json:"resource,omitempty"` +} + // NodeMetricStatus defines the observed state of NodeMetric type NodeMetricStatus struct { // UpdateTime is the last time this NodeMetric was updated. @@ -83,6 +85,9 @@ type NodeMetricStatus struct { // PodsMetric contains the metrics for pods belong to this node. PodsMetric []*PodMetricInfo `json:"podsMetric,omitempty"` + + // ProdReclaimableMetric is the indicator statistics of Prod type resources reclaimable + ProdReclaimableMetric *ReclaimableMetric `json:"prodReclaimableMetric,omitempty"` } // +genclient diff --git a/slo/v1alpha1/nodeslo_types.go b/slo/v1alpha1/nodeslo_types.go index 287f0f8..beaa8f0 100644 --- a/slo/v1alpha1/nodeslo_types.go +++ b/slo/v1alpha1/nodeslo_types.go @@ -26,7 +26,7 @@ import ( // CPUQOS enables cpu qos features. type CPUQOS struct { // group identity value for pods, default = 0 - GroupIdentity *int64 `json:"groupIdentity,omitempty"` + GroupIdentity *int64 `json:"groupIdentity,omitempty" validate:"omitempty,min=-1,max=2"` } // MemoryQOS enables memory qos features. @@ -36,12 +36,12 @@ type MemoryQOS struct { // 1. `memory.min` := spec.requests.memory * minLimitFactor / 100 (use 0 if requests.memory is not set) // 2. `memory.low` := spec.requests.memory * lowLimitFactor / 100 (use 0 if requests.memory is not set) // 3. `memory.limit_in_bytes` := spec.limits.memory (set $node.allocatable.memory if limits.memory is not set) - // 4. `memory.high` := memory.limit_in_bytes * throttlingFactor / 100 (use "max" if memory.high <= memory.min) + // 4. `memory.high` := floor[(spec.requests.memory + throttlingFactor / 100 * (memory.limit_in_bytes or node allocatable memory - spec.requests.memory))/pageSize] * pageSize // MinLimitPercent specifies the minLimitFactor percentage to calculate `memory.min`, which protects memory // from global reclamation when memory usage does not exceed the min limit. // Close: 0. // +kubebuilder:validation:Minimum=0 - MinLimitPercent *int64 `json:"minLimitPercent,omitempty"` + MinLimitPercent *int64 `json:"minLimitPercent,omitempty" validate:"omitempty,min=0,max=100"` // LowLimitPercent specifies the lowLimitFactor percentage to calculate `memory.low`, which TRIES BEST // protecting memory from global reclamation when memory usage does not exceed the low limit unless no unprotected // memcg can be reclaimed. @@ -49,13 +49,13 @@ type MemoryQOS struct { // pod `memory.low` and `memory.high` become invalid, while `memory.wmark_ratio` is still in effect. // Close: 0. // +kubebuilder:validation:Minimum=0 - LowLimitPercent *int64 `json:"lowLimitPercent,omitempty"` + LowLimitPercent *int64 `json:"lowLimitPercent,omitempty" validate:"omitempty,min=0,max=100"` // ThrottlingPercent specifies the throttlingFactor percentage to calculate `memory.high` with pod // memory.limits or node allocatable memory, which triggers memcg direct reclamation when memory usage exceeds. // Lower the factor brings more heavier reclaim pressure. // Close: 0. // +kubebuilder:validation:Minimum=0 - ThrottlingPercent *int64 `json:"throttlingPercent,omitempty"` + ThrottlingPercent *int64 `json:"throttlingPercent,omitempty" validate:"omitempty,min=0,max=100"` // wmark_ratio (Anolis OS required) // Async memory reclamation is triggered when cgroup memory usage exceeds `memory.wmark_high` and the reclamation @@ -67,13 +67,13 @@ type MemoryQOS struct { // Close: 0. Recommended: 95. // +kubebuilder:validation:Maximum=100 // +kubebuilder:validation:Minimum=0 - WmarkRatio *int64 `json:"wmarkRatio,omitempty"` + WmarkRatio *int64 `json:"wmarkRatio,omitempty" validate:"omitempty,min=0,max=100"` // WmarkScalePermill specifies `memory.wmark_scale_factor` that helps calculate `memory.wmark_low`, which // stops async memory reclamation when memory usage belows. // Close: 50. Recommended: 20. // +kubebuilder:validation:Maximum=1000 // +kubebuilder:validation:Minimum=1 - WmarkScalePermill *int64 `json:"wmarkScalePermill,omitempty"` + WmarkScalePermill *int64 `json:"wmarkScalePermill,omitempty" validate:"omitempty,min=1,max=1000"` // wmark_min_adj (Anolis OS required) // WmarkMinAdj specifies `memory.wmark_min_adj` which adjusts per-memcg threshold for global memory @@ -84,12 +84,12 @@ type MemoryQOS struct { // Close: [LSR:0, LS:0, BE:0]. Recommended: [LSR:-25, LS:-25, BE:50]. // +kubebuilder:validation:Maximum=50 // +kubebuilder:validation:Minimum=-25 - WmarkMinAdj *int64 `json:"wmarkMinAdj,omitempty"` + WmarkMinAdj *int64 `json:"wmarkMinAdj,omitempty" validate:"omitempty,min=-25,max=50"` // TODO: enhance the usages of oom priority and oom kill group - PriorityEnable *int64 `json:"priorityEnable,omitempty"` - Priority *int64 `json:"priority,omitempty"` - OomKillGroup *int64 `json:"oomKillGroup,omitempty"` + PriorityEnable *int64 `json:"priorityEnable,omitempty" validate:"omitempty,min=0,max=1"` + Priority *int64 `json:"priority,omitempty" validate:"omitempty,min=0,max=12"` + OomKillGroup *int64 `json:"oomKillGroup,omitempty" validate:"omitempty,min=0,max=1"` } type PodMemoryQOSPolicy string @@ -125,9 +125,63 @@ type MemoryQOSCfg struct { MemoryQOS `json:",inline"` } +type BlockType string + +const ( + // Device, such as /dev/sdb + // Only used for RootClass blk-iocost configuration + BlockTypeDevice BlockType = "device" + // LVM volume group + BlockTypeVolumeGroup BlockType = "volumegroup" + // Pod volume + BlockTypePodVolume BlockType = "podvolume" +) + +type IOCfg struct { + // Throttling of IOPS + // The value is set to 0, which indicates that the feature is disabled. + // +kubebuilder:validation:Minimum=0 + ReadIOPS *int64 `json:"readIOPS,omitempty"` + // +kubebuilder:validation:Minimum=0 + WriteIOPS *int64 `json:"writeIOPS,omitempty"` + // Throttling of throughput + // The value is set to 0, which indicates that the feature is disabled. + // +kubebuilder:validation:Minimum=0 + ReadBPS *int64 `json:"readBPS,omitempty"` + // +kubebuilder:validation:Minimum=0 + WriteBPS *int64 `json:"writeBPS,omitempty"` + // This field is used to set the weight of a sub-group. Default value: 100. Valid values: 1 to 100. + // +kubebuilder:validation:Maximum=100 + // +kubebuilder:validation:Minimum=1 + IOWeightPercent *int64 `json:"ioWeightPercent,omitempty"` + // Configure the weight-based throttling feature of blk-iocost + // Only used for RootClass + // After blk-iocost is enabled, the kernel calculates the proportion of requests that exceed the read or write latency threshold out of all requests. When the proportion is greater than the read or write latency percentile (95%), the kernel considers the disk to be saturated and reduces the rate at which requests are sent to the disk. + // the read latency threshold. Unit: microseconds. + ReadLatency *int64 `json:"readLatency,omitempty"` + // the write latency threshold. Unit: microseconds. + WriteLatency *int64 `json:"writeLatency,omitempty"` +} + +type BlockCfg struct { + Name string `json:"name,omitempty"` + BlockType BlockType `json:"type,omitempty"` + IOCfg IOCfg `json:"ioCfg,omitempty"` +} + +type BlkIOQOS struct { + Blocks []*BlockCfg `json:"blocks,omitempty"` +} + +type BlkIOQOSCfg struct { + Enable *bool `json:"enable,omitempty"` + BlkIOQOS `json:",inline"` +} + type ResourceQOS struct { CPUQOS *CPUQOSCfg `json:"cpuQOS,omitempty"` MemoryQOS *MemoryQOSCfg `json:"memoryQOS,omitempty"` + BlkIOQOS *BlkIOQOSCfg `json:"blkioQOS,omitempty"` ResctrlQOS *ResctrlQOSCfg `json:"resctrlQOS,omitempty"` } @@ -162,31 +216,31 @@ type ResourceThresholdStrategy struct { // cpu suppress threshold percentage (0,100), default = 65 // +kubebuilder:validation:Maximum=100 // +kubebuilder:validation:Minimum=0 - CPUSuppressThresholdPercent *int64 `json:"cpuSuppressThresholdPercent,omitempty"` + CPUSuppressThresholdPercent *int64 `json:"cpuSuppressThresholdPercent,omitempty" validate:"omitempty,min=0,max=100"` // CPUSuppressPolicy CPUSuppressPolicy CPUSuppressPolicy `json:"cpuSuppressPolicy,omitempty"` // upper: memory evict threshold percentage (0,100), default = 70 // +kubebuilder:validation:Maximum=100 // +kubebuilder:validation:Minimum=0 - MemoryEvictThresholdPercent *int64 `json:"memoryEvictThresholdPercent,omitempty"` + MemoryEvictThresholdPercent *int64 `json:"memoryEvictThresholdPercent,omitempty" validate:"omitempty,min=0,max=100,gtfield=MemoryEvictLowerPercent"` // lower: memory release util usage under MemoryEvictLowerPercent, default = MemoryEvictThresholdPercent - 2 // +kubebuilder:validation:Maximum=100 // +kubebuilder:validation:Minimum=0 - MemoryEvictLowerPercent *int64 `json:"memoryEvictLowerPercent,omitempty"` + MemoryEvictLowerPercent *int64 `json:"memoryEvictLowerPercent,omitempty" validate:"omitempty,min=0,max=100,ltfield=MemoryEvictThresholdPercent"` // be.satisfactionRate = be.CPURealLimit/be.CPURequest // if be.satisfactionRate > CPUEvictBESatisfactionUpperPercent/100, then stop to evict. - CPUEvictBESatisfactionUpperPercent *int64 `json:"cpuEvictBESatisfactionUpperPercent,omitempty"` + CPUEvictBESatisfactionUpperPercent *int64 `json:"cpuEvictBESatisfactionUpperPercent,omitempty" validate:"omitempty,min=0,max=100,gtfield=CPUEvictBESatisfactionLowerPercent"` // be.satisfactionRate = be.CPURealLimit/be.CPURequest; be.cpuUsage = be.CPUUsed/be.CPURealLimit // if be.satisfactionRate < CPUEvictBESatisfactionLowerPercent/100 && be.usage >= CPUEvictBEUsageThresholdPercent/100, // then start to evict pod, and will evict to ${CPUEvictBESatisfactionUpperPercent} - CPUEvictBESatisfactionLowerPercent *int64 `json:"cpuEvictBESatisfactionLowerPercent,omitempty"` + CPUEvictBESatisfactionLowerPercent *int64 `json:"cpuEvictBESatisfactionLowerPercent,omitempty" validate:"omitempty,min=0,max=100,ltfield=CPUEvictBESatisfactionUpperPercent"` // if be.cpuUsage >= CPUEvictBEUsageThresholdPercent/100, then start to calculate the resources need to be released. - CPUEvictBEUsageThresholdPercent *int64 `json:"cpuEvictBEUsageThresholdPercent,omitempty"` + CPUEvictBEUsageThresholdPercent *int64 `json:"cpuEvictBEUsageThresholdPercent,omitempty" validate:"omitempty,min=0,max=100"` // when avg(cpuusage) > CPUEvictThresholdPercent, will start to evict pod by cpu, // and avg(cpuusage) is calculated based on the most recent CPUEvictTimeWindowSeconds data - CPUEvictTimeWindowSeconds *int64 `json:"cpuEvictTimeWindowSeconds,omitempty"` + CPUEvictTimeWindowSeconds *int64 `json:"cpuEvictTimeWindowSeconds,omitempty" validate:"omitempty,gt=0"` } // ResctrlQOSCfg stores node-level config of resctrl qos @@ -200,15 +254,15 @@ type ResctrlQOS struct { // LLC available range start for pods by percentage // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=100 - CATRangeStartPercent *int64 `json:"catRangeStartPercent,omitempty"` + CATRangeStartPercent *int64 `json:"catRangeStartPercent,omitempty" validate:"omitempty,min=0,max=100,ltfield=CATRangeEndPercent"` // LLC available range end for pods by percentage // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=100 - CATRangeEndPercent *int64 `json:"catRangeEndPercent,omitempty"` + CATRangeEndPercent *int64 `json:"catRangeEndPercent,omitempty" validate:"omitempty,min=0,max=100,gtfield=CATRangeStartPercent"` // MBA percent // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=100 - MBAPercent *int64 `json:"mbaPercent,omitempty"` + MBAPercent *int64 `json:"mbaPercent,omitempty" validate:"omitempty,min=0,max=100"` } type CPUBurstPolicy string @@ -229,26 +283,26 @@ type CPUBurstConfig struct { // cpu burst percentage for setting cpu.cfs_burst_us, legal range: [0, 10000], default as 1000 (1000%) // +kubebuilder:validation:Maximum=10000 // +kubebuilder:validation:Minimum=0 - CPUBurstPercent *int64 `json:"cpuBurstPercent,omitempty"` + CPUBurstPercent *int64 `json:"cpuBurstPercent,omitempty" validate:"omitempty,min=1,max=10000"` // pod cfs quota scale up ceil percentage, default = 300 (300%) - CFSQuotaBurstPercent *int64 `json:"cfsQuotaBurstPercent,omitempty"` + CFSQuotaBurstPercent *int64 `json:"cfsQuotaBurstPercent,omitempty" validate:"omitempty,min=100"` // specifies a period of time for pod can use at burst, default = -1 (unlimited) - CFSQuotaBurstPeriodSeconds *int64 `json:"cfsQuotaBurstPeriodSeconds,omitempty"` + CFSQuotaBurstPeriodSeconds *int64 `json:"cfsQuotaBurstPeriodSeconds,omitempty" validate:"omitempty,min=-1"` } type CPUBurstStrategy struct { CPUBurstConfig `json:",inline"` // scale down cfs quota if node cpu overload, default = 50 - SharePoolThresholdPercent *int64 `json:"sharePoolThresholdPercent,omitempty"` + SharePoolThresholdPercent *int64 `json:"sharePoolThresholdPercent,omitempty" validate:"omitempty,min=0,max=100"` } type SystemStrategy struct { // for /proc/sys/vm/min_free_kbytes, min_free_kbytes = minFreeKbytesFactor * nodeTotalMemory /10000 - MinFreeKbytesFactor *int64 `json:"minFreeKbytesFactor,omitempty"` + MinFreeKbytesFactor *int64 `json:"minFreeKbytesFactor,omitempty" validate:"omitempty,gt=0"` // /proc/sys/vm/watermark_scale_factor - WatermarkScaleFactor *int64 `json:"watermarkScaleFactor,omitempty"` + WatermarkScaleFactor *int64 `json:"watermarkScaleFactor,omitempty" validate:"omitempty,gt=0,max=400"` // /sys/kernel/mm/memcg_reaper/reap_background - MemcgReapBackGround *int64 `json:"memcgReapBackGround,omitempty"` + MemcgReapBackGround *int64 `json:"memcgReapBackGround,omitempty" validate:"omitempty,min=0,max=1"` } // NodeSLOSpec defines the desired state of NodeSLO diff --git a/extension/pod.go b/slo/v1alpha1/pod.go similarity index 72% rename from extension/pod.go rename to slo/v1alpha1/pod.go index a100614..8efa11a 100644 --- a/extension/pod.go +++ b/slo/v1alpha1/pod.go @@ -14,23 +14,25 @@ See the License for the specific language governing permissions and limitations under the License. */ -package extension +package v1alpha1 import ( "encoding/json" corev1 "k8s.io/api/core/v1" - slov1alpha1 "github.com/koordinator-sh/apis/slo/v1alpha1" + apiext "github.com/koordinator-sh/apis/extension" ) const ( - AnnotationPodCPUBurst = DomainPrefix + "cpuBurst" + AnnotationPodCPUBurst = apiext.DomainPrefix + "cpuBurst" - AnnotationPodMemoryQoS = DomainPrefix + "memoryQOS" + AnnotationPodMemoryQoS = apiext.DomainPrefix + "memoryQOS" + + AnnotationPodBlkioQoS = apiext.DomainPrefix + "blkioQOS" ) -func GetPodCPUBurstConfig(pod *corev1.Pod) (*slov1alpha1.CPUBurstConfig, error) { +func GetPodCPUBurstConfig(pod *corev1.Pod) (*CPUBurstConfig, error) { if pod == nil || pod.Annotations == nil { return nil, nil } @@ -38,7 +40,7 @@ func GetPodCPUBurstConfig(pod *corev1.Pod) (*slov1alpha1.CPUBurstConfig, error) if !exist { return nil, nil } - cpuBurst := slov1alpha1.CPUBurstConfig{} + cpuBurst := CPUBurstConfig{} err := json.Unmarshal([]byte(annotation), &cpuBurst) if err != nil { @@ -47,7 +49,7 @@ func GetPodCPUBurstConfig(pod *corev1.Pod) (*slov1alpha1.CPUBurstConfig, error) return &cpuBurst, nil } -func GetPodMemoryQoSConfig(pod *corev1.Pod) (*slov1alpha1.PodMemoryQOSConfig, error) { +func GetPodMemoryQoSConfig(pod *corev1.Pod) (*PodMemoryQOSConfig, error) { if pod == nil || pod.Annotations == nil { return nil, nil } @@ -55,7 +57,7 @@ func GetPodMemoryQoSConfig(pod *corev1.Pod) (*slov1alpha1.PodMemoryQOSConfig, er if !exist { return nil, nil } - cfg := slov1alpha1.PodMemoryQOSConfig{} + cfg := PodMemoryQOSConfig{} err := json.Unmarshal([]byte(value), &cfg) if err != nil { return nil, err diff --git a/slo/v1alpha1/zz_generated.deepcopy.go b/slo/v1alpha1/zz_generated.deepcopy.go index 5eb9660..cee9996 100644 --- a/slo/v1alpha1/zz_generated.deepcopy.go +++ b/slo/v1alpha1/zz_generated.deepcopy.go @@ -22,6 +22,7 @@ limitations under the License. package v1alpha1 import ( + "github.com/koordinator-sh/apis/extension" schedulingv1alpha1 "github.com/koordinator-sh/apis/scheduling/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -53,7 +54,7 @@ func (in *AggregatedUsage) DeepCopyInto(out *AggregatedUsage) { *out = *in if in.Usage != nil { in, out := &in.Usage, &out.Usage - *out = make(map[AggregationType]ResourceMap, len(*in)) + *out = make(map[extension.AggregationType]ResourceMap, len(*in)) for key, val := range *in { (*out)[key] = *val.DeepCopy() } @@ -71,6 +72,69 @@ func (in *AggregatedUsage) DeepCopy() *AggregatedUsage { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlkIOQOS) DeepCopyInto(out *BlkIOQOS) { + *out = *in + if in.Blocks != nil { + in, out := &in.Blocks, &out.Blocks + *out = make([]*BlockCfg, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(BlockCfg) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlkIOQOS. +func (in *BlkIOQOS) DeepCopy() *BlkIOQOS { + if in == nil { + return nil + } + out := new(BlkIOQOS) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlkIOQOSCfg) DeepCopyInto(out *BlkIOQOSCfg) { + *out = *in + if in.Enable != nil { + in, out := &in.Enable, &out.Enable + *out = new(bool) + **out = **in + } + in.BlkIOQOS.DeepCopyInto(&out.BlkIOQOS) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlkIOQOSCfg. +func (in *BlkIOQOSCfg) DeepCopy() *BlkIOQOSCfg { + if in == nil { + return nil + } + out := new(BlkIOQOSCfg) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockCfg) DeepCopyInto(out *BlockCfg) { + *out = *in + in.IOCfg.DeepCopyInto(&out.IOCfg) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockCfg. +func (in *BlockCfg) DeepCopy() *BlockCfg { + if in == nil { + return nil + } + out := new(BlockCfg) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CPUBurstConfig) DeepCopyInto(out *CPUBurstConfig) { *out = *in @@ -163,6 +227,56 @@ func (in *CPUQOSCfg) DeepCopy() *CPUQOSCfg { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IOCfg) DeepCopyInto(out *IOCfg) { + *out = *in + if in.ReadIOPS != nil { + in, out := &in.ReadIOPS, &out.ReadIOPS + *out = new(int64) + **out = **in + } + if in.WriteIOPS != nil { + in, out := &in.WriteIOPS, &out.WriteIOPS + *out = new(int64) + **out = **in + } + if in.ReadBPS != nil { + in, out := &in.ReadBPS, &out.ReadBPS + *out = new(int64) + **out = **in + } + if in.WriteBPS != nil { + in, out := &in.WriteBPS, &out.WriteBPS + *out = new(int64) + **out = **in + } + if in.IOWeightPercent != nil { + in, out := &in.IOWeightPercent, &out.IOWeightPercent + *out = new(int64) + **out = **in + } + if in.ReadLatency != nil { + in, out := &in.ReadLatency, &out.ReadLatency + *out = new(int64) + **out = **in + } + if in.WriteLatency != nil { + in, out := &in.WriteLatency, &out.WriteLatency + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IOCfg. +func (in *IOCfg) DeepCopy() *IOCfg { + if in == nil { + return nil + } + out := new(IOCfg) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemoryQOS) DeepCopyInto(out *MemoryQOS) { *out = *in @@ -312,6 +426,14 @@ func (in *NodeMetricInfo) DeepCopyInto(out *NodeMetricInfo) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.SystemUsage.DeepCopyInto(&out.SystemUsage) + if in.AggregatedSystemUsages != nil { + in, out := &in.AggregatedSystemUsages, &out.AggregatedSystemUsages + *out = make([]AggregatedUsage, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeMetricInfo. @@ -399,6 +521,11 @@ func (in *NodeMetricStatus) DeepCopyInto(out *NodeMetricStatus) { } } } + if in.ProdReclaimableMetric != nil { + in, out := &in.ProdReclaimableMetric, &out.ProdReclaimableMetric + *out = new(ReclaimableMetric) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeMetricStatus. @@ -560,6 +687,22 @@ func (in *PodMetricInfo) DeepCopy() *PodMetricInfo { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReclaimableMetric) DeepCopyInto(out *ReclaimableMetric) { + *out = *in + in.Resource.DeepCopyInto(&out.Resource) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReclaimableMetric. +func (in *ReclaimableMetric) DeepCopy() *ReclaimableMetric { + if in == nil { + return nil + } + out := new(ReclaimableMetric) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResctrlQOS) DeepCopyInto(out *ResctrlQOS) { *out = *in @@ -653,6 +796,11 @@ func (in *ResourceQOS) DeepCopyInto(out *ResourceQOS) { *out = new(MemoryQOSCfg) (*in).DeepCopyInto(*out) } + if in.BlkIOQOS != nil { + in, out := &in.BlkIOQOS, &out.BlkIOQOS + *out = new(BlkIOQOSCfg) + (*in).DeepCopyInto(*out) + } if in.ResctrlQOS != nil { in, out := &in.ResctrlQOS, &out.ResctrlQOS *out = new(ResctrlQOSCfg)