release v1.2.0

extension/constants.go

@@ -20,7 +20,7 @@ import corev1 "k8s.io/api/core/v1"
 
 const (
 	DomainPrefix = "koordinator.sh/"
-	// use prefix "kubernetes.io/" for extend resource
+	// ResourceDomainPrefix is a prefix "kubernetes.io/" used by particular extend resources (e.g. batch resources)
 	ResourceDomainPrefix = corev1.ResourceDefaultNamespacePrefix
 	// SchedulingDomainPrefix represents the scheduling domain prefix
 	SchedulingDomainPrefix = "scheduling.koordinator.sh"

extension/deprecated.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package extension
+
+import corev1 "k8s.io/api/core/v1"
+
+const (
+	// Deprecated: because of the limitation of extended resource naming
+	KoordBatchCPU corev1.ResourceName = DomainPrefix + "batch-cpu"
+	// Deprecated: because of the limitation of extended resource naming
+	KoordBatchMemory corev1.ResourceName = DomainPrefix + "batch-memory"
+
+	// Deprecated: Device extension resource names should use the prefix `koordinator.sh`
+	DeprecatedKoordRDMA corev1.ResourceName = ResourceDomainPrefix + "rdma"
+	// Deprecated: Device extension resource names should use the prefix `koordinator.sh`
+	DeprecatedKoordFPGA corev1.ResourceName = ResourceDomainPrefix + "fpga"
+	// Deprecated: Device extension resource names should use the prefix `koordinator.sh`
+	DeprecatedKoordGPU corev1.ResourceName = ResourceDomainPrefix + "gpu"
+	// Deprecated: Device extension resource names should use the prefix `koordinator.sh`
+	DeprecatedGPUCore corev1.ResourceName = ResourceDomainPrefix + "gpu-core"
+	// Deprecated: Device extension resource names should use the prefix `koordinator.sh`
+	DeprecatedGPUMemory corev1.ResourceName = ResourceDomainPrefix + "gpu-memory"
+	// Deprecated: Device extension resource names should use the prefix `koordinator.sh`
+	DeprecatedGPUMemoryRatio corev1.ResourceName = ResourceDomainPrefix + "gpu-memory-ratio"
+)
+
+const (
+	// Deprecated: Device extension resource names should use the prefix `koordinator.sh`
+	DeprecatedGPUDriver string = ResourceDomainPrefix + "gpu-driver"
+	// Deprecated: Device extension resource names should use the prefix `koordinator.sh`
+	DeprecatedGPUModel string = ResourceDomainPrefix + "gpu-model"
+)
+
+var deprecatedDeviceResourceNameMapper = map[corev1.ResourceName]corev1.ResourceName{
+	DeprecatedKoordRDMA:      ResourceRDMA,
+	DeprecatedKoordFPGA:      ResourceFPGA,
+	DeprecatedKoordGPU:       ResourceGPU,
+	DeprecatedGPUCore:        ResourceGPUCore,
+	DeprecatedGPUMemory:      ResourceGPUMemory,
+	DeprecatedGPUMemoryRatio: ResourceGPUMemoryRatio,
+}
+
+func TransformDeprecatedDeviceResources(resList corev1.ResourceList) corev1.ResourceList {
+	r := make(corev1.ResourceList, len(resList))
+	for k, v := range resList {
+		newResName := deprecatedDeviceResourceNameMapper[k]
+		if newResName != "" {
+			r[newResName] = v
+		} else {
+			r[k] = v
+		}
+	}
+	return r
+}

extension/descheduling.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package extension
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	// AnnotationEvictionCost indicates the eviction cost. It can be used to set to an int32.
+	// Although the K8s community has [Pod Deletion Cost #2255](https://github.com/kubernetes/enhancements/issues/2255),
+	// it is not a general mechanism. To avoid conflicts with components that use `Pod Deletion Cost`,
+	// users can individually mark the eviction cost for Pods.
+	// The implicit eviction cost for pods that don't set the annotation is 0, negative values are permitted.
+	// If set the cost with `math.MaxInt32`, it means the Pod will not be evicted.
+	// Pods with lower eviction cost are preferred to be evicted before pods with higher eviction cost.
+	// If a batch of Pods to be evicted have the same priority, they will be sorted by cost,
+	// and the Pod with the smallest cost will be evicted.
+	AnnotationEvictionCost = SchedulingDomainPrefix + "/eviction-cost"
+)
+
+const (
+	// AnnotationSoftEviction indicates custom eviction. It can be used to set to an "true".
+	AnnotationSoftEviction = SchedulingDomainPrefix + "/soft-eviction"
+)
+
+type SoftEvictionSpec struct {
+	// Timestamp indicates time when custom eviction occurs . It can be used to set a second timestamp.
+	Timestamp *metav1.Time `json:"timestamp,omitempty"`
+	// DeleteOptions indicates the options to delete the pod.
+	DeleteOptions *metav1.DeleteOptions `json:"deleteOptions,omitempty"`
+	// Initiator indicates the initiator of the eviction.
+	Initiator string `json:"initiator,omitempty"`
+	// Reason indicates reason for eviction.
+	Reason string `json:"reason,omitempty"`
+}
+
+func GetSoftEvictionSpec(annotations map[string]string) (*SoftEvictionSpec, error) {
+	evictionSpec := &SoftEvictionSpec{}
+	data, ok := annotations[AnnotationSoftEviction]
+	if !ok {
+		return evictionSpec, nil
+	}
+	err := json.Unmarshal([]byte(data), evictionSpec)
+	if err != nil {
+		return evictionSpec, err
+	}
+	return evictionSpec, err
+}
+
+func GetEvictionCost(annotations map[string]string) (int32, error) {
+	if value, exist := annotations[AnnotationEvictionCost]; exist {
+		// values that start with plus sign (e.g, "+10") or leading zeros (e.g., "008") are not valid.
+		if !validFirstDigit(value) {
+			return 0, fmt.Errorf("invalid value %q", value)
+		}
+
+		i, err := strconv.ParseInt(value, 10, 32)
+		if err != nil {
+			// make sure we default to 0 on error.
+			return 0, err
+		}
+		return int32(i), nil
+	}
+	return 0, nil
+}
+
+func validFirstDigit(str string) bool {
+	if len(str) == 0 {
+		return false
+	}
+	return str[0] == '-' || (str[0] == '0' && str == "0") || (str[0] >= '1' && str[0] <= '9')
+}

extension/elastic_quota.go

@@ -27,9 +27,9 @@ import (
 
 // RootQuotaName means quotaTree's root\head.
 const (
-	SystemQuotaName        = "system"
-	RootQuotaName          = "root"
-	DefaultQuotaName       = "default"
+	SystemQuotaName        = "koordinator-system-quota"
+	RootQuotaName          = "koordinator-root-quota"
+	DefaultQuotaName       = "koordinator-default-quota"
 	QuotaKoordinatorPrefix = "quota.scheduling.koordinator.sh"
 	LabelQuotaIsParent     = QuotaKoordinatorPrefix + "/is-parent"
 	LabelQuotaParent       = QuotaKoordinatorPrefix + "/parent"

extension/node_reservation.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package extension
+
+import (
+	"encoding/json"
+	"math"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+)
+
+const (
+	AnnotationNodeReservation = NodeDomainPrefix + "/reservation"
+)
+
+// NodeReservation resource reserved by node.annotation,
+// If node.annotation declares the resources to be reserved, like this:
+//  annotations:
+//    node.koordinator.sh/reservation: >-
+//	    {"reservedCPUs":"0-5"}
+
+//   In the filter phase it needs to satisfy: node.alloc - node.req - reserved(6c) > pod.req
+//   if qos==LSE/LSR: the cores 0-5 are not used in the reserve phase
+type NodeReservation struct {
+	// resources need to be reserved. like, {"cpu":"1C", "memory":"2Gi"}
+	Resources corev1.ResourceList `json:"resources,omitempty"`
+	// reserved cpus need to be reserved, such as 1-6, or 2,4,6,8
+	ReservedCPUs string `json:"reservedCPUs,omitempty"`
+}
+
+func GetReservedCPUs(anno map[string]string) (string, int) {
+	specificCPUsReservedStr := ""
+	numReservedCPUs := 0
+
+	val, ok := anno[AnnotationNodeReservation]
+	if !ok || val == "" {
+		return specificCPUsReservedStr, numReservedCPUs
+	}
+
+	reserved := NodeReservation{}
+	if err := json.Unmarshal([]byte(val), &reserved); err != nil {
+		klog.Errorf("failed to unmarshal reserved resources from node.annotation in nodenumaresource scheduler plugin.err:%v", err)
+		return specificCPUsReservedStr, numReservedCPUs
+	}
+
+	CPUsQuantityReserved, ok := reserved.Resources[corev1.ResourceCPU]
+	if ok && CPUsQuantityReserved.MilliValue() > 0 {
+		reservedCPUsFloat := float64(CPUsQuantityReserved.MilliValue()) / 1000
+		numReservedCPUs = int(math.Ceil(reservedCPUsFloat))
+	}
+
+	if reserved.ReservedCPUs != "" {
+		numReservedCPUs = 0
+	}
+	specificCPUsReservedStr = reserved.ReservedCPUs
+
+	return specificCPUsReservedStr, numReservedCPUs
+}

extension/priority.go

@@ -24,6 +24,7 @@ import (
 
 type PriorityClass string
 
+// https://koordinator.sh/docs/architecture/priority/
 const (
 	PriorityProd  PriorityClass = "koord-prod"
 	PriorityMid   PriorityClass = "koord-mid"

extension/qos.go

@@ -16,10 +16,9 @@ limitations under the License.
 
 package extension
 
-import corev1 "k8s.io/api/core/v1"
-
 type QoSClass string
 
+// https://koordinator.sh/docs/architecture/qos/
 const (
 	QoSLSE    QoSClass = "LSE"
 	QoSLSR    QoSClass = "LSR"
@@ -29,21 +28,6 @@ const (
 	QoSNone   QoSClass = ""
 )
 
-func GetPodQoSClass(pod *corev1.Pod) QoSClass {
-	if pod == nil || pod.Labels == nil {
-		return QoSNone
-	}
-	return GetQoSClassByAttrs(pod.Labels, pod.Annotations)
-}
-
-func GetQoSClassByAttrs(labels, annotations map[string]string) QoSClass {
-	// annotations are for old format adaption reason
-	if q, exist := labels[LabelPodQoS]; exist {
-		return GetPodQoSClassByName(q)
-	}
-	return QoSNone
-}
-
 func GetPodQoSClassByName(qos string) QoSClass {
 	q := QoSClass(qos)
 

extension/qos_utils.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package extension
+
+import corev1 "k8s.io/api/core/v1"
+
+// NOTE: functions in this file can be overwritten for extension
+
+func GetPodQoSClass(pod *corev1.Pod) QoSClass {
+	if pod == nil || pod.Labels == nil {
+		return QoSNone
+	}
+	return GetQoSClassByAttrs(pod.Labels, pod.Annotations)
+}
+
+func GetQoSClassByAttrs(labels, annotations map[string]string) QoSClass {
+	// annotations are for old format adaption reason
+	if q, exist := labels[LabelPodQoS]; exist {
+		return GetPodQoSClassByName(q)
+	}
+	return QoSNone
+}