Skip to content

Commit

Permalink
koordlet: tc plugin for netqos
Browse files Browse the repository at this point in the history
  • Loading branch information
@lucming
lucming committed Mar 5, 2024
2 parents 17cbdbd + c590941 commit 278d46a
Show file tree
Hide file tree
Showing 79 changed files with 3,930 additions and 553 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/e2e-k8s-1.22.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
with:
go-version: ${{ env.GO_VERSION }}
- name: Setup Kind Cluster
uses: helm/kind-action@v1.8.0
uses: helm/kind-action@v1.9.0
with:
node_image: ${{ env.KIND_IMAGE }}
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/e2e-k8s-1.24.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
with:
go-version: ${{ env.GO_VERSION }}
- name: Setup Kind Cluster
uses: helm/kind-action@v1.8.0
uses: helm/kind-action@v1.9.0
with:
node_image: ${{ env.KIND_IMAGE }}
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/e2e-k8s-latest.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
with:
go-version: ${{ env.GO_VERSION }}
- name: Setup Kind Cluster
uses: helm/kind-action@v1.8.0
uses: helm/kind-action@v1.9.0
with:
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
config: ./test/kind-conf.yaml
Expand Down
1 change: 1 addition & 0 deletions .licenseignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,4 @@ test/e2e/e2e_test.go
test/e2e/suites.go
test/utils/image
pkg/util/histogram
pkg/util/metricserver
4 changes: 3 additions & 1 deletion apis/configuration/slo_controller_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -484,7 +484,9 @@ data:
{
"ioCfg": {
"readLatency": 3000,
"writeLatency": 3000
"writeLatency": 3000,
"readLatencyPercent": 95,
"writeLatencyPercent": 95
},
"name": "ackdistro-pool",
"type": "volumegroup"
Expand Down
4 changes: 4 additions & 0 deletions apis/extension/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ const (
// LabelPodMutatingUpdate is a label key that pods with `pod.koordinator.sh/mutating-update=true` will
// be mutated by Koordinator webhook when updating.
LabelPodMutatingUpdate = PodDomainPrefix + "/mutating-update"

// AnnotationNetworkQOS are used to set bandwidth for Pod. The unit is bps.
// For example, 10M means 10 megabits per second.
AnnotationNetworkQOS = DomainPrefix + "networkQOS"
)

type AggregationType string
Expand Down
11 changes: 11 additions & 0 deletions apis/extension/elastic_quota.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ const (
AnnotationChildRequest = QuotaKoordinatorPrefix + "/child-request"
AnnotationResourceKeys = QuotaKoordinatorPrefix + "/resource-keys"
AnnotationTotalResource = QuotaKoordinatorPrefix + "/total-resource"
AnnotationUnschedulableResource = QuotaKoordinatorPrefix + "/unschedulable-resource"
AnnotationQuotaNamespaces = QuotaKoordinatorPrefix + "/namespaces"
AnnotationGuaranteed = QuotaKoordinatorPrefix + "/guaranteed"
AnnotationAllocated = QuotaKoordinatorPrefix + "/allocated"
Expand Down Expand Up @@ -195,3 +196,13 @@ func GetChildRequest(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error)
}
return request, nil
}

func GetUnschedulableResource(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error) {
unschedulable := corev1.ResourceList{}
if quota.Annotations[AnnotationUnschedulableResource] != "" {
if err := json.Unmarshal([]byte(quota.Annotations[AnnotationUnschedulableResource]), &unschedulable); err != nil {
return unschedulable, err
}
}
return unschedulable, nil
}
35 changes: 35 additions & 0 deletions apis/extension/reservation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ const (

// AnnotationReservationAffinity represents the constraints of Pod selection Reservation
AnnotationReservationAffinity = SchedulingDomainPrefix + "/reservation-affinity"

// AnnotationReservationRestrictedOptions represent the Reservation Restricted options
AnnotationReservationRestrictedOptions = SchedulingDomainPrefix + "/reservation-restricted-options"
)

type ReservationAllocated struct {
Expand Down Expand Up @@ -67,6 +70,14 @@ type ReservationAffinitySelector struct {
ReservationSelectorTerms []corev1.NodeSelectorTerm `json:"reservationSelectorTerms,omitempty"`
}

type ReservationRestrictedOptions struct {
// Resources means that when the Pod intersects with these resources,
// it can only allocate the reserved amount at most.
// If the Reservation's AllocatePolicy is Restricted, and no resources configured,
// by default the resources equal all reserved resources by the Reservation.
Resources []corev1.ResourceName `json:"resources,omitempty"`
}

func GetReservationAllocated(pod *corev1.Pod) (*ReservationAllocated, error) {
if pod.Annotations == nil {
return nil, nil
Expand Down Expand Up @@ -122,3 +133,27 @@ func SetReservationAffinity(obj metav1.Object, affinity *ReservationAffinity) er
obj.SetAnnotations(annotations)
return nil
}

func GetReservationRestrictedOptions(annotations map[string]string) (*ReservationRestrictedOptions, error) {
var options ReservationRestrictedOptions
if s, ok := annotations[AnnotationReservationRestrictedOptions]; ok && s != "" {
if err := json.Unmarshal([]byte(s), &options); err != nil {
return nil, err
}
}
return &options, nil
}

func SetReservationRestrictedOptions(obj metav1.Object, options *ReservationRestrictedOptions) error {
data, err := json.Marshal(options)
if err != nil {
return err
}
annotations := obj.GetAnnotations()
if annotations == nil {
annotations = map[string]string{}
}
annotations[AnnotationReservationRestrictedOptions] = string(data)
obj.SetAnnotations(annotations)
return nil
}
29 changes: 19 additions & 10 deletions apis/slo/v1alpha1/nodeslo_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,15 @@ const (
CPUQOSPolicyCoreSched CPUQOSPolicy = "coreSched"
)

type NETQOSPolicy string

const (
// NETQOSPolicyTC indicates implement netqos by tc.
NETQOSPolicyTC NETQOSPolicy = "tc"
// NETQOSPolicyTerwayQos indicates implement netqos by terway-qos.
NETQOSPolicyTerwayQos NETQOSPolicy = "terway-qos"
)

// MemoryQOS enables memory qos features.
type MemoryQOS struct {
// memcg qos
Expand Down Expand Up @@ -182,6 +191,14 @@ type IOCfg struct {
ReadLatency *int64 `json:"readLatency,omitempty"`
// the write latency threshold. Unit: microseconds.
WriteLatency *int64 `json:"writeLatency,omitempty"`
// the read latency percentile
// +kubebuilder:validation:Maximum=100
// +kubebuilder:validation:Minimum=0
ReadLatencyPercent *int64 `json:"readLatencyPercent,omitempty"`
// the write latency percentile
// +kubebuilder:validation:Maximum=100
// +kubebuilder:validation:Minimum=0
WriteLatencyPercent *int64 `json:"writeLatencyPercent,omitempty"`
}

type BlockCfg struct {
Expand Down Expand Up @@ -243,19 +260,11 @@ type NetworkQOS struct {
type ResourceQOSPolicies struct {
// applied policy for the CPU QoS, default = "groupIdentity"
CPUPolicy *CPUQOSPolicy `json:"cpuPolicy,omitempty"`

// applied policy for the Net QoS, default = "tc"
NETQOSPolicy *NETQOSPolicy `json:"NETQOSPolicy,omitempty"`
NETQOSPolicy *NETQOSPolicy `json:"netQOSPolicy,omitempty"`
}

type NETQOSPolicy string

const (
// NETQOSPolicyTerwayQos indicates implement netqos by terway-qos.
NETQOSPolicyTerwayQos NETQOSPolicy = "terway-qos"
// NETQOSPolicyTC indicates implement netqos by tc.
NETQOSPolicyTC NETQOSPolicy = "tc"
)

type ResourceQOSStrategy struct {
// Policies of pod QoS.
Policies *ResourceQOSPolicies `json:"policies,omitempty"`
Expand Down
10 changes: 10 additions & 0 deletions apis/slo/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

66 changes: 63 additions & 3 deletions config/crd/bases/slo.koordinator.sh_nodeslos.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,12 @@ spec:
Unit: microseconds.'
format: int64
type: integer
readLatencyPercent:
description: the read latency percentile
format: int64
maximum: 100
minimum: 0
type: integer
writeBPS:
format: int64
minimum: 0
Expand All @@ -163,6 +169,12 @@ spec:
microseconds.'
format: int64
type: integer
writeLatencyPercent:
description: the write latency percentile
format: int64
maximum: 100
minimum: 0
type: integer
type: object
name:
type: string
Expand Down Expand Up @@ -428,6 +440,12 @@ spec:
Unit: microseconds.'
format: int64
type: integer
readLatencyPercent:
description: the read latency percentile
format: int64
maximum: 100
minimum: 0
type: integer
writeBPS:
format: int64
minimum: 0
Expand All @@ -441,6 +459,12 @@ spec:
microseconds.'
format: int64
type: integer
writeLatencyPercent:
description: the write latency percentile
format: int64
maximum: 100
minimum: 0
type: integer
type: object
name:
type: string
Expand Down Expand Up @@ -706,6 +730,12 @@ spec:
Unit: microseconds.'
format: int64
type: integer
readLatencyPercent:
description: the read latency percentile
format: int64
maximum: 100
minimum: 0
type: integer
writeBPS:
format: int64
minimum: 0
Expand All @@ -719,6 +749,12 @@ spec:
microseconds.'
format: int64
type: integer
writeLatencyPercent:
description: the write latency percentile
format: int64
maximum: 100
minimum: 0
type: integer
type: object
name:
type: string
Expand Down Expand Up @@ -984,6 +1020,12 @@ spec:
Unit: microseconds.'
format: int64
type: integer
readLatencyPercent:
description: the read latency percentile
format: int64
maximum: 100
minimum: 0
type: integer
writeBPS:
format: int64
minimum: 0
Expand All @@ -997,6 +1039,12 @@ spec:
microseconds.'
format: int64
type: integer
writeLatencyPercent:
description: the write latency percentile
format: int64
maximum: 100
minimum: 0
type: integer
type: object
name:
type: string
Expand Down Expand Up @@ -1219,12 +1267,12 @@ spec:
policies:
description: Policies of pod QoS.
properties:
NETQOSPolicy:
description: applied policy for the Net QoS, default = "tc"
type: string
cpuPolicy:
description: applied policy for the CPU QoS, default = "groupIdentity"
type: string
netQOSPolicy:
description: applied policy for the Net QoS, default = "tc"
type: string
type: object
systemClass:
description: ResourceQOS for system pods
Expand Down Expand Up @@ -1272,6 +1320,12 @@ spec:
Unit: microseconds.'
format: int64
type: integer
readLatencyPercent:
description: the read latency percentile
format: int64
maximum: 100
minimum: 0
type: integer
writeBPS:
format: int64
minimum: 0
Expand All @@ -1285,6 +1339,12 @@ spec:
microseconds.'
format: int64
type: integer
writeLatencyPercent:
description: the write latency percentile
format: int64
maximum: 100
minimum: 0
type: integer
type: object
name:
type: string
Expand Down
Binary file added docs/images/multi-quota-tree.jpg
View file
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit 278d46a

Please sign in to comment.