Skip to content

Commit

Permalink
koordlet: tc plugin for netqos
Browse files Browse the repository at this point in the history
  • Loading branch information
lucming committed Mar 25, 2024
1 parent c590941 commit e656c39
Show file tree
Hide file tree
Showing 15 changed files with 1,347 additions and 2 deletions.
3 changes: 3 additions & 0 deletions config/crd/bases/slo.koordinator.sh_nodeslos.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1267,6 +1267,9 @@ spec:
policies:
description: Policies of pod QoS.
properties:
NETQOSPolicy:
description: applied policy for the Net QoS, default = "tc"
type: string
cpuPolicy:
description: applied policy for the CPU QoS, default = "groupIdentity"
type: string
Expand Down
2 changes: 2 additions & 0 deletions docker/koordlet.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ ARG TARGETARCH
ENV VERSION $VERSION
ENV GOOS linux
ENV GOARCH $TARGETARCH
ENV GOPROXY https://goproxy.cn,direct

COPY go.mod go.mod
COPY go.sum go.sum
Expand Down Expand Up @@ -35,6 +36,7 @@ RUN go build -a -o koordlet cmd/koordlet/main.go
FROM --platform=$TARGETPLATFORM nvidia/cuda:11.6.2-base-ubuntu20.04
WORKDIR /
RUN apt-get update && apt-get install -y lvm2 && rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get install -y iptables
COPY --from=builder /go/src/github.com/koordinator-sh/koordinator/koordlet .
COPY --from=builder /usr/local/lib /usr/lib
ENTRYPOINT ["/koordlet"]
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82
github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5
github.com/containerd/nri v0.3.0
github.com/coreos/go-iptables v0.5.0
github.com/docker/docker v20.10.21+incompatible
github.com/evanphx/json-patch v5.6.0+incompatible
github.com/fsnotify/fsnotify v1.6.0
Expand Down Expand Up @@ -185,7 +186,7 @@ require (
github.com/stretchr/objx v0.5.0 // indirect
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/ugorji/go/codec v1.2.7 // indirect
github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 // indirect
github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
github.com/vmware/govmomi v0.30.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.5 // indirect
Expand Down
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,7 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-iptables v0.5.0 h1:mw6SAibtHKZcNzAsOxjoHIG0gy5YFHhypWSSNc6EjbQ=
github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
Expand Down
8 changes: 8 additions & 0 deletions pkg/koordlet/runtimehooks/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/cpuset"
"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/gpu"
"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/groupidentity"
"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/tc"
"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/terwayqos"
"github.com/koordinator-sh/koordinator/pkg/koordlet/util/system"
)
Expand Down Expand Up @@ -80,6 +81,11 @@ const (
// owner: @l1b0k
// alpha: v1.5
TerwayQoS featuregate.Feature = "TerwayQoS"

// TCNetworkQoS indicates a network qos implementation based on tc.
// owner: @lucming
// alpha: v1.5
TCNetworkQoS featuregate.Feature = "TCNetworkQoS"
)

var (
Expand All @@ -91,6 +97,7 @@ var (
CPUNormalization: {Default: false, PreRelease: featuregate.Alpha},
CoreSched: {Default: false, PreRelease: featuregate.Alpha},
TerwayQoS: {Default: false, PreRelease: featuregate.Alpha},
TCNetworkQoS: {Default: false, PreRelease: featuregate.Alpha},
}

runtimeHookPlugins = map[featuregate.Feature]HookPlugin{
Expand All @@ -101,6 +108,7 @@ var (
CPUNormalization: cpunormalization.Object(),
CoreSched: coresched.Object(),
TerwayQoS: terwayqos.Object(),
TCNetworkQoS: tc.Object(),
}
)

Expand Down
94 changes: 94 additions & 0 deletions pkg/koordlet/runtimehooks/hooks/tc/helper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
Copyright 2022 The Koordinator Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tc

import (
"fmt"

Check failure on line 20 in pkg/koordlet/runtimehooks/hooks/tc/helper.go

View workflow job for this annotation

GitHub Actions / golangci-lint

File is not `goimports`-ed with -local github.com/koordinator-sh/koordinator (goimports)
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/util/intstr"

slov1alpha1 "github.com/koordinator-sh/koordinator/apis/slo/v1alpha1"
)

func loadConfigFromNodeSlo(nodesloSpec *slov1alpha1.NodeSLOSpec) *NetQosGlobalConfig {
res := NetQosGlobalConfig{}
var total uint64 = 0
if nodesloSpec != nil && nodesloSpec.SystemStrategy != nil {
total = uint64(nodesloSpec.SystemStrategy.TotalNetworkBandwidth.Value())
res.HwRxBpsMax = total
res.HwTxBpsMax = total
}

if nodesloSpec.ResourceQOSStrategy == nil {
return &res
}

strategy := nodesloSpec.ResourceQOSStrategy
if strategy.LSClass != nil &&
strategy.LSClass.NetworkQOS != nil &&
*strategy.LSClass.NetworkQOS.Enable {
cur := strategy.LSClass.NetworkQOS
res.L1RxBpsMin = getBandwidthVal(total, cur.IngressRequest)
res.L1RxBpsMax = getBandwidthVal(total, cur.IngressLimit)
res.L1TxBpsMin = getBandwidthVal(total, cur.EgressRequest)
res.L1TxBpsMax = getBandwidthVal(total, cur.EgressLimit)
}

if strategy.BEClass != nil &&
strategy.BEClass.NetworkQOS != nil &&
*strategy.BEClass.NetworkQOS.Enable {
cur := strategy.BEClass.NetworkQOS
res.L2RxBpsMin = getBandwidthVal(total, cur.IngressRequest)
res.L2RxBpsMax = getBandwidthVal(total, cur.IngressLimit)
res.L2TxBpsMin = getBandwidthVal(total, cur.EgressRequest)
res.L2TxBpsMax = getBandwidthVal(total, cur.EgressLimit)
}

return &res
}

func getBandwidthVal(total uint64, intOrPercent *intstr.IntOrString) uint64 {
if intOrPercent == nil {
return 0
}

switch intOrPercent.Type {
case intstr.String:
return getBandwidthByQuantityFormat(intOrPercent.StrVal)
case intstr.Int:
return getBandwidthByPercentageFormat(total, intOrPercent.IntValue())
default:
return 0
}
}

func getBandwidthByQuantityFormat(quanityStr string) uint64 {
val, err := resource.ParseQuantity(quanityStr)
if err != nil {
return 0
}

return uint64(val.Value())
}

func getBandwidthByPercentageFormat(total uint64, percentage int) uint64 {
return total * uint64(percentage) / 100
}

func convertToClassId(major, minor int) string {
return fmt.Sprintf("%d:%d", major, minor)
}
78 changes: 78 additions & 0 deletions pkg/koordlet/runtimehooks/hooks/tc/ipset.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
//go:build linux
// +build linux

/*
Copyright 2022 The Koordinator Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tc

import (
"fmt"

"github.com/vishvananda/netlink"
apierror "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/klog/v2"
)

func (p *tcPlugin) ipsetExisted() (bool, error) {
var errs []error
for _, cur := range ipsets {
if _, err := netlink.IpsetList(cur); err != nil {
errs = append(errs, err)
}
}

if apierror.NewAggregate(errs) != nil {
return false, apierror.NewAggregate(errs)
}

return true, nil
}

func (p *tcPlugin) EnsureIpset() error {
klog.V(5).Infof("start to create ipset.")
var errs []error
for _, cur := range ipsets {
result, err := netlink.IpsetList(cur)
if err == nil && result != nil {
continue
}

err = netlink.IpsetCreate(cur, "hash:ip", netlink.IpsetCreateOptions{})
if err != nil {
err = fmt.Errorf("failed to create ipset. err=%v", err)
errs = append(errs, err)
}
}

return apierror.NewAggregate(errs)
}

func (p *tcPlugin) DestoryIpset() error {
klog.V(5).Infof("start to delete ipset rules crated by tc plugin.")
var errs []error
for _, cur := range ipsets {
result, err := netlink.IpsetList(cur)
if err == nil && result != nil {
if err := netlink.IpsetDestroy(cur); err != nil {
err = fmt.Errorf("failed to destroy ipset. err=%v", err)
errs = append(errs, err)
}
}
}

return apierror.NewAggregate(errs)
}
Loading

0 comments on commit e656c39

Please sign in to comment.