Skip to content

Commit

Permalink
koordlet: tc plugin for netqos
Browse files Browse the repository at this point in the history
  • Loading branch information
lucming committed Mar 28, 2024
1 parent c590941 commit d190e52
Show file tree
Hide file tree
Showing 24 changed files with 1,660 additions and 6 deletions.
2 changes: 2 additions & 0 deletions docker/koordlet.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ ARG TARGETARCH
ENV VERSION $VERSION
ENV GOOS linux
ENV GOARCH $TARGETARCH
ENV GOPROXY https://goproxy.cn,direct

COPY go.mod go.mod
COPY go.sum go.sum
Expand Down Expand Up @@ -35,6 +36,7 @@ RUN go build -a -o koordlet cmd/koordlet/main.go
FROM --platform=$TARGETPLATFORM nvidia/cuda:11.6.2-base-ubuntu20.04
WORKDIR /
RUN apt-get update && apt-get install -y lvm2 && rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get install -y iptables
COPY --from=builder /go/src/github.com/koordinator-sh/koordinator/koordlet .
COPY --from=builder /usr/local/lib /usr/lib
ENTRYPOINT ["/koordlet"]
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82
github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5
github.com/containerd/nri v0.3.0
github.com/coreos/go-iptables v0.5.0
github.com/docker/docker v20.10.21+incompatible
github.com/evanphx/json-patch v5.6.0+incompatible
github.com/fsnotify/fsnotify v1.6.0
Expand Down Expand Up @@ -185,7 +186,7 @@ require (
github.com/stretchr/objx v0.5.0 // indirect
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/ugorji/go/codec v1.2.7 // indirect
github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 // indirect
github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
github.com/vmware/govmomi v0.30.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.5 // indirect
Expand Down
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,7 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-iptables v0.5.0 h1:mw6SAibtHKZcNzAsOxjoHIG0gy5YFHhypWSSNc6EjbQ=
github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
Expand Down
1 change: 1 addition & 0 deletions pkg/koordlet/resourceexecutor/updater.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ func init() {
sysutil.MemoryPriorityName,
sysutil.MemoryUsePriorityOomName,
sysutil.MemoryOomGroupName,
sysutil.NetClsClassIdName,
)
// special cases
DefaultCgroupUpdaterFactory.Register(NewCgroupUpdaterWithUpdateFunc(CgroupUpdateCPUSharesFunc), sysutil.CPUSharesName)
Expand Down
8 changes: 8 additions & 0 deletions pkg/koordlet/runtimehooks/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/cpuset"
"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/gpu"
"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/groupidentity"
"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/tc"
"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/terwayqos"
"github.com/koordinator-sh/koordinator/pkg/koordlet/util/system"
)
Expand Down Expand Up @@ -80,6 +81,11 @@ const (
// owner: @l1b0k
// alpha: v1.5
TerwayQoS featuregate.Feature = "TerwayQoS"

// TCNetworkQoS indicates a network qos implementation based on tc.
// owner: @lucming
// alpha: v1.5
TCNetworkQoS featuregate.Feature = "TCNetworkQoS"
)

var (
Expand All @@ -91,6 +97,7 @@ var (
CPUNormalization: {Default: false, PreRelease: featuregate.Alpha},
CoreSched: {Default: false, PreRelease: featuregate.Alpha},
TerwayQoS: {Default: false, PreRelease: featuregate.Alpha},
TCNetworkQoS: {Default: false, PreRelease: featuregate.Alpha},
}

runtimeHookPlugins = map[featuregate.Feature]HookPlugin{
Expand All @@ -101,6 +108,7 @@ var (
CPUNormalization: cpunormalization.Object(),
CoreSched: coresched.Object(),
TerwayQoS: terwayqos.Object(),
TCNetworkQoS: tc.Object(),
}
)

Expand Down
99 changes: 99 additions & 0 deletions pkg/koordlet/runtimehooks/hooks/tc/helper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*
Copyright 2022 The Koordinator Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tc

import (
"fmt"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/util/intstr"

slov1alpha1 "github.com/koordinator-sh/koordinator/apis/slo/v1alpha1"
)

func loadConfigFromNodeSlo(nodesloSpec *slov1alpha1.NodeSLOSpec) *NetQosGlobalConfig {
res := NetQosGlobalConfig{}
var total uint64 = 0
if nodesloSpec != nil && nodesloSpec.SystemStrategy != nil {
total = uint64(nodesloSpec.SystemStrategy.TotalNetworkBandwidth.Value())
res.HwRxBpsMax = total
res.HwTxBpsMax = total
}

if nodesloSpec.ResourceQOSStrategy == nil {
return &res
}

strategy := nodesloSpec.ResourceQOSStrategy
if strategy.LSClass != nil &&
strategy.LSClass.NetworkQOS != nil &&
*strategy.LSClass.NetworkQOS.Enable {
cur := strategy.LSClass.NetworkQOS
res.L1RxBpsMin = getBandwidthVal(total, cur.IngressRequest)
res.L1RxBpsMax = getBandwidthVal(total, cur.IngressLimit)
res.L1TxBpsMin = getBandwidthVal(total, cur.EgressRequest)
res.L1TxBpsMax = getBandwidthVal(total, cur.EgressLimit)
}

if strategy.BEClass != nil &&
strategy.BEClass.NetworkQOS != nil &&
*strategy.BEClass.NetworkQOS.Enable {
cur := strategy.BEClass.NetworkQOS
res.L2RxBpsMin = getBandwidthVal(total, cur.IngressRequest)
res.L2RxBpsMax = getBandwidthVal(total, cur.IngressLimit)
res.L2TxBpsMin = getBandwidthVal(total, cur.EgressRequest)
res.L2TxBpsMax = getBandwidthVal(total, cur.EgressLimit)
}

return &res
}

func getBandwidthVal(total uint64, intOrPercent *intstr.IntOrString) uint64 {
if intOrPercent == nil {
return 0
}

switch intOrPercent.Type {
case intstr.String:
return getBandwidthByQuantityFormat(intOrPercent.StrVal)
case intstr.Int:
return getBandwidthByPercentageFormat(total, intOrPercent.IntValue())
default:
return 0
}
}

func getBandwidthByQuantityFormat(quanityStr string) uint64 {
val, err := resource.ParseQuantity(quanityStr)
if err != nil {
return 0
}

return uint64(val.Value())
}

func getBandwidthByPercentageFormat(total uint64, percentage int) uint64 {
return total * uint64(percentage) / 100
}

func convertToClassId(major, minor int) string {
return fmt.Sprintf("%d:%d", major, minor)
}

// convertToHexClassId get class id in hex.
func convertToHexClassId(major, minor int) string {
return fmt.Sprintf("0x%d%04d", major, minor)
}
42 changes: 42 additions & 0 deletions pkg/koordlet/runtimehooks/hooks/tc/helper_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package tc

import (
"testing"

"github.com/stretchr/testify/assert"
)

func Test_convertToHexClassId(t *testing.T) {
type args struct {
major int
minor int
}
tests := []struct {
name string
args args
want string
}{
// TODO: Add test cases.
{
name: "",
args: args{
major: 11,
minor: 2,
},
want: "0x110002",
},
{
name: "",
args: args{
major: 1,
minor: 2222,
},
want: "0x12222",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, convertToHexClassId(tt.args.major, tt.args.minor), "convertToHexClassId(%v, %v)", tt.args.major, tt.args.minor)
})
}
}
78 changes: 78 additions & 0 deletions pkg/koordlet/runtimehooks/hooks/tc/ipset.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
//go:build linux
// +build linux

/*
Copyright 2022 The Koordinator Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tc

import (
"fmt"

"github.com/vishvananda/netlink"
apierror "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/klog/v2"
)

func (p *tcPlugin) ipsetExisted() (bool, error) {
var errs []error
for _, cur := range ipsets {
if _, err := netlink.IpsetList(cur); err != nil {
errs = append(errs, err)
}
}

if apierror.NewAggregate(errs) != nil {
return false, apierror.NewAggregate(errs)
}

return true, nil
}

func (p *tcPlugin) EnsureIpset() error {
klog.V(5).Infof("start to create ipset.")
var errs []error
for _, cur := range ipsets {
result, err := netlink.IpsetList(cur)
if err == nil && result != nil {
continue
}

err = netlink.IpsetCreate(cur, "hash:ip", netlink.IpsetCreateOptions{})
if err != nil {
err = fmt.Errorf("failed to create ipset. err=%v", err)
errs = append(errs, err)
}
}

return apierror.NewAggregate(errs)
}

func (p *tcPlugin) DestoryIpset() error {
klog.V(5).Infof("start to delete ipset rules crated by tc plugin.")
var errs []error
for _, cur := range ipsets {
result, err := netlink.IpsetList(cur)
if err == nil && result != nil {
if err := netlink.IpsetDestroy(cur); err != nil {
err = fmt.Errorf("failed to destroy ipset. err=%v", err)
errs = append(errs, err)
}
}
}

return apierror.NewAggregate(errs)
}
Loading

0 comments on commit d190e52

Please sign in to comment.