diff --git a/apis/slo/v1alpha1/nodeslo_types.go b/apis/slo/v1alpha1/nodeslo_types.go index 581954d5e..140c0f459 100644 --- a/apis/slo/v1alpha1/nodeslo_types.go +++ b/apis/slo/v1alpha1/nodeslo_types.go @@ -199,6 +199,27 @@ type IOCfg struct { // +kubebuilder:validation:Maximum=100 // +kubebuilder:validation:Minimum=0 WriteLatencyPercent *int64 `json:"writeLatencyPercent,omitempty"` + // configure the cost model of blkio-cost manually + // whether the user model is enabled. Default value: false + EnableUserModel *bool `json:"enableUserModel,omitempty"` + // the read BPS of user model + // +kubebuilder:validation:Minimum=1 + ModelReadBPS *int64 `json:"modelReadBPS,omitempty"` + // the write BPS of user model + // +kubebuilder:validation:Minimum=1 + ModelWriteBPS *int64 `json:"modelWriteBPS,omitempty"` + // the sequential read iops of user model + // +kubebuilder:validation:Minimum=1 + ModelReadSeqIOPS *int64 `json:"modelReadSeqIOPS,omitempty"` + // the sequential write iops of user model + // +kubebuilder:validation:Minimum=1 + ModelWriteSeqIOPS *int64 `json:"modelWriteSeqIOPS,omitempty"` + // the random read iops of user model + // +kubebuilder:validation:Minimum=1 + ModelReadRandIOPS *int64 `json:"modelReadRandIOPS,omitempty"` + // the random write iops of user model + // +kubebuilder:validation:Minimum=1 + ModelWriteRandIOPS *int64 `json:"modelWriteRandIOPS,omitempty"` } type BlockCfg struct { diff --git a/apis/slo/v1alpha1/zz_generated.deepcopy.go b/apis/slo/v1alpha1/zz_generated.deepcopy.go index df9eacacd..575209cac 100644 --- a/apis/slo/v1alpha1/zz_generated.deepcopy.go +++ b/apis/slo/v1alpha1/zz_generated.deepcopy.go @@ -357,6 +357,41 @@ func (in *IOCfg) DeepCopyInto(out *IOCfg) { *out = new(int64) **out = **in } + if in.EnableUserModel != nil { + in, out := &in.EnableUserModel, &out.EnableUserModel + *out = new(bool) + **out = **in + } + if in.ModelReadBPS != nil { + in, out := &in.ModelReadBPS, &out.ModelReadBPS + *out = new(int64) + **out = **in + } + if in.ModelWriteBPS != nil { + in, out := &in.ModelWriteBPS, &out.ModelWriteBPS + *out = new(int64) + **out = **in + } + if in.ModelReadSeqIOPS != nil { + in, out := &in.ModelReadSeqIOPS, &out.ModelReadSeqIOPS + *out = new(int64) + **out = **in + } + if in.ModelWriteSeqIOPS != nil { + in, out := &in.ModelWriteSeqIOPS, &out.ModelWriteSeqIOPS + *out = new(int64) + **out = **in + } + if in.ModelReadRandIOPS != nil { + in, out := &in.ModelReadRandIOPS, &out.ModelReadRandIOPS + *out = new(int64) + **out = **in + } + if in.ModelWriteRandIOPS != nil { + in, out := &in.ModelWriteRandIOPS, &out.ModelWriteRandIOPS + *out = new(int64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IOCfg. diff --git a/config/crd/bases/slo.koordinator.sh_nodeslos.yaml b/config/crd/bases/slo.koordinator.sh_nodeslos.yaml index 1f5970e82..e279fddaa 100644 --- a/config/crd/bases/slo.koordinator.sh_nodeslos.yaml +++ b/config/crd/bases/slo.koordinator.sh_nodeslos.yaml @@ -114,6 +114,11 @@ spec: properties: ioCfg: properties: + enableUserModel: + description: 'configure the cost model of blkio-cost + manually whether the user model is enabled. + Default value: false' + type: boolean ioWeightPercent: description: 'This field is used to set the weight of a sub-group. Default value: 100. @@ -122,6 +127,38 @@ spec: maximum: 100 minimum: 1 type: integer + modelReadBPS: + description: the read BPS of user model + format: int64 + minimum: 1 + type: integer + modelReadRandIOPS: + description: the random read iops of user model + format: int64 + minimum: 1 + type: integer + modelReadSeqIOPS: + description: the sequential read iops of user + model + format: int64 + minimum: 1 + type: integer + modelWriteBPS: + description: the write BPS of user model + format: int64 + minimum: 1 + type: integer + modelWriteRandIOPS: + description: the random write iops of user model + format: int64 + minimum: 1 + type: integer + modelWriteSeqIOPS: + description: the sequential write iops of user + model + format: int64 + minimum: 1 + type: integer readBPS: description: Throttling of throughput The value is set to 0, which indicates that the feature @@ -404,6 +441,11 @@ spec: properties: ioCfg: properties: + enableUserModel: + description: 'configure the cost model of blkio-cost + manually whether the user model is enabled. + Default value: false' + type: boolean ioWeightPercent: description: 'This field is used to set the weight of a sub-group. Default value: 100. @@ -412,6 +454,38 @@ spec: maximum: 100 minimum: 1 type: integer + modelReadBPS: + description: the read BPS of user model + format: int64 + minimum: 1 + type: integer + modelReadRandIOPS: + description: the random read iops of user model + format: int64 + minimum: 1 + type: integer + modelReadSeqIOPS: + description: the sequential read iops of user + model + format: int64 + minimum: 1 + type: integer + modelWriteBPS: + description: the write BPS of user model + format: int64 + minimum: 1 + type: integer + modelWriteRandIOPS: + description: the random write iops of user model + format: int64 + minimum: 1 + type: integer + modelWriteSeqIOPS: + description: the sequential write iops of user + model + format: int64 + minimum: 1 + type: integer readBPS: description: Throttling of throughput The value is set to 0, which indicates that the feature @@ -694,6 +768,11 @@ spec: properties: ioCfg: properties: + enableUserModel: + description: 'configure the cost model of blkio-cost + manually whether the user model is enabled. + Default value: false' + type: boolean ioWeightPercent: description: 'This field is used to set the weight of a sub-group. Default value: 100. @@ -702,6 +781,38 @@ spec: maximum: 100 minimum: 1 type: integer + modelReadBPS: + description: the read BPS of user model + format: int64 + minimum: 1 + type: integer + modelReadRandIOPS: + description: the random read iops of user model + format: int64 + minimum: 1 + type: integer + modelReadSeqIOPS: + description: the sequential read iops of user + model + format: int64 + minimum: 1 + type: integer + modelWriteBPS: + description: the write BPS of user model + format: int64 + minimum: 1 + type: integer + modelWriteRandIOPS: + description: the random write iops of user model + format: int64 + minimum: 1 + type: integer + modelWriteSeqIOPS: + description: the sequential write iops of user + model + format: int64 + minimum: 1 + type: integer readBPS: description: Throttling of throughput The value is set to 0, which indicates that the feature @@ -984,6 +1095,11 @@ spec: properties: ioCfg: properties: + enableUserModel: + description: 'configure the cost model of blkio-cost + manually whether the user model is enabled. + Default value: false' + type: boolean ioWeightPercent: description: 'This field is used to set the weight of a sub-group. Default value: 100. @@ -992,6 +1108,38 @@ spec: maximum: 100 minimum: 1 type: integer + modelReadBPS: + description: the read BPS of user model + format: int64 + minimum: 1 + type: integer + modelReadRandIOPS: + description: the random read iops of user model + format: int64 + minimum: 1 + type: integer + modelReadSeqIOPS: + description: the sequential read iops of user + model + format: int64 + minimum: 1 + type: integer + modelWriteBPS: + description: the write BPS of user model + format: int64 + minimum: 1 + type: integer + modelWriteRandIOPS: + description: the random write iops of user model + format: int64 + minimum: 1 + type: integer + modelWriteSeqIOPS: + description: the sequential write iops of user + model + format: int64 + minimum: 1 + type: integer readBPS: description: Throttling of throughput The value is set to 0, which indicates that the feature @@ -1284,6 +1432,11 @@ spec: properties: ioCfg: properties: + enableUserModel: + description: 'configure the cost model of blkio-cost + manually whether the user model is enabled. + Default value: false' + type: boolean ioWeightPercent: description: 'This field is used to set the weight of a sub-group. Default value: 100. @@ -1292,6 +1445,38 @@ spec: maximum: 100 minimum: 1 type: integer + modelReadBPS: + description: the read BPS of user model + format: int64 + minimum: 1 + type: integer + modelReadRandIOPS: + description: the random read iops of user model + format: int64 + minimum: 1 + type: integer + modelReadSeqIOPS: + description: the sequential read iops of user + model + format: int64 + minimum: 1 + type: integer + modelWriteBPS: + description: the write BPS of user model + format: int64 + minimum: 1 + type: integer + modelWriteRandIOPS: + description: the random write iops of user model + format: int64 + minimum: 1 + type: integer + modelWriteSeqIOPS: + description: the sequential write iops of user + model + format: int64 + minimum: 1 + type: integer readBPS: description: Throttling of throughput The value is set to 0, which indicates that the feature diff --git a/pkg/koordlet/qosmanager/plugins/blkio/blkio_reconcile.go b/pkg/koordlet/qosmanager/plugins/blkio/blkio_reconcile.go index 36d50586c..17aec7660 100644 --- a/pkg/koordlet/qosmanager/plugins/blkio/blkio_reconcile.go +++ b/pkg/koordlet/qosmanager/plugins/blkio/blkio_reconcile.go @@ -420,8 +420,12 @@ func (b *blkIOReconcile) getDiskNumberFromBlockCfg(block *slov1alpha1.BlockCfg, // configure cgroup root // dynamicPath for root: "" func getDiskConfigUpdaterFromBlockCfg(block *slov1alpha1.BlockCfg, diskNumber string, dynamicPath string) (resources []resourceexecutor.ResourceUpdater) { - var readlat, writelat int64 = DefaultIOLatency, DefaultIOLatency - var readlatPercent, writelatPercent int64 = DefaultLatencyPercent, DefaultLatencyPercent + var ( + readlat, writelat int64 = DefaultIOLatency, DefaultIOLatency + readlatPercent, writelatPercent int64 = DefaultLatencyPercent, DefaultLatencyPercent + enableUserModel bool = false + modelReadBPS, modelWriteBPS, modelReadSeqIOPS, modelWriteSeqIOPS, modelReadRandIOPS, modelWriteRandIOPS int64 = 0, 0, 0, 0, 0, 0 + ) // disk io weight latency if value := block.IOCfg.ReadLatency; value != nil { readlat = *value @@ -438,6 +442,31 @@ func getDiskConfigUpdaterFromBlockCfg(block *slov1alpha1.BlockCfg, diskNumber st writelatPercent = *value } + // user cost model configuration + if value := block.IOCfg.EnableUserModel; value != nil { + enableUserModel = *value + } + if enableUserModel { + if value := block.IOCfg.ModelReadBPS; value != nil { + modelReadBPS = *value + } + if value := block.IOCfg.ModelWriteBPS; value != nil { + modelWriteBPS = *value + } + if value := block.IOCfg.ModelReadSeqIOPS; value != nil { + modelReadSeqIOPS = *value + } + if value := block.IOCfg.ModelWriteSeqIOPS; value != nil { + modelWriteSeqIOPS = *value + } + if value := block.IOCfg.ModelReadRandIOPS; value != nil { + modelReadRandIOPS = *value + } + if value := block.IOCfg.ModelWriteRandIOPS; value != nil { + modelWriteRandIOPS = *value + } + } + ioQoSUpdater, _ := resourceexecutor.NewBlkIOResourceUpdater( system.BlkioIOQoSName, dynamicPath, @@ -447,6 +476,26 @@ func getDiskConfigUpdaterFromBlockCfg(block *slov1alpha1.BlockCfg, diskNumber st resources = append(resources, ioQoSUpdater) + if enableUserModel { + ioModelUpdater, _ := resourceexecutor.NewBlkIOResourceUpdater( + system.BlkioIOModelName, + dynamicPath, + fmt.Sprintf("%s ctrl=user rbps=%d rseqiops=%d rrandiops=%d wbps=%d wseqiops=%d wrandiops=%d", diskNumber, modelReadBPS, modelReadSeqIOPS, modelReadRandIOPS, modelWriteBPS, modelWriteSeqIOPS, modelWriteRandIOPS), + audit.V(3).Group("blkio").Reason("UpdateBlkIO").Message("update %s/%s to %s", dynamicPath, system.BlkioIOModelName, fmt.Sprintf("%s ctrl=user rbps=%d rseqiops=%d rrandiops=%d wbps=%d wseqiops=%d wrandiops=%d", diskNumber, modelReadBPS, modelReadSeqIOPS, modelReadRandIOPS, modelWriteBPS, modelWriteSeqIOPS, modelWriteRandIOPS)), + ) + + resources = append(resources, ioModelUpdater) + } else { + ioModelUpdater, _ := resourceexecutor.NewBlkIOResourceUpdater( + system.BlkioIOModelName, + dynamicPath, + fmt.Sprintf("%s ctrl=auto", diskNumber), + audit.V(3).Group("blkio").Reason("UpdateBlkIO").Message("update %s/%s to %s", dynamicPath, system.BlkioIOModelName, fmt.Sprintf("%s ctrl=auto", diskNumber)), + ) + + resources = append(resources, ioModelUpdater) + } + return } diff --git a/pkg/koordlet/qosmanager/plugins/blkio/blkio_reconcile_test.go b/pkg/koordlet/qosmanager/plugins/blkio/blkio_reconcile_test.go index 628f730dd..0664b2e0f 100644 --- a/pkg/koordlet/qosmanager/plugins/blkio/blkio_reconcile_test.go +++ b/pkg/koordlet/qosmanager/plugins/blkio/blkio_reconcile_test.go @@ -250,6 +250,13 @@ func newNodeSLO() *slov1alpha1.NodeSLO { WriteLatency: pointer.Int64(1000), ReadLatencyPercent: pointer.Int64(90), WriteLatencyPercent: pointer.Int64(90), + EnableUserModel: pointer.Bool(true), + ModelReadBPS: pointer.Int64(3324911720), + ModelWriteBPS: pointer.Int64(2765819289), + ModelReadSeqIOPS: pointer.Int64(168274), + ModelWriteSeqIOPS: pointer.Int64(367565), + ModelReadRandIOPS: pointer.Int64(352545), + ModelWriteRandIOPS: pointer.Int64(339390), }, }, }, diff --git a/pkg/koordlet/resourceexecutor/updater.go b/pkg/koordlet/resourceexecutor/updater.go index 896c6024c..526226a8c 100644 --- a/pkg/koordlet/resourceexecutor/updater.go +++ b/pkg/koordlet/resourceexecutor/updater.go @@ -75,6 +75,7 @@ func init() { sysutil.BlkioTWIopsName, sysutil.BlkioTWBpsName, sysutil.BlkioIOQoSName, + sysutil.BlkioIOModelName, sysutil.BlkioIOWeightName, ) } @@ -563,7 +564,7 @@ func cgroupBlkIOFileWriteIfDifferent(cgroupTaskDir string, file sysutil.Resource } switch file.ResourceType() { - case sysutil.BlkioIOQoSName: + case sysutil.BlkioIOQoSName, sysutil.BlkioIOModelName: needUpdate = CheckIfBlkRootConfigNeedUpdate(currentValue, value) case sysutil.BlkioTRIopsName, sysutil.BlkioTRBpsName, sysutil.BlkioTWIopsName, sysutil.BlkioTWBpsName, sysutil.BlkioIOWeightName: needUpdate = CheckIfBlkQOSNeedUpdate(currentValue, value) diff --git a/pkg/koordlet/util/system/cgroup_resource.go b/pkg/koordlet/util/system/cgroup_resource.go index 5ad67f18a..d89ebbd95 100644 --- a/pkg/koordlet/util/system/cgroup_resource.go +++ b/pkg/koordlet/util/system/cgroup_resource.go @@ -169,6 +169,7 @@ const ( BlkioTWBpsName = "blkio.throttle.write_bps_device" BlkioIOWeightName = "blkio.cost.weight" BlkioIOQoSName = "blkio.cost.qos" + BlkioIOModelName = "blkio.cost.model" ) var ( @@ -192,6 +193,7 @@ var ( BlkioTWBpsValidator = &BlkIORangeValidator{min: 0, max: math.MaxInt64, resource: BlkioTWBpsName} BlkioIOWeightValidator = &BlkIORangeValidator{min: 1, max: 100, resource: BlkioIOWeightName} BlkioIOQoSValidator = &BlkIORangeValidator{min: 0, max: math.MaxInt64, resource: BlkioIOQoSName} + BlkioIOModelValidator = &BlkIORangeValidator{min: 1, max: math.MaxInt64, resource: BlkioIOModelName} CPUSetCPUSValidator = &CPUSetStrValidator{} ) @@ -239,6 +241,7 @@ var ( BlkioWriteBps = DefaultFactory.New(BlkioTWBpsName, CgroupBlkioDir).WithValidator(BlkioTWBpsValidator).WithCheckSupported(SupportedIfFileExistsInKubepods).WithCheckOnce(true) BlkioIOWeight = DefaultFactory.New(BlkioIOWeightName, CgroupBlkioDir).WithValidator(BlkioIOWeightValidator).WithCheckSupported(SupportedIfFileExistsInKubepods).WithCheckOnce(true) BlkioIOQoS = DefaultFactory.New(BlkioIOQoSName, CgroupBlkioDir).WithValidator(BlkioIOQoSValidator).WithSupported(SupportedIfFileExistsInRootCgroup(BlkioIOQoSName, CgroupBlkioDir)) + BlkioIOModel = DefaultFactory.New(BlkioIOModelName, CgroupBlkioDir).WithValidator(BlkioIOModelValidator).WithSupported(SupportedIfFileExistsInRootCgroup(BlkioIOModelName, CgroupBlkioDir)) knownCgroupResources = []Resource{ CPUStat, @@ -276,6 +279,7 @@ var ( BlkioWriteBps, BlkioIOWeight, BlkioIOQoS, + BlkioIOModel, } CPUCFSQuotaV2 = DefaultFactory.NewV2(CPUCFSQuotaName, CPUMaxName) @@ -340,8 +344,7 @@ var ( MemoryPriorityV2, MemoryUsePriorityOomV2, MemoryOomGroupV2, - BlkioIOWeight, - BlkioIOQoS, + // TODO: register BlkioIOWeight, BlkioIOQoS and BlkioIOModel } ) diff --git a/pkg/koordlet/util/system/validator.go b/pkg/koordlet/util/system/validator.go index bb4f5e672..9b89a6ef2 100644 --- a/pkg/koordlet/util/system/validator.go +++ b/pkg/koordlet/util/system/validator.go @@ -86,11 +86,18 @@ func (r *BlkIORangeValidator) Validate(value string) (bool, string) { newValues = append(newValues, rst[1]) } case BlkioIOQoSName: - // 253:16 enable=1 ctrl=user rlat=3000 wlat=4000 + // 253:16 enable=1 ctrl=user rpct=95 rlat=3000 wpct=95 wlat=4000 // 253:16 enable=0 rst := strings.Split(value, " ") - if len(rst) == 5 { - newValues = append(newValues, []string{rst[3][5:], rst[4][5:]}...) + if len(rst) == 7 { + newValues = append(newValues, []string{rst[3][5:], rst[4][5:], rst[5][5:], rst[6][5:]}...) + } + case BlkioIOModelName: + // 253:16 ctrl=user rbps=3324911720 rseqiops=168274 rrandiops=352545 wbps=2765819289 wseqiops=367565 wrandiops=339390 + // 253:16 ctrl=auto + rst := strings.Split(value, " ") + if len(rst) == 8 { + newValues = append(newValues, []string{rst[2][5:], rst[3][9:], rst[4][10:], rst[5][5:], rst[6][9:], rst[7][10:]}...) } default: return false, "unknown blkio resource name"