feat: add downloadRate and uploadRate for host

go.mod

@@ -3,7 +3,7 @@ module d7y.io/dragonfly/v2
 go 1.21
 
 require (
-	d7y.io/api/v2 v2.0.164
+	d7y.io/api/v2 v2.0.166
 	github.com/MysteriousPotato/go-lockable v1.0.0
 	github.com/RichardKnop/machinery v1.10.8
 	github.com/Showmax/go-fqdn v1.0.0

go.sum

@@ -53,8 +53,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-d7y.io/api/v2 v2.0.164 h1:bveQTYw6LFVTj6uEiXqqKCKxrFSbJPIFpiEugw2BUHs=
-d7y.io/api/v2 v2.0.164/go.mod h1:VOnTWgLrGtivgyyofZCfiSDTAKDJ9ohVqM6l3S8EPCE=
+d7y.io/api/v2 v2.0.166 h1:9/MV4qUIXEw/YdQoou2IckYAOy2Vu9G1x/+el54wBIk=
+d7y.io/api/v2 v2.0.166/go.mod h1:VOnTWgLrGtivgyyofZCfiSDTAKDJ9ohVqM6l3S8EPCE=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=

scheduler/resource/persistentcache/host.go

@@ -187,6 +187,18 @@ type Network struct {
 
 	// IDC where the peer host is located
 	IDC string `csv:"idc"`
+
+	// Download rate of the host, unit is byte/s.
+	DownloadRate uint64 `csv:"downloadRate"`
+
+	// Download rate limit of the host, unit is byte/s.
+	DownloadRateLimit uint64 `csv:"downloadRateLimit"`
+
+	// Upload rate of the host, unit is byte/s.
+	UploadRate uint64 `csv:"uploadRate"`
+
+	// Upload rate limit of the host, unit is byte/s.
+	UploadRateLimit uint64 `csv:"uploadRateLimit"`
 }
 
 // Build contains content for build.

scheduler/resource/persistentcache/host_manager.go

@@ -278,11 +278,39 @@ func (t *hostManager) Load(ctx context.Context, hostID string) (*Host, bool) {
 		return nil, false
 	}
 
+	downloadRate, err := strconv.ParseUint(rawHost["network_download_rate"], 10, 64)
+	if err != nil {
+		fmt.Println("parsing download rate failed:", err)
+		return nil, false
+	}
+
+	downloadRateLimit, err := strconv.ParseUint(rawHost["network_download_rate_limit"], 10, 64)
+	if err != nil {
+		fmt.Println("parsing download rate limit failed:", err)
+		return nil, false
+	}
+
+	uploadRate, err := strconv.ParseUint(rawHost["network_upload_rate"], 10, 64)
+	if err != nil {
+		fmt.Println("parsing upload rate failed:", err)
+		return nil, false
+	}
+
+	uploadRateLimit, err := strconv.ParseUint(rawHost["network_upload_rate_limit"], 10, 64)
+	if err != nil {
+		fmt.Println("parsing upload rate limit failed:", err)
+		return nil, false
+	}
+
 	network := Network{
 		TCPConnectionCount:       uint32(networkTCPConnectionCount),
 		UploadTCPConnectionCount: uint32(networkUploadTCPConnectionCount),
 		Location:                 rawHost["network_location"],
 		IDC:                      rawHost["network_idc"],
+		DownloadRate:             downloadRate,
+		DownloadRateLimit:        downloadRateLimit,
+		UploadRate:               uploadRate,
+		UploadRateLimit:          uploadRateLimit,
 	}
 
 	// Set disk fields from raw host.
@@ -440,6 +468,10 @@ func (t *hostManager) Store(ctx context.Context, host *Host) {
 		"network_upload_tcp_connection_count", host.Network.UploadTCPConnectionCount,
 		"network_location", host.Network.Location,
 		"network_idc", host.Network.IDC,
+		"network_download_rate", host.Network.DownloadRate,
+		"network_download_rate_limit", host.Network.DownloadRateLimit,
+		"network_upload_rate", host.Network.UploadRate,
+		"network_upload_rate_limit", host.Network.UploadRateLimit,
 		"disk_total", host.Disk.Total,
 		"disk_free", host.Disk.Free,
 		"disk_used", host.Disk.Used,

scheduler/resource/standard/host.go

@@ -312,6 +312,18 @@ type Network struct {
 
 	// IDC where the peer host is located
 	IDC string `csv:"idc"`
+
+	// Download rate of the host, unit is byte/s.
+	DownloadRate uint64 `csv:"downloadRate"`
+
+	// Download rate limit of the host, unit is byte/s.
+	DownloadRateLimit uint64 `csv:"downloadRateLimit"`
+
+	// Upload rate of the host, unit is byte/s.
+	UploadRate uint64 `csv:"uploadRate"`
+
+	// Upload rate limit of the host, unit is byte/s.
+	UploadRateLimit uint64 `csv:"uploadRateLimit"`
 }
 
 // Build contains content for build.

scheduler/resource/standard/host_test.go

@@ -106,6 +106,10 @@ var (
 		UploadTCPConnectionCount: 1,
 		Location:                 mockHostLocation,
 		IDC:                      mockHostIDC,
+		DownloadRate:             100,
+		DownloadRateLimit:        200,
+		UploadRate:               100,
+		UploadRateLimit:          200,
 	}
 
 	mockDisk = Disk{

scheduler/scheduling/evaluator/evaluator_base_test.go

@@ -109,6 +109,10 @@ var (
 		UploadTCPConnectionCount: 1,
 		Location:                 mockHostLocation,
 		IDC:                      mockHostIDC,
+		DownloadRate:             100,
+		DownloadRateLimit:        200,
+		UploadRate:               100,
+		UploadRateLimit:          200,
 	}
 
 	mockDisk = resource.Disk{

scheduler/scheduling/scheduling.go

@@ -668,6 +668,10 @@ func ConstructSuccessNormalTaskResponse(candidateParents []*resource.Peer) *sche
 				UploadTcpConnectionCount: candidateParent.Host.Network.UploadTCPConnectionCount,
 				Location:                 &candidateParent.Host.Network.Location,
 				Idc:                      &candidateParent.Host.Network.IDC,
+				DownloadRate:             candidateParent.Host.Network.DownloadRate,
+				DownloadRateLimit:        candidateParent.Host.Network.DownloadRateLimit,
+				UploadRate:               candidateParent.Host.Network.UploadRate,
+				UploadRateLimit:          candidateParent.Host.Network.UploadRateLimit,
 			},
 			Disk: &commonv2.Disk{
 				Total:             candidateParent.Host.Disk.Total,

scheduler/scheduling/scheduling_test.go

@@ -137,6 +137,10 @@ var (
 		UploadTCPConnectionCount: 1,
 		Location:                 mockHostLocation,
 		IDC:                      mockHostIDC,
+		DownloadRate:             100,
+		DownloadRateLimit:        200,
+		UploadRate:               100,
+		UploadRateLimit:          200,
 	}
 
 	mockDisk = resource.Disk{
@@ -1724,6 +1728,10 @@ func TestScheduling_ConstructSuccessNormalTaskResponse(t *testing.T) {
 										UploadTcpConnectionCount: candidateParents[0].Host.Network.UploadTCPConnectionCount,
 										Location:                 &candidateParents[0].Host.Network.Location,
 										Idc:                      &candidateParents[0].Host.Network.IDC,
+										DownloadRate:             candidateParents[0].Host.Network.DownloadRate,
+										DownloadRateLimit:        candidateParents[0].Host.Network.DownloadRateLimit,
+										UploadRate:               candidateParents[0].Host.Network.UploadRate,
+										UploadRateLimit:          candidateParents[0].Host.Network.UploadRateLimit,
 									},
 									Disk: &commonv2.Disk{
 										Total:             candidateParents[0].Host.Disk.Total,

scheduler/service/service_v2.go

@@ -346,6 +346,10 @@ func (v *V2) StatPeer(ctx context.Context, req *schedulerv2.StatPeerRequest) (*c
 			UploadTcpConnectionCount: peer.Host.Network.UploadTCPConnectionCount,
 			Location:                 &peer.Host.Network.Location,
 			Idc:                      &peer.Host.Network.IDC,
+			DownloadRate:             peer.Host.Network.DownloadRate,
+			DownloadRateLimit:        peer.Host.Network.DownloadRateLimit,
+			UploadRate:               peer.Host.Network.UploadRate,
+			UploadRateLimit:          peer.Host.Network.UploadRateLimit,
 		},
 		Disk: &commonv2.Disk{
 			Total:             peer.Host.Disk.Total,
@@ -557,6 +561,10 @@ func (v *V2) AnnounceHost(ctx context.Context, req *schedulerv2.AnnounceHostRequ
 				UploadTCPConnectionCount: req.Host.Network.GetUploadTcpConnectionCount(),
 				Location:                 req.Host.Network.GetLocation(),
 				IDC:                      req.Host.Network.GetIdc(),
+				DownloadRate:             req.Host.Network.GetDownloadRate(),
+				DownloadRateLimit:        req.Host.Network.GetDownloadRateLimit(),
+				UploadRate:               req.Host.Network.GetUploadRate(),
+				UploadRateLimit:          req.Host.Network.GetUploadRateLimit(),
 			}))
 		}
 
@@ -655,6 +663,10 @@ func (v *V2) AnnounceHost(ctx context.Context, req *schedulerv2.AnnounceHostRequ
 			UploadTCPConnectionCount: req.Host.Network.GetUploadTcpConnectionCount(),
 			Location:                 req.Host.Network.GetLocation(),
 			IDC:                      req.Host.Network.GetIdc(),
+			DownloadRate:             req.Host.Network.GetDownloadRate(),
+			DownloadRateLimit:        req.Host.Network.GetDownloadRateLimit(),
+			UploadRate:               req.Host.Network.GetUploadRate(),
+			UploadRateLimit:          req.Host.Network.GetUploadRateLimit(),
 		}
 	}
 

scheduler/service/service_v2_test.go

@@ -207,6 +207,10 @@ func TestServiceV2_StatPeer(t *testing.T) {
 							UploadTcpConnectionCount: peer.Host.Network.UploadTCPConnectionCount,
 							Location:                 &peer.Host.Network.Location,
 							Idc:                      &peer.Host.Network.IDC,
+							DownloadRate:             peer.Host.Network.DownloadRate,
+							DownloadRateLimit:        peer.Host.Network.DownloadRateLimit,
+							UploadRate:               peer.Host.Network.UploadRate,
+							UploadRateLimit:          peer.Host.Network.UploadRateLimit,
 						},
 						Disk: &commonv2.Disk{
 							Total:             peer.Host.Disk.Total,
@@ -464,6 +468,10 @@ func TestServiceV2_AnnounceHost(t *testing.T) {
 						UploadTcpConnectionCount: mockNetwork.UploadTCPConnectionCount,
 						Location:                 &mockNetwork.Location,
 						Idc:                      &mockNetwork.IDC,
+						DownloadRate:             mockNetwork.DownloadRate,
+						DownloadRateLimit:        mockNetwork.DownloadRateLimit,
+						UploadRate:               mockNetwork.UploadRate,
+						UploadRateLimit:          mockNetwork.UploadRateLimit,
 					},
 					Disk: &commonv2.Disk{
 						Total:             mockDisk.Total,
@@ -572,6 +580,10 @@ func TestServiceV2_AnnounceHost(t *testing.T) {
 						UploadTcpConnectionCount: mockNetwork.UploadTCPConnectionCount,
 						Location:                 &mockNetwork.Location,
 						Idc:                      &mockNetwork.IDC,
+						DownloadRate:             mockNetwork.DownloadRate,
+						DownloadRateLimit:        mockNetwork.DownloadRateLimit,
+						UploadRate:               mockNetwork.UploadRate,
+						UploadRateLimit:          mockNetwork.UploadRateLimit,
 					},
 					Disk: &commonv2.Disk{
 						Total:             mockDisk.Total,
@@ -680,6 +692,10 @@ func TestServiceV2_AnnounceHost(t *testing.T) {
 						UploadTcpConnectionCount: mockNetwork.UploadTCPConnectionCount,
 						Location:                 &mockNetwork.Location,
 						Idc:                      &mockNetwork.IDC,
+						DownloadRate:             mockNetwork.DownloadRate,
+						DownloadRateLimit:        mockNetwork.DownloadRateLimit,
+						UploadRate:               mockNetwork.UploadRate,
+						UploadRateLimit:          mockNetwork.UploadRateLimit,
 					},
 					Disk: &commonv2.Disk{
 						Total:             mockDisk.Total,
@@ -784,6 +800,10 @@ func TestServiceV2_AnnounceHost(t *testing.T) {
 						UploadTcpConnectionCount: mockNetwork.UploadTCPConnectionCount,
 						Location:                 &mockNetwork.Location,
 						Idc:                      &mockNetwork.IDC,
+						DownloadRate:             mockNetwork.DownloadRate,
+						DownloadRateLimit:        mockNetwork.DownloadRateLimit,
+						UploadRate:               mockNetwork.UploadRate,
+						UploadRateLimit:          mockNetwork.UploadRateLimit,
 					},
 					Disk: &commonv2.Disk{
 						Total:             mockDisk.Total,

scheduler/storage/storage_test.go

@@ -93,6 +93,10 @@ var (
 			UploadTCPConnectionCount: 200,
 			Location:                 "china",
 			IDC:                      "e1",
+			DownloadRate:             100,
+			DownloadRateLimit:        200,
+			UploadRate:               100,
+			UploadRateLimit:          200,
 		},
 		Disk: resource.Disk{
 			Total:             100,