更新上传下载, 新增上传、下载设置限速

#692, #672, #635, #151

README.md

@@ -738,6 +738,12 @@ Windows: `%APPDATA%\BaiduPCS-Go`
 
 可通过设置环境变量 `BAIDUPCS_GO_CONFIG_DIR`, 指定配置文件存放的目录.
 
+谨慎修改 `appid`, `user_agent`, `pcs_ua`, `pan_ua` 的值, 否则访问网盘服务器时, 可能会出现错误.
+
+`cache_size` 的值支持可选设置单位了, 单位不区分大小写, `b` 和 `B` 均表示字节的意思, 如 `64KB`, `1MB`, `32kb`, `65536b`, `65536`.
+
+`max_upload_parallel`, `max_download_load` 的值支持可选设置单位了, 单位为每秒的传输速率, 后缀`/s` 可省略, 如 `2MB/s`, `2MB`, `2m`, `2mb` 均为一个意思.
+
 #### 例子
 ```
 # 显示所有可以设置的值

baidupcs/cache.go

@@ -11,9 +11,10 @@ import (
 func (pcs *BaiduPCS) deleteCache(dirs []string) {
 	cache := pcs.cacheOpMap.LazyInitCachePoolOp(OperationFilesDirectoriesList)
 	for _, v := range dirs {
-		data, ok := cache.Load(v + "_" + defaultOrderOptionsStr)
+		key := v + "_" + defaultOrderOptionsStr
+		_, ok := cache.Load(key)
 		if ok {
-			data.SetExpires(true)
+			cache.Delete(key)
 		}
 	}
 }

baidupcs/expires/cachemap/cachemap.go

@@ -38,3 +38,8 @@ func (cm *CacheOpMap) ClearInvalidate() {
 		return true
 	})
 }
+
+// PrintAll 输出所有缓冲项目
+func (cm *CacheOpMap) PrintAll() {
+
+}

internal/pcscommand/download.go

@@ -93,7 +93,6 @@ func download(id int, downloadURL, savePath string, loadBalansers []string, clie
 		writer    downloader.Writer
 		file      *os.File
 		warn, err error
-		exitChan  chan struct{}
 	)
 
 	if !newCfg.IsTest {
@@ -130,61 +129,49 @@ func download(id int, downloadURL, savePath string, loadBalansers []string, clie
 		return pcserror.DecodePCSJSONError(baidupcs.OperationDownloadFile, respBody)
 	})
 
-	exitChan = make(chan struct{})
-
-	download.OnExecute(func() {
+	var (
+		format = downloadPrintFormat(downloadOptions.Load)
+	)
+	download.OnDownloadStatusEvent(func(status downloader.DownloadStatuser, workersCallback func(downloader.RangeWorkerFunc)) {
 		if downloadOptions.IsPrintStatus {
-			go func() {
-				for {
-					time.Sleep(1 * time.Second)
-					select {
-					case <-exitChan:
-						return
-					default:
-						download.PrintAllWorkers()
-					}
-				}
-			}()
+			// 输出所有的worker状态
+			var (
+				builder = &strings.Builder{}
+				tb      = pcstable.NewTable(builder)
+			)
+			tb.SetHeader([]string{"#", "status", "range", "left", "speeds", "error"})
+			workersCallback(func(key int, worker *downloader.Worker) bool {
+				wrange := worker.GetRange()
+				tb.Append([]string{fmt.Sprint(worker.ID()), worker.GetStatus().StatusText(), wrange.ShowDetails(), strconv.FormatInt(wrange.Len(), 10), strconv.FormatInt(worker.GetSpeedsPerSecond(), 10), fmt.Sprint(worker.Err())})
+				return true
+			})
+			tb.Render()
+			fmt.Fprintf(downloadOptions.Out, "\n\n"+builder.String())
 		}
 
-		if newCfg.IsTest {
-			fmt.Fprintf(downloadOptions.Out, "[%d] 测试下载开始\n\n", id)
+		var leftStr string
+		left := status.TimeLeft()
+		if left < 0 {
+			leftStr = "-"
+		} else {
+			leftStr = left.String()
 		}
 
-		var (
-			ds                            = download.GetDownloadStatusChan()
-			format                        = downloadPrintFormat(downloadOptions.Load)
-			downloaded, totalSize, speeds int64
-			leftStr                       string
+		fmt.Fprintf(downloadOptions.Out, format, id,
+			converter.ConvertFileSize(status.Downloaded(), 2),
+			converter.ConvertFileSize(status.TotalSize(), 2),
+			converter.ConvertFileSize(status.SpeedsPerSecond(), 2),
+			status.TimeElapsed()/1e7*1e7, leftStr,
 		)
-		for {
-			select {
-			case <-exitChan:
-				return
-			case v, ok := <-ds:
-				if !ok { // channel 已经关闭
-					return
-				}
-
-				downloaded, totalSize, speeds = v.Downloaded(), v.TotalSize(), v.SpeedsPerSecond()
-				if speeds <= 0 {
-					leftStr = "-"
-				} else {
-					leftStr = (time.Duration((totalSize-downloaded)/(speeds)) * time.Second).String()
-				}
+	})
 
-				fmt.Fprintf(downloadOptions.Out, format, id,
-					converter.ConvertFileSize(v.Downloaded(), 2),
-					converter.ConvertFileSize(v.TotalSize(), 2),
-					converter.ConvertFileSize(v.SpeedsPerSecond(), 2),
-					v.TimeElapsed()/1e7*1e7, leftStr,
-				)
-			}
+	download.OnExecute(func() {
+		if newCfg.IsTest {
+			fmt.Fprintf(downloadOptions.Out, "[%d] 测试下载开始\n\n", id)
 		}
 	})
 
 	err = download.Execute()
-	close(exitChan)
 	fmt.Fprintf(downloadOptions.Out, "\n")
 	if err != nil {
 		if !newCfg.IsTest {
@@ -271,6 +258,7 @@ func RunDownload(paths []string, options *DownloadOptions) {
 		Mode:                       downloader.RangeGenMode_BlockSize,
 		CacheSize:                  pcsconfig.Config.CacheSize,
 		BlockSize:                  baidupcs.MaxDownloadRangeSize,
+		MaxRate:                    pcsconfig.Config.MaxDownloadRate,
 		InstanceStateStorageFormat: downloader.InstanceStateStorageFormatProto3,
 		IsTest:                     options.IsTest,
 		TryHTTP:                    !pcsconfig.Config.EnableHTTPS,

internal/pcscommand/upload.go

@@ -338,51 +338,39 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
 		stepUploadUpload:
 			task.step = StepUploadUpload
 			{
-				muer := uploader.NewMultiUploader(pcsupload.NewPCSUpload(pcs, task.savePath), rio.NewFileReaderAtLen64(task.localFileChecksum.GetFile()))
-				muer.SetParallel(opt.Parallel)
-
 				var blockSize int64
 				if opt.NotSplitFile {
 					blockSize = task.localFileChecksum.Length
 				} else {
 					blockSize = getBlockSize(task.localFileChecksum.Length)
 				}
-				muer.SetBlockSize(blockSize)
+
+				muer := uploader.NewMultiUploader(pcsupload.NewPCSUpload(pcs, task.savePath), rio.NewFileReaderAtLen64(task.localFileChecksum.GetFile()), &uploader.MultiUploaderConfig{
+					Parallel:  opt.Parallel,
+					BlockSize: blockSize,
+					MaxRate:   pcsconfig.Config.MaxUploadRate,
+				})
 
 				// 设置断点续传
 				if state != nil {
 					muer.SetInstanceState(state)
 				}
 
 				exitChan := make(chan struct{})
-				muer.OnExecute(func() {
-					statusChan := muer.GetStatusChan()
-					updateChan := muer.UpdateInstanceStateChan()
-					for {
-						select {
-						case <-exitChan:
-							return
-						case v, ok := <-statusChan:
-							if !ok {
-								return
-							}
-
-							if v.TotalSize() == 0 {
-								fmt.Printf("\r[%d] Prepareing upload...", task.ID)
-								continue
-							}
-
-							fmt.Printf("\r[%d] ↑ %s/%s %s/s in %s ............", task.ID,
-								converter.ConvertFileSize(v.Uploaded(), 2),
-								converter.ConvertFileSize(v.TotalSize(), 2),
-								converter.ConvertFileSize(v.SpeedsPerSecond(), 2),
-								v.TimeElapsed(),
-							)
-						case <-updateChan:
-							uploadDatabase.UpdateUploading(&task.localFileChecksum.LocalFileMeta, muer.InstanceState())
-							uploadDatabase.Save()
-						}
+				muer.OnUploadStatusEvent(func(status uploader.Status, updateChan <-chan struct{}) {
+					select {
+					case <-updateChan:
+						uploadDatabase.UpdateUploading(&task.localFileChecksum.LocalFileMeta, muer.InstanceState())
+						uploadDatabase.Save()
+					default:
 					}
+
+					fmt.Printf("\r[%d] ↑ %s/%s %s/s in %s ............", task.ID,
+						converter.ConvertFileSize(status.Uploaded(), 2),
+						converter.ConvertFileSize(status.TotalSize(), 2),
+						converter.ConvertFileSize(status.SpeedsPerSecond(), 2),
+						status.TimeElapsed(),
+					)
 				})
 				muer.OnSuccess(func() {
 					close(exitChan)

internal/pcsconfig/export.go

@@ -5,6 +5,7 @@ import (
 	"github.com/iikira/BaiduPCS-Go/baidupcs"
 	"github.com/iikira/BaiduPCS-Go/baidupcs/dlinkclient"
 	"github.com/iikira/BaiduPCS-Go/pcstable"
+	"github.com/iikira/BaiduPCS-Go/pcsutil/converter"
 	"github.com/iikira/BaiduPCS-Go/requester"
 	"github.com/olekukonko/tablewriter"
 	"os"
@@ -80,10 +81,12 @@ func (c *PCSConfig) PrintTable() {
 	tb.SetColumnAlignment([]int{tablewriter.ALIGN_DEFAULT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT})
 	tb.AppendBulk([][]string{
 		[]string{"appid", fmt.Sprint(c.AppID), "", "百度 PCS 应用ID"},
-		[]string{"cache_size", strconv.Itoa(c.CacheSize), "1024 ~ 262144", "下载缓存, 如果硬盘占用高或下载速度慢, 请尝试调大此值"},
+		[]string{"cache_size", converter.ConvertFileSize(int64(c.CacheSize), 2), "1KB ~ 256KB", "下载缓存, 如果硬盘占用高或下载速度慢, 请尝试调大此值"},
 		[]string{"max_parallel", strconv.Itoa(c.MaxParallel), "50 ~ 500", "下载最大并发量"},
 		[]string{"max_upload_parallel", strconv.Itoa(c.MaxUploadParallel), "1 ~ 100", "上传最大并发量"},
 		[]string{"max_download_load", strconv.Itoa(c.MaxDownloadLoad), "1 ~ 5", "同时进行下载文件的最大数量"},
+		[]string{"max_download_rate", showMaxRate(c.MaxDownloadRate), "", "限制最大下载速度, 0代表不限制"},
+		[]string{"max_upload_rate", showMaxRate(c.MaxUploadRate), "", "限制最大上传速度, 0代表不限制"},
 		[]string{"savedir", c.SaveDir, "", "下载文件的储存目录"},
 		[]string{"enable_https", fmt.Sprint(c.EnableHTTPS), "true", "启用 https"},
 		[]string{"user_agent", c.UserAgent, requester.DefaultUserAgent, "浏览器标识"},

internal/pcsconfig/maniper.go

@@ -1,6 +1,7 @@
 package pcsconfig
 
 import (
+	"github.com/iikira/BaiduPCS-Go/pcsutil/converter"
 	"github.com/iikira/BaiduPCS-Go/requester"
 	"strings"
 )
@@ -140,6 +141,36 @@ func (c *PCSConfig) SetAppID(appID int) {
 	}
 }
 
+// SetCacheSizeByStr 设置cache_size
+func (c *PCSConfig) SetCacheSizeByStr(sizeStr string) error {
+	size, err := converter.ParseFileSizeStr(sizeStr)
+	if err != nil {
+		return err
+	}
+	c.CacheSize = int(size)
+	return nil
+}
+
+// SetMaxDownloadRateByStr 设置 max_download_rate
+func (c *PCSConfig) SetMaxDownloadRateByStr(sizeStr string) error {
+	size, err := converter.ParseFileSizeStr(stripPerSecond(sizeStr))
+	if err != nil {
+		return err
+	}
+	c.MaxDownloadRate = size
+	return nil
+}
+
+// SetMaxUploadRateByStr 设置 max_upload_rate
+func (c *PCSConfig) SetMaxUploadRateByStr(sizeStr string) error {
+	size, err := converter.ParseFileSizeStr(stripPerSecond(sizeStr))
+	if err != nil {
+		return err
+	}
+	c.MaxUploadRate = size
+	return nil
+}
+
 // SetUserAgent 设置User-Agent
 func (c *PCSConfig) SetUserAgent(userAgent string) {
 	c.UserAgent = userAgent

internal/pcsconfig/pcsconfig.go

@@ -43,6 +43,9 @@ type PCSConfig struct {
 	MaxUploadParallel int `json:"max_upload_parallel"` // 最大上传并发量
 	MaxDownloadLoad   int `json:"max_download_load"`   // 同时进行下载文件的最大数量
 
+	MaxDownloadRate int64 `json:"max_download_rate"` // 限制最大下载速度
+	MaxUploadRate   int64 `json:"max_upload_rate"`   // 限制最大上传速度
+
 	UserAgent   string `json:"user_agent"`   // 浏览器标识
 	PCSUA       string `json:"pcs_ua"`       // PCS浏览器标识
 	PanUA       string `json:"pan_ua"`       // PAN浏览器标识

internal/pcsconfig/util.go

@@ -1,5 +1,10 @@
 package pcsconfig
 
+import (
+	"github.com/iikira/BaiduPCS-Go/pcsutil/converter"
+	"strings"
+)
+
 // AverageParallel 返回平均的下载最大并发量
 func AverageParallel(parallel, downloadLoad int) int {
 	if downloadLoad < 1 {
@@ -12,3 +17,18 @@ func AverageParallel(parallel, downloadLoad int) int {
 	}
 	return p
 }
+
+func stripPerSecond(sizeStr string) string {
+	i := strings.LastIndex(sizeStr, "/")
+	if i < 0 {
+		return sizeStr
+	}
+	return sizeStr[:i]
+}
+
+func showMaxRate(size int64) string {
+	if size <= 0 {
+		return "不限制"
+	}
+	return converter.ConvertFileSize(size, 2) + "/s"
+}