v3.8.6

README.md

@@ -100,6 +100,12 @@ iikira/BaiduPCS-Go was largely inspired by [GangZhuo/BaiduPCS](https://github.co
 [离线下载](#离线下载), 支持http/https/ftp/电驴/磁力链协议.
 
 # 版本更新
+**2022.2.14** v3.8.6:
+- fix #160 #173, 修复上传出现空文件的bug
+- fix #165, 支持自带提取码的转存链接
+- fix #175, upload增加-policy=rsync策略, 配合--norapid使用, 只跳过大小未发生改变的文件
+- 鉴于 #172, 建议下载线程数最大不超过12
+
 **2022.1.1** v3.8.5:
 - 2022新年好, 本次更新增加较多特性, 欢迎测试
 - fix #146, 提前fail和skip上传策略中重名文件的检测环节

baidupcs/prepare.go

@@ -288,7 +288,7 @@ func (pcs *BaiduPCS) prepareRapidUpload(targetPath, contentMD5, sliceMD5, crc32
 // PrepareRapidUpload 秒传文件, 只返回服务器响应数据和错误信息
 func (pcs *BaiduPCS) PrepareRapidUpload(targetPath, contentMD5, sliceMD5, crc32 string, length int64) (dataReadCloser io.ReadCloser, pcsError pcserror.Error) {
 	pcs.lazyInit()
-	pcsError = pcs.checkIsdir(OperationRapidUpload, targetPath)
+	pcsError = pcs.checkIsdir(OperationRapidUpload, targetPath, "", length)
 	if pcsError != nil {
 		return nil, pcsError
 	}
@@ -299,7 +299,7 @@ func (pcs *BaiduPCS) PrepareRapidUpload(targetPath, contentMD5, sliceMD5, crc32
 // PrepareRapidUploadV2 秒传文件, 新接口
 func (pcs *BaiduPCS) PrepareRapidUploadV2(targetPath, contentMD5 string, length int64) (dataReadCloser io.ReadCloser, pcsError pcserror.Error) {
 	pcs.lazyInit()
-	pcsError = pcs.checkIsdir(OperationRapidUpload, targetPath)
+	pcsError = pcs.checkIsdir(OperationRapidUpload, targetPath, "", length)
 	if pcsError != nil {
 		return nil, pcsError
 	}
@@ -399,16 +399,16 @@ func (pcs *BaiduPCS) PrepareLocatePanAPIDownload(fidList ...int64) (dataReadClos
 }
 
 // PrepareUpload 上传单个文件, 只返回服务器响应数据和错误信息（分片上传中的预上传部分）
-func (pcs *BaiduPCS) PrepareUpload(policy string, targetPath string, uploadFunc UploadFunc) (dataReadCloser io.ReadCloser, pcsError pcserror.Error) {
+func (pcs *BaiduPCS) PrepareUpload(fileSize int64, policy string, targetPath string, uploadFunc UploadFunc) (dataReadCloser io.ReadCloser, pcsError pcserror.Error) {
 	pcs.lazyInit()
-	pcsError = pcs.checkIsdir(OperationUpload, targetPath)
+	pcsError = pcs.checkIsdir(OperationUpload, targetPath, policy, fileSize)
 	if pcsError != nil {
 		return nil, pcsError
 	}
 
 	pcsURL := pcs.generatePCSURL("file", "upload", map[string]string{
 		"path":  targetPath,
-		"ondup": policy,
+		"ondup": strings.Replace(policy, "rsync", "overwrite", -1),
 	})
 	baiduPCSVerbose.Infof("%s URL: %s\n", OperationUpload, pcsURL)
 
@@ -462,7 +462,7 @@ func (pcs *BaiduPCS) PrepareUploadCreateSuperFile(policy string, checkDir bool,
 
 	if checkDir {
 		// 检查是否为目录
-		pcsError = pcs.checkIsdir(OperationUploadCreateSuperFile, targetPath)
+		pcsError = pcs.checkIsdir(OperationUploadCreateSuperFile, targetPath, "", 0)
 		if pcsError != nil {
 			return nil, pcsError
 		}
@@ -479,7 +479,7 @@ func (pcs *BaiduPCS) PrepareUploadCreateSuperFile(policy string, checkDir bool,
 
 	pcsURL := pcs.generatePCSURL("file", "createsuperfile", map[string]string{
 		"path":  targetPath,
-		"ondup": policy,
+		"ondup": strings.Replace(policy, "rsync", "overwrite", -1),
 	})
 	baiduPCSVerbose.Infof("%s URL: %s\n", OperationUploadCreateSuperFile, pcsURL)
 

baidupcs/upload.go

@@ -201,8 +201,8 @@ func (pcs *BaiduPCS) RapidUploadNoCheckDir(targetPath, contentMD5, sliceMD5, crc
 }
 
 // Upload 上传单个文件
-func (pcs *BaiduPCS) Upload(policy, targetPath string, uploadFunc UploadFunc) (pcsError pcserror.Error, newpath string) {
-	dataReadCloser, pcsError := pcs.PrepareUpload(policy, targetPath, uploadFunc)
+func (pcs *BaiduPCS) Upload(fileSize int64, policy, targetPath string, uploadFunc UploadFunc) (pcsError pcserror.Error, newpath string) {
+	dataReadCloser, pcsError := pcs.PrepareUpload(fileSize, policy, targetPath, uploadFunc)
 	if pcsError != nil {
 		return pcsError, ""
 	}

baidupcs/util.go

@@ -14,23 +14,23 @@ import (
 )
 
 // Isdir 检查路径在网盘中是否为目录
-func (pcs *BaiduPCS) Isdir(pcspath string) (isdir bool, pcsError pcserror.Error) {
+func (pcs *BaiduPCS) Isdir(pcspath string) (fileSize int64, isdir bool, pcsError pcserror.Error) {
 	if path.Clean(pcspath) == PathSeparator {
-		return true, nil
+		return 0, true, nil
 	}
 
 	f, pcsError := pcs.FilesDirectoriesMeta(pcspath)
 	if pcsError != nil {
-		return false, pcsError
+		return 0, false, pcsError
 	}
 
-	return f.Isdir, nil
+	return f.Size, f.Isdir, nil
 }
 
-func (pcs *BaiduPCS) checkIsdir(op string, targetPath string) pcserror.Error {
+func (pcs *BaiduPCS) checkIsdir(op string, targetPath string, policy string, fileSize int64) pcserror.Error {
 	// 检测文件是否存在于网盘路径
 	// 很重要, 如果文件存在会直接覆盖!!! 即使是根目录!
-	isdir, pcsError := pcs.Isdir(targetPath)
+	onlineSize, isdir, pcsError := pcs.Isdir(targetPath)
 	if pcsError != nil {
 		// 忽略远程服务端返回的错误
 		if pcsError.GetErrType() != pcserror.ErrTypeRemoteError {
@@ -44,6 +44,30 @@ func (pcs *BaiduPCS) checkIsdir(op string, targetPath string) pcserror.Error {
 		errInfo.Err = errors.New("保存路径不可以覆盖目录")
 		return errInfo
 	}
+	// 如果存在文件, 则根据upload策略选择返回的错误码
+	if pcsError == nil {
+		switch policy {
+		case "fail":
+			errInfo.ErrCode = 114514
+			errInfo.ErrType = pcserror.ErrTypeRemoteError
+			errInfo.ErrMsg = "目标位置存在同名文件"
+			return errInfo
+		case "skip":
+			errInfo.ErrCode = 114514
+			errInfo.ErrMsg = "目标位置存在同名文件"
+			errInfo.ErrType = pcserror.ErrTypeRemoteError
+			return errInfo
+		case "rsync":
+			if onlineSize == fileSize {
+				errInfo.ErrCode = 1919810
+				errInfo.ErrMsg = "目标位置文件大小与源文件一致"
+				errInfo.ErrType = pcserror.ErrTypeRemoteError
+				return errInfo
+			}
+		default:
+			return nil
+		}
+	}
 	return nil
 }
 

internal/pcscommand/transfer.go

@@ -22,6 +22,10 @@ func RunShareTransfer(params []string, opt *baidupcs.TransferOption) {
 			return
 		}
 		extracode = "none"
+		if strings.Contains(link, "?pwd=") {
+			extracode = strings.Split(link, "?pwd=")[1]
+			link = strings.Split(link, "?pwd=")[0]
+		}
 	} else if len(params) == 2 {
 		link = params[0]
 		extracode = params[1]

internal/pcscommand/upload.go

@@ -97,7 +97,7 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
 		opt.Load = pcsconfig.Config.MaxUploadLoad
 	}
 
-	if opt.Policy!="fail" && opt.Policy!="newcopy" && opt.Policy!="overwrite" && opt.Policy!="skip" {
+	if opt.Policy!="fail" && opt.Policy!="newcopy" && opt.Policy!="overwrite" && opt.Policy!="skip" && opt.Policy!="rsync" {
 		opt.Policy = pcsconfig.Config.UPolicy
 	}
 

internal/pcsconfig/export.go

@@ -68,7 +68,7 @@ func (c *PCSConfig) PrintTable() {
 	tb.AppendBulk([][]string{
 		[]string{"appid", fmt.Sprint(c.AppID), "", "百度 PCS 应用ID"},
 		[]string{"cache_size", converter.ConvertFileSize(int64(c.CacheSize), 2), "1KB ~ 256KB", "下载缓存, 如果硬盘占用高或下载速度慢, 请尝试调大此值"},
-		[]string{"max_parallel", strconv.Itoa(c.MaxParallel), "1 ~ 20", "下载总最大并发量"},
+		[]string{"max_parallel", strconv.Itoa(c.MaxParallel), "1 ~ 14", "下载总最大并发量"},
 		[]string{"max_upload_parallel", strconv.Itoa(c.MaxUploadParallel), "1 ~ 100", "上传单文件最大并发量"},
 		[]string{"max_download_load", strconv.Itoa(c.MaxDownloadLoad), "1 ~ 5", "同时进行下载文件的最大数量"},
 		[]string{"max_download_rate", showMaxRate(c.MaxDownloadRate), "", "限制最大下载速度, 0代表不限制"},
@@ -79,7 +79,7 @@ func (c *PCSConfig) PrintTable() {
 		[]string{"force_login_username", fmt.Sprint(c.ForceLogin), "留空", "强制登录指定用户名, 适用于tieba用户信息接口不可用的情况, 如登录正常请留空"},
 		[]string{"no_check", fmt.Sprint(c.NoCheck), "true", "关闭下载文件md5校验"},
 		[]string{"ignore_illegal", fmt.Sprint(c.IgnoreIllegal), "false", "关闭上传文件的文件名非法字符检查"},
-		[]string{"upload_policy", fmt.Sprint(c.UPolicy), "fail", "上传遇到重名文件时的处理策略, fail(默认，直接返回失败)、newcopy(重命名文件)、overwrite、skip"},
+		[]string{"upload_policy", fmt.Sprint(c.UPolicy), "fail", "上传遇到重名文件时的处理策略, fail(默认，直接返回失败)、newcopy(重命名文件)、overwrite(覆盖)、skip(跳过)、rsync(仅跳过大小未变化的文件)"},
 		[]string{"user_agent", c.UserAgent, requester.DefaultUserAgent, "浏览器标识"},
 		[]string{"pcs_ua", c.PCSUA, "", "PCS 浏览器标识"},
 		[]string{"pcs_addr", c.PCSAddr, "pcs.baidu.com", "PCS 服务器地址"},

internal/pcsconfig/pcsconfig.go

@@ -319,7 +319,7 @@ func (c *PCSConfig) fix() {
 	if c.MaxUploadLoad < 1 {
 		c.MaxUploadLoad = 1
 	}
-	if c.UPolicy != "fail" && c.UPolicy != "newcopy" && c.UPolicy != "overwrite" && c.UPolicy != "skip" {
+	if c.UPolicy != "fail" && c.UPolicy != "newcopy" && c.UPolicy != "overwrite" && c.UPolicy != "skip" && c.UPolicy != "rsync" {
 		c.UPolicy = "fail"
 	}
 }

internal/pcsfunctions/pcsupload/upload.go

@@ -48,12 +48,8 @@ func (pu *PCSUpload) lazyInit() {
 }
 
 // Precreate 检查网盘的目标路径是否已存在同名文件
-func (pu *PCSUpload) Precreate(policy string) (err error) {
-	err = pu.CreateSuperFile("fail")
-	if policy != "fail" && policy != "skip" {
-		return nil
-	}
-	return
+func (pu *PCSUpload) Precreate() (err error) {
+	return nil
 }
 
 func (pu *PCSUpload) TmpFile(ctx context.Context, partseq int, partOffset int64, r rio.ReaderLen64) (checksum string, uperr error) {
@@ -102,12 +98,12 @@ func (pu *PCSUpload) TmpFile(ctx context.Context, partseq int, partOffset int64,
 	return checksum, pcsError
 }
 
-func (pu *PCSUpload) CreateSuperFile(policy string, checksumList ...string) (err error) {
+func (pu *PCSUpload) CreateSuperFile(fileSize int64, policy string, checksumList ...string) (err error) {
 	pu.lazyInit()
 	//newpath := ""
 	// 先在网盘目标位置, 上传一个空文件
 	// 防止出现file does not exist
-	pcsError, newpath := pu.pcs.Upload(policy, pu.targetPath, func(uploadURL string, jar http.CookieJar) (resp *http.Response, err error) {
+	pcsError, newpath := pu.pcs.Upload(fileSize, policy, pu.targetPath, func(uploadURL string, jar http.CookieJar) (resp *http.Response, err error) {
 		mr := multipartreader.NewMultipartReader()
 		mr.AddFormFile("file", "file", &EmptyReaderLen64{})
 		mr.CloseMultipart()

internal/pcsfunctions/pcsupload/upload_task_unit.go

@@ -249,6 +249,24 @@ func (utu *UploadTaskUnit) upload() (result *taskframework.TaskUnitRunResult) {
 		case pcserror.ErrTypeRemoteError:
 			// 远程百度服务器的错误
 			switch pcsError.GetRemoteErrCode() {
+			case 114514:
+				// 自定义错误码, 仅在fail和skip策略下出现
+				result.ResultMessage = StrUploadFailed
+				result.Err = pcsError
+				if utu.Policy == "skip" {
+					result.Extra = "skip"
+					result.Err = nil
+					result.ResultMessage = fmt.Sprintf("%s 目标已存在, 跳过", utu.SavePath)
+				}
+				result.NeedRetry = false
+				return
+			case 1919810:
+				// 自定义错误码, 仅在rsync策略下出现
+				result.Extra = "skip"
+				result.Err = nil
+				result.ResultMessage = fmt.Sprintf("%s 目标大小未发生改变, 跳过", utu.SavePath)
+				result.NeedRetry = false
+				return
 			case 31363:
 				// block miss in superfile2, 上传状态过期
 				// 需要重试的

main.go

@@ -55,7 +55,7 @@ const (
 
 var (
 	// Version 版本号
-	Version = "v3.8.5-devel"
+	Version = "v3.8.6-devel"
 
 	historyFilePath = filepath.Join(pcsconfig.GetConfigDir(), "pcs_command_history.txt")
 	reloadFn        = func(c *cli.Context) error {

requester/uploader/multiuploader.go

@@ -14,9 +14,9 @@ import (
 type (
 	// MultiUpload 支持多线程的上传, 可用于断点续传
 	MultiUpload interface {
-		Precreate(policy string) (perr error)
+		Precreate() (err error)
 		TmpFile(ctx context.Context, partseq int, partOffset int64, readerlen64 rio.ReaderLen64) (checksum string, terr error)
-		CreateSuperFile(policy string, checksumList ...string) (cerr error)
+		CreateSuperFile(fileSize int64, policy string, checksumList ...string) (cerr error)
 	}
 
 	// MultiUploader 多线程上传
@@ -104,7 +104,6 @@ func (muer *MultiUploader) check() {
 func (muer *MultiUploader) Execute() {
 	muer.check()
 	muer.lazyInit()
-
 	// 初始化限速
 	if muer.config.MaxRate > 0 {
 		muer.rateLimit = speeds.NewRateLimit(muer.config.MaxRate)

requester/uploader/multiworker.go

@@ -37,11 +37,6 @@ func (werl *workerList) Readed() int64 {
 }
 
 func (muer *MultiUploader) upload() (uperr error) {
-	err := muer.multiUpload.Precreate(muer.config.Policy)
-	if err != nil {
-		return err
-	}
-
 	var (
 		uploadDeque = lane.NewDeque()
 	)
@@ -125,7 +120,7 @@ func (muer *MultiUploader) upload() (uperr error) {
 	default:
 	}
 
-	cerr := muer.multiUpload.CreateSuperFile(muer.config.Policy, muer.workers.CheckSumList()...)
+	cerr := muer.multiUpload.CreateSuperFile(muer.file.Len(), muer.config.Policy, muer.workers.CheckSumList()...)
 	if cerr != nil {
 		return cerr
 	}