Skip to content

Commit

Permalink
update uploadId cache
Browse files Browse the repository at this point in the history
  • Loading branch information
carsonxu committed Feb 4, 2020
1 parent 14a2769 commit 47037d0
Show file tree
Hide file tree
Showing 8 changed files with 475 additions and 429 deletions.
475 changes: 250 additions & 225 deletions dist/cos-js-sdk-v5.js

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion dist/cos-js-sdk-v5.min.js

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
"description": "JavaScript SDK for [腾讯云对象存储](https://cloud.tencent.com/product/cos)",
"main": "index.js",
"scripts": {
"dev": "cross-env NODE_ENV=development webpack -w",
"server": "node server/sts.js",
"dev": "cross-env NODE_ENV=development webpack -w",
"build": "cross-env NODE_ENV=production webpack",
"cos-auth.min.js": "uglifyjs ./demo/common/cos-auth.js -o ./demo/common/cos-auth.min.js -c -m"
},
Expand Down
190 changes: 60 additions & 130 deletions src/advance.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
var session = require('./session');
var Async = require('./async');
var EventProxy = require('./event').EventProxy;
var util = require('./util');
Expand Down Expand Up @@ -41,13 +42,13 @@ function sliceUploadFile(params, callback) {
SliceList: UploadData.SliceList,
}, function (err, data) {
if (!self._isRunningTask(TaskId)) return;
delete uploadIdUsing[UploadData.UploadId];
session.removeUsing(UploadData.UploadId);
if (err) {
onProgress(null, true);
return ep.emit('error', err);
}
session.removeUploadId(UploadData.UploadId);
onProgress({loaded: FileSize, total: FileSize}, true);
removeUploadId.call(self, UploadData.UploadId);
ep.emit('upload_complete', data);
});
});
Expand All @@ -56,14 +57,9 @@ function sliceUploadFile(params, callback) {
ep.on('get_upload_data_finish', function (UploadData) {

// 处理 UploadId 缓存
var uuid = util.getFileUUID(Body, params.ChunkSize);
uuid && setUploadId.call(self, uuid, UploadData.UploadId); // 缓存 UploadId
uploadIdUsing[UploadData.UploadId] = true; // 标记 UploadId 为正在使用
TaskId && self.on('inner-kill-task', function (data) {
if (data.TaskId === TaskId && data.toState === 'canceled') {
delete uploadIdUsing[UploadData.UploadId]; // 去除 UploadId 正在使用的标记
}
});
var uuid = session.getFileId(Body, params.ChunkSize, Bucket, Key);
uuid && session.saveUploadId(uuid, UploadData.UploadId, self.options.UploadIdCacheLimit); // 缓存 UploadId
session.setUsing(UploadData.UploadId); // 标记 UploadId 为正在使用

// 获取 UploadId
onProgress(null, true); // 任务状态开始 uploading
Expand Down Expand Up @@ -157,75 +153,6 @@ function sliceUploadFile(params, callback) {

}


// 按照文件特征值,缓存 UploadId
var uploadIdCache;
var uploadIdUsing = {};
var uploadIdCacheKey = 'cos_sdk_upload_cache';
function initUploadId() {
var cacheLimit = this.options.UploadIdCacheLimit;
if (!uploadIdCache) {
if (cacheLimit) {
try {
uploadIdCache = JSON.parse(localStorage.getItem(uploadIdCacheKey)) || [];
} catch (e) {}
}
if (!uploadIdCache) {
uploadIdCache = [];
}
}
}
function setUploadId(uuid, UploadId, isDisabled) {
initUploadId.call(this);
for (var i = uploadIdCache.length - 1; i >= 0; i--) {
if (uploadIdCache[i][0] === uuid && uploadIdCache[i][1] === UploadId) {
uploadIdCache.splice(i, 1);
}
}
uploadIdCache.unshift([uuid, UploadId]);
var cacheLimit = this.options.UploadIdCacheLimit;
if (uploadIdCache.length > cacheLimit) {
uploadIdCache.splice(cacheLimit);
}
cacheLimit && setTimeout(function () {
try {
localStorage.setItem(uploadIdCacheKey, JSON.stringify(uploadIdCache));
} catch (e) {}
});
}
function removeUploadId(UploadId) {
initUploadId.call(this);
delete uploadIdUsing[UploadId];
for (var i = uploadIdCache.length - 1; i >= 0; i--) {
if (uploadIdCache[i][1] === UploadId) {
uploadIdCache.splice(i, 1)
}
}
var cacheLimit = this.options.UploadIdCacheLimit;
if (uploadIdCache.length > cacheLimit) {
uploadIdCache.splice(cacheLimit);
}
cacheLimit && setTimeout(function () {
try {
if (uploadIdCache.length) {
localStorage.setItem(uploadIdCacheKey, JSON.stringify(uploadIdCache));
} else {
localStorage.removeItem(uploadIdCacheKey);
}
} catch (e) {}
});
}
function getUploadId(uuid) {
initUploadId.call(this);
var CacheUploadIdList = [];
for (var i = 0; i < uploadIdCache.length; i++) {
if (uploadIdCache[i][0] === uuid) {
CacheUploadIdList.push(uploadIdCache[i][1]);
}
}
return CacheUploadIdList.length ? CacheUploadIdList : null;
}

// 获取上传任务的 UploadId
function getUploadIdAndPartList(params, callback) {
var TaskId = params.TaskId;
Expand Down Expand Up @@ -316,7 +243,7 @@ function getUploadIdAndPartList(params, callback) {
});

// 存在 UploadId
ep.on('upload_id_ready', function (UploadData) {
ep.on('upload_id_available', function (UploadData) {
// 转换成 map
var map = {};
var list = [];
Expand Down Expand Up @@ -364,18 +291,18 @@ function getUploadIdAndPartList(params, callback) {
if (!UploadId) {
return callback({Message: 'no upload id'});
}
ep.emit('upload_id_ready', {UploadId: UploadId, PartList: []});
ep.emit('upload_id_available', {UploadId: UploadId, PartList: []});
});
});

// 如果已存在 UploadId,找一个可以用的 UploadId
ep.on('has_upload_id', function (UploadIdList) {
ep.on('has_and_check_upload_id', function (UploadIdList) {
// 串行地,找一个内容一致的 UploadId
UploadIdList = UploadIdList.reverse();
Async.eachLimit(UploadIdList, 1, function (UploadId, asyncCallback) {
if (!self._isRunningTask(TaskId)) return;
// 如果正在上传,跳过
if (uploadIdUsing[UploadId]) {
if (session.using[UploadId]) {
asyncCallback(); // 检查下一个 UploadId
return;
}
Expand All @@ -388,7 +315,7 @@ function getUploadIdAndPartList(params, callback) {
}, function (err, PartListData) {
if (!self._isRunningTask(TaskId)) return;
if (err) {
removeUploadId.call(self, UploadId);
session.removeUsing(UploadId);
return ep.emit('error', err);
}
var PartList = PartListData.PartList;
Expand All @@ -414,7 +341,7 @@ function getUploadIdAndPartList(params, callback) {
if (!self._isRunningTask(TaskId)) return;
onHashProgress(null, true);
if (AvailableUploadData && AvailableUploadData.UploadId) {
ep.emit('upload_id_ready', AvailableUploadData);
ep.emit('upload_id_available', AvailableUploadData);
} else {
ep.emit('no_available_upload_id');
}
Expand All @@ -424,50 +351,52 @@ function getUploadIdAndPartList(params, callback) {
// 在本地缓存找可用的 UploadId
ep.on('seek_local_avail_upload_id', function (RemoteUploadIdList) {
// 在本地找可用的 UploadId
var uuid = util.getFileUUID(params.Body, params.ChunkSize), LocalUploadIdList;
if (uuid && (LocalUploadIdList = getUploadId.call(self, uuid))) {
var next = function (index) {
// 如果本地找不到可用 UploadId,再一个个遍历校验远端
if (index >= LocalUploadIdList.length) {
ep.emit('has_upload_id', RemoteUploadIdList);
return;
}
var UploadId = LocalUploadIdList[index];
// 如果不在远端 UploadId 列表里,跳过并删除
if (!util.isInArray(RemoteUploadIdList, UploadId)) {
removeUploadId.call(self, UploadId);
next(index + 1);
return;
}
// 如果正在上传,跳过
if (uploadIdUsing[UploadId]) {
var uuid = session.getFileId(params.Body, params.ChunkSize, Bucket, Key);
var LocalUploadIdList = session.getUploadIdList(uuid);
if (!uuid || !LocalUploadIdList) {
ep.emit('has_and_check_upload_id', RemoteUploadIdList);
return;
}
var next = function (index) {
// 如果本地找不到可用 UploadId,再一个个遍历校验远端
if (index >= LocalUploadIdList.length) {
ep.emit('has_and_check_upload_id', RemoteUploadIdList);
return;
}
var UploadId = LocalUploadIdList[index];
// 如果不在远端 UploadId 列表里,跳过并删除
if (!util.isInArray(RemoteUploadIdList, UploadId)) {
session.removeUploadId(UploadId);
next(index + 1);
return;
}
// 如果正在上传,跳过
if (session.using[UploadId]) {
next(index + 1);
return;
}
// 判断 UploadId 是否存在线上
wholeMultipartListPart.call(self, {
Bucket: Bucket,
Region: Region,
Key: Key,
UploadId: UploadId,
}, function (err, PartListData) {
if (!self._isRunningTask(TaskId)) return;
if (err) {
// 如果 UploadId 获取会出错,跳过并删除
session.removeUploadId(UploadId);
next(index + 1);
return;
} else {
// 找到可用 UploadId
ep.emit('upload_id_available', {
UploadId: UploadId,
PartList: PartListData.PartList,
});
}
// 判断 UploadId 是否存在线上
wholeMultipartListPart.call(self, {
Bucket: Bucket,
Region: Region,
Key: Key,
UploadId: UploadId,
}, function (err, PartListData) {
if (!self._isRunningTask(TaskId)) return;
if (err) {
removeUploadId.call(self, UploadId);
next(index + 1);
} else {
// 找到可用 UploadId
ep.emit('upload_id_ready', {
UploadId: UploadId,
PartList: PartListData.PartList,
});
}
});
};
next(0);
} else {
ep.emit('has_upload_id', RemoteUploadIdList);
}
});
};
next(0);
});

// 获取线上 UploadId 列表
Expand All @@ -491,10 +420,11 @@ function getUploadIdAndPartList(params, callback) {
if (RemoteUploadIdList.length) {
ep.emit('seek_local_avail_upload_id', RemoteUploadIdList);
} else {
var uuid = util.getFileUUID(params.Body, params.ChunkSize), LocalUploadIdList;
if (uuid && (LocalUploadIdList = getUploadId.call(self, uuid))) {
// 远端没有 UploadId,清理缓存的 UploadId
var uuid = session.getFileId(params.Body, params.ChunkSize, Bucket, Key), LocalUploadIdList;
if (uuid && (LocalUploadIdList = session.getUploadIdList(uuid))) {
util.each(LocalUploadIdList, function (UploadId) {
removeUploadId.call(self, UploadId);
session.removeUploadId(UploadId);
});
}
ep.emit('no_available_upload_id');
Expand Down
3 changes: 1 addition & 2 deletions src/base.js
Original file line number Diff line number Diff line change
Expand Up @@ -1611,9 +1611,8 @@ function restoreObject(params, callback) {
function multipartInit(params, callback) {

var self = this;
var headers = params.Headers;

// 特殊处理 Cache-Control
var headers = params.Headers;
!headers['Cache-Control'] && (headers['Cache-Control'] = '');
util.getBodyMd5(params.Body && (params.UploadAddMetaMd5 || self.options.UploadAddMetaMd5), params.Body, function (md5) {
if (md5) params.Headers['x-cos-meta-md5'] = md5;
Expand Down
89 changes: 89 additions & 0 deletions src/session.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
var util = require('./util');

// 按照文件特征值,缓存 UploadId
var cacheKey = 'cos_sdk_upload_cache';
var expires = 30 * 24 * 3600;
var cache;
var timer;

var init = function () {
if (cache) return;
cache = JSON.parse(localStorage.getItem(cacheKey) || '[]') || [];
// 清理太老旧的数据
var changed = false;
var now = Math.round(Date.now() / 1000);
for (var i = cache.length - 1; i >= 0; i--) {
var mtime = cache[i][2];
if (!mtime || mtime + expires < now) {
cache.splice(i, 1);
changed = true;
}
}
changed && localStorage.setItem(cacheKey, JSON.stringify(cache));
};

// 把缓存存到本地
var save = function () {
if (timer) return;
timer = setTimeout(function () {
localStorage.setItem(cacheKey, JSON.stringify(cache));
timer = null;
}, 400);
};

var mod = {
using: {},
// 标记 UploadId 正在使用
setUsing: function (uuid) {
mod.using[uuid] = true;
},
// 标记 UploadId 已经没在使用
removeUsing: function (uuid) {
delete mod.using[uuid];
},
// 用上传参数生成哈希值
getFileId: function (file, ChunkSize, Bucket, Key) {
if (file.name && file.size && file.lastModifiedDate && ChunkSize) {
return util.md5([file.name, file.size, file.lastModifiedDate, ChunkSize, Bucket, Key].join('::'));
} else {
return null;
}
},
// 获取文件对应的 UploadId 列表
getUploadIdList: function (uuid) {
if (!uuid) return null;
init();
var list = [];
for (var i = 0; i < cache.length; i++) {
if (cache[i][0] === uuid)
list.push(cache[i][1]);
}
return list.length ? list : null;
},
// 缓存 UploadId
saveUploadId: function (uuid, UploadId, limit) {
init();
if (!uuid) return;
// 清理没用的 UploadId
for (var i = cache.length - 1; i >= 0; i--) {
var item = cache[i];
if (item[0] === uuid && item[1] === UploadId) {
cache.splice(i, 1);
}
}
cache.unshift([uuid, UploadId, Math.round(Date.now() / 1000)]);
if (cache.length > limit) cache.splice(limit);
save();
},
// UploadId 已用完,移除掉
removeUploadId: function (UploadId) {
init();
delete mod.using[UploadId];
for (var i = cache.length - 1; i >= 0; i--) {
if (cache[i][1] === UploadId) cache.splice(i, 1)
}
save();
},
};

module.exports = mod;
Loading

0 comments on commit 47037d0

Please sign in to comment.