Skip to content

Commit

Permalink
cache UploadId
Browse files Browse the repository at this point in the history
  • Loading branch information
@carsonxu
carsonxu committed Apr 19, 2018
1 parent 11b942a commit 5495244
Show file tree
Hide file tree
Showing 7 changed files with 299 additions and 84 deletions.
1 change: 0 additions & 1 deletion demo/demo.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@ var getAuthorization = function (options, callback) {
};

var cos = new COS({
UploadIdCacheLimit: 3,
getAuthorization: getAuthorization,
});
var TaskId;
Expand Down
190 changes: 149 additions & 41 deletions dist/cos-js-sdk-v5.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,16 @@ function extend(target, source) {
function isArray(arr) {
return arr instanceof Array;
}
function isInArray(arr, item) {
var flag = false;
for (var i = 0; i < arr.length; i++) {
if (item === arr[i]) {
flag = true;
break;
}
}
return flag;
}
function each(obj, fn) {
for (var i in obj) {
if (obj.hasOwnProperty(i)) {
Expand Down Expand Up @@ -464,6 +474,7 @@ var util = {
binaryBase64: binaryBase64,
extend: extend,
isArray: isArray,
isInArray: isInArray,
each: each,
map: map,
filter: filter,
Expand Down Expand Up @@ -1461,7 +1472,7 @@ var initTask = function (cos) {
return;
}
task.state = switchToState;
cos.emit('inner-kill-task', { TaskId: id });
cos.emit('inner-kill-task', { TaskId: id, toState: switchToState });
emitListUpdate();
if (running) {
uploadingFileCount--;
Expand Down Expand Up @@ -8035,28 +8046,36 @@ function sliceUploadFile(params, callback) {
});

// 上传分块完成，开始 uploadSliceComplete 操作
ep.on('upload_slice_complete', function (data) {
ep.on('upload_slice_complete', function (UploadData) {
uploadSliceComplete.call(self, {
Bucket: Bucket,
Region: Region,
Key: Key,
UploadId: data.UploadId,
SliceList: data.SliceList
UploadId: UploadData.UploadId,
SliceList: UploadData.SliceList
}, function (err, data) {
if (!self._isRunningTask(TaskId)) return;
delete uploadIdUsing[UploadData.UploadId];
if (err) {
return ep.emit('error', err);
}
removeUploadId.call(self, UploadData.UploadId);
ep.emit('upload_complete', data);
});
});

// 获取 UploadId 完成，开始上传每个分片
ep.on('get_upload_data_finish', function (UploadData) {
if (UploadData.UploadId) {
var uuid = getFileUuid(Body, params.ChunkSize);
uuid && setUploadId.call(self, uuid, UploadData.UploadId);
}

// 处理 UploadId 缓存
var uuid = getFileUuid(Body, params.ChunkSize);
uuid && setUploadId.call(self, uuid, UploadData.UploadId); // 缓存 UploadId
uploadIdUsing[UploadData.UploadId] = true; // 标记 UploadId 为正在使用
TaskId && self.on('inner-kill-task', function (data) {
if (data.TaskId === TaskId && data.toState === 'canceled') {
delete uploadIdUsing[UploadData.UploadId]; // 去除 UploadId 正在使用的标记
}
});

// 获取 UploadId
uploadSliceList.call(self, {
Expand Down Expand Up @@ -8126,6 +8145,7 @@ function sliceUploadFile(params, callback) {
params.ChunkSize = params.SliceSize = ChunkSize = Math.max(ChunkSize, AutoChunkSize);
})();

// 开始上传
if (FileSize === 0) {
params.Body = '';
self.putObject(params, callback);
Expand All @@ -8135,14 +8155,26 @@ function sliceUploadFile(params, callback) {
}

// 按照文件特征值，缓存 UploadId
var uploadIdCache;
var uploadIdUsing = {};
var uploadIdCacheKey = 'cos_sdk_upload_cache';
var uploadIdCache = [];
try {
uploadIdCache = JSON.parse(localStorage.getItem(uploadIdCacheKey)) || [];
} catch (e) {}
function setUploadId(uuid, UploadId) {
function initUploadId() {
var cacheLimit = this.options.UploadIdCacheLimit;
if (!uploadIdCache) {
if (cacheLimit) {
try {
uploadIdCache = JSON.parse(localStorage.getItem(uploadIdCacheKey)) || [];
} catch (e) {}
}
if (!uploadIdCache) {
uploadIdCache = [];
}
}
}
function setUploadId(uuid, UploadId, isDisabled) {
initUploadId.call(this);
for (var i = uploadIdCache.length - 1; i >= 0; i--) {
if (uploadIdCache[i][0] === uuid) {
if (uploadIdCache[i][0] === uuid && uploadIdCache[i][1] === UploadId) {
uploadIdCache.splice(i, 1);
}
}
Expand All @@ -8157,15 +8189,37 @@ function setUploadId(uuid, UploadId) {
} catch (e) {}
});
}
function removeUploadId(UploadId) {
initUploadId.call(this);
delete uploadIdUsing[UploadId];
for (var i = uploadIdCache.length - 1; i >= 0; i--) {
if (uploadIdCache[i][1] === UploadId) {
uploadIdCache.splice(i, 1);
}
}
var cacheLimit = this.options.UploadIdCacheLimit;
if (uploadIdCache.length > cacheLimit) {
uploadIdCache.splice(cacheLimit);
}
cacheLimit && setTimeout(function () {
try {
if (uploadIdCache.length) {
localStorage.setItem(uploadIdCacheKey, JSON.stringify(uploadIdCache));
} else {
localStorage.removeItem(uploadIdCacheKey);
}
} catch (e) {}
});
}
function getUploadId(uuid) {
var UploadId;
initUploadId.call(this);
var CacheUploadIdList = [];
for (var i = 0; i < uploadIdCache.length; i++) {
if (uploadIdCache[i][0] === uuid) {
UploadId = uploadIdCache[i][1];
break;
CacheUploadIdList.push(uploadIdCache[i][1]);
}
}
return UploadId;
return CacheUploadIdList.length ? CacheUploadIdList : null;
}
function getFileUuid(file, ChunkSize) {
// 如果信息不完整，不获取
Expand Down Expand Up @@ -8318,14 +8372,23 @@ function getUploadIdAndPartList(params, callback) {
UploadIdList = UploadIdList.reverse();
Async.eachLimit(UploadIdList, 1, function (UploadId, asyncCallback) {
if (!self._isRunningTask(TaskId)) return;
// 如果正在上传，跳过
if (uploadIdUsing[UploadId]) {
asyncCallback(); // 检查下一个 UploadId
return;
}
// 判断 UploadId 是否可用
wholeMultipartListPart.call(self, {
Bucket: Bucket,
Region: Region,
Key: Key,
UploadId: UploadId
}, function (err, PartListData) {
if (!self._isRunningTask(TaskId)) return;
if (err) return ep.emit('error', err);
if (err) {
removeUploadId.call(self, UploadId);
return ep.emit('error', err);
}
var PartList = PartListData.PartList;
PartList.forEach(function (item) {
item.PartNumber *= 1;
Expand Down Expand Up @@ -8356,24 +8419,58 @@ function getUploadIdAndPartList(params, callback) {
});
});

// 获取缓存的 UploadId
var uuid = getFileUuid(params.Body, params.ChunkSize),
UploadId;
if (uuid && (UploadId = getUploadId(uuid))) {
wholeMultipartListPart.call(self, {
Bucket: Bucket,
Region: Region,
Key: Key,
UploadId: UploadId
}, function (err, PartListData) {
if (!self._isRunningTask(TaskId)) return;
if (err) return ep.emit('error', err);
ep.emit('upload_id_ready', {
UploadId: UploadId,
PartList: PartListData.PartList
});
});
} else {
// 在本地缓存找可用的 UploadId
ep.on('seek_local_avail_upload_id', function (RemoteUploadIdList) {
// 在本地找可用的 UploadId
var uuid = getFileUuid(params.Body, params.ChunkSize),
LocalUploadIdList;
if (uuid && (LocalUploadIdList = getUploadId.call(self, uuid))) {
var next = function (index) {
// 如果找不到，到线上列出 UploadId
if (index >= LocalUploadIdList.length) {
ep.emit('has_upload_id', RemoteUploadIdList);
return;
}
var UploadId = LocalUploadIdList[index];
// 如果不在远端 UploadId 列表里，跳过并删除
if (!util.isInArray(RemoteUploadIdList, UploadId)) {
removeUploadId.call(self, UploadId);
next(index + 1);
return;
}
// 如果正在上传，跳过
if (uploadIdUsing[UploadId]) {
next(index + 1);
return;
}
// 判断 UploadId 是否存在线上
wholeMultipartListPart.call(self, {
Bucket: Bucket,
Region: Region,
Key: Key,
UploadId: UploadId
}, function (err, PartListData) {
if (!self._isRunningTask(TaskId)) return;
if (err) {
removeUploadId.call(self, UploadId);
next(index + 1);
} else {
// 找到可用 UploadId
ep.emit('upload_id_ready', {
UploadId: UploadId,
PartList: PartListData.PartList
});
}
});
};
next(0);
} else {
ep.emit('has_upload_id', RemoteUploadIdList);
}
});

// 获取线上 UploadId 列表
ep.on('get_remote_upload_id_list', function (RemoteUploadIdList) {
// 获取符合条件的 UploadId 列表，因为同一个文件可以有多个上传任务。
wholeMultipartList.call(self, {
Bucket: Bucket,
Expand All @@ -8384,18 +8481,29 @@ function getUploadIdAndPartList(params, callback) {
if (err) {
return ep.emit('error', err);
}
var UploadIdList = data.UploadList.filter(function (item) {
// 整理远端 UploadId 列表
var RemoteUploadIdList = data.UploadList.filter(function (item) {
return item.Key === Key && (!StorageClass || item.StorageClass.toUpperCase() === StorageClass.toUpperCase());
}).reverse().map(function (item) {
return item.UploadId || item.UploadID;
});
if (UploadIdList.length) {
ep.emit('has_upload_id', UploadIdList);
if (RemoteUploadIdList.length) {
ep.emit('seek_local_avail_upload_id', RemoteUploadIdList);
} else {
var uuid = getFileUuid(params.Body, params.ChunkSize),
LocalUploadIdList;
if (uuid && (LocalUploadIdList = getUploadId.call(self, uuid))) {
util.each(LocalUploadIdList, function (UploadId) {
removeUploadId.call(self, UploadId);
});
}
ep.emit('no_available_upload_id');
}
});
}
});

// 开始找可用 UploadId
ep.emit('get_remote_upload_id_list');
}

// 获取符合条件的全部上传任务 (条件包括 Bucket, Region, Prefix)
Expand Down
2 changes: 1 addition & 1 deletion dist/cos-js-sdk-v5.min.js
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion server/sts.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ function getTempKeys() {
array(
'action'=> array(
// // 这里可以从临时密钥的权限上控制前端允许的操作
// 'name/cos:*', // 这样写可以包含下面所有权限
'name/cos:*', // 这样写可以包含下面所有权限

// // 列出所有允许的操作
// // ACL 读写
Expand Down
Loading

0 comments on commit 5495244

Please sign in to comment.