Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

storage: update code that now uses streamrouter #770

Merged
merged 1 commit into from
Jul 31, 2015
Merged

storage: update code that now uses streamrouter #770

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 42 additions & 79 deletions lib/storage/bucket.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,61 +382,43 @@ Bucket.prototype.deleteFiles = function(query, callback) {

query = query || {};

var self = this;

var MAX_PARALLEL_LIMIT = 10;
var errors = [];

// Start deleting files, iteratively fetching more as necessary.
deleteFiles(query, function(err) {
if (err || errors.length > 0) {
callback(err || errors);
this.getFiles(query, function(err, files) {
if (err) {
callback(err);
return;
}

callback(null);
});

function deleteFiles(query, callback) {
self.getFiles(query, function(err, files, nextQuery) {
if (err) {
callback(err);
return;
}

// Iterate through each file and attempt to delete it.
async.eachLimit(files, MAX_PARALLEL_LIMIT, deleteFile, function(err) {
function deleteFile(file, callback) {
file.delete(function(err) {
if (err) {
callback(err);
return;
}
if (query.force) {
errors.push(err);
callback();
return;
}

if (nextQuery) {
deleteFiles(nextQuery, callback);
callback(err);
return;
}

callback();
});
});
}

function deleteFile(file, callback) {
file.delete(function(err) {
if (err) {
if (query.force) {
errors.push(err);
callback();
return;
}
}

callback(err);
// Iterate through each file and attempt to delete it.
async.eachLimit(files, MAX_PARALLEL_LIMIT, deleteFile, function(err) {
if (err || errors.length > 0) {
callback(err || errors);
return;
}

callback();
});
}
});

};

/**
Expand Down Expand Up @@ -1033,69 +1015,50 @@ Bucket.prototype.upload = function(localPath, options, callback) {
* @param {function} callback - The callback function.
*/
Bucket.prototype.makeAllFilesPublicPrivate_ = function(options, callback) {
var self = this;

var MAX_PARALLEL_LIMIT = 10;
var errors = [];
var updatedFiles = [];

// Start processing files, iteratively fetching more as necessary.
processFiles({}, function(err) {
if (err || errors.length > 0) {
callback(err || errors, updatedFiles);
this.getFiles(function(err, files) {
if (err) {
callback(err);
return;
}

callback(null, updatedFiles);
});

function processFiles(query, callback) {
self.getFiles(query, function(err, files, nextQuery) {
if (err) {
callback(err);
return;
function processFile(file, callback) {
if (options.public) {
file.makePublic(processedCallback);
} else if (options.private) {
file.makePrivate(processedCallback);
}

// Iterate through each file and make it public or private.
async.eachLimit(files, MAX_PARALLEL_LIMIT, processFile, function(err) {
function processedCallback(err) {
if (err) {
callback(err);
return;
}
if (options.force) {
errors.push(err);
callback();
return;
}

if (nextQuery) {
processFiles(nextQuery, callback);
callback(err);
return;
}

updatedFiles.push(file);
callback();
});
});
}

function processFile(file, callback) {
if (options.public) {
file.makePublic(processedCallback);
} else if (options.private) {
file.makePrivate(processedCallback);
}
}

function processedCallback(err) {
if (err) {
if (options.force) {
errors.push(err);
callback();
return;
}

callback(err);
// Iterate through each file and make it public or private.
async.eachLimit(files, MAX_PARALLEL_LIMIT, processFile, function(err) {
if (err || errors.length > 0) {
callback(err || errors, updatedFiles);
return;
}

updatedFiles.push(file);
callback();
}
}
callback(null, updatedFiles);
});
});
};

/**
Expand Down
13 changes: 8 additions & 5 deletions system-test/storage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -818,12 +818,15 @@ describe('storage', function() {
});
});

afterEach(function(done) {
versionedBucket.deleteFiles({ versions: true }, done);
});

after(function(done) {
versionedBucket.delete(done);
versionedBucket.deleteFiles({ versions: true }, function(err) {
if (err) {
done(err);
return;
}

versionedBucket.delete(done);
});
});

it('should overwrite file, then get older version', function(done) {
Expand Down
48 changes: 7 additions & 41 deletions test/storage/bucket.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,23 +406,6 @@ describe('Bucket', function() {
});
});

it('should get more files if more exist', function(done) {
var fakeNextQuery = { a: 'b', c: 'd' };

bucket.getFiles = function(query, callback) {
if (Object.keys(query).length === 0) {
// First time through, return a `nextQuery` value.
callback(null, [], fakeNextQuery);
} else {
// Second time through.
assert.deepEqual(query, fakeNextQuery);
done();
}
};

bucket.deleteFiles({}, assert.ifError);
});

it('should execute callback with error from getting files', function(done) {
var error = new Error('Error.');

Expand Down Expand Up @@ -1024,7 +1007,7 @@ describe('Bucket', function() {
done();
};

bucket.getFiles = function(query, callback) {
bucket.getFiles = function(callback) {
callback(null, []);
};

Expand All @@ -1042,7 +1025,7 @@ describe('Bucket', function() {
callback();
}));

bucket.getFiles = function(query, callback) {
bucket.getFiles = function(callback) {
callback(null, files);
};

Expand All @@ -1064,7 +1047,7 @@ describe('Bucket', function() {
callback();
}));

bucket.getFiles = function(query, callback) {
bucket.getFiles = function(callback) {
callback(null, files);
};

Expand All @@ -1075,27 +1058,10 @@ describe('Bucket', function() {
});
});

it('should get more files if more exist', function(done) {
var fakeNextQuery = { a: 'b', c: 'd' };

bucket.getFiles = function(query, callback) {
if (Object.keys(query).length === 0) {
// First time through, return a `nextQuery` value.
callback(null, [], fakeNextQuery);
} else {
// Second time through.
assert.deepEqual(query, fakeNextQuery);
done();
}
};

bucket.makeAllFilesPublicPrivate_({}, assert.ifError);
});

it('should execute callback with error from getting files', function(done) {
var error = new Error('Error.');

bucket.getFiles = function(query, callback) {
bucket.getFiles = function(callback) {
callback(error);
};

Expand All @@ -1115,7 +1081,7 @@ describe('Bucket', function() {
callback(error);
}));

bucket.getFiles = function(query, callback) {
bucket.getFiles = function(callback) {
callback(null, files);
};

Expand All @@ -1135,7 +1101,7 @@ describe('Bucket', function() {
callback(error);
}));

bucket.getFiles = function(query, callback) {
bucket.getFiles = function(callback) {
callback(null, files);
};

Expand Down Expand Up @@ -1165,7 +1131,7 @@ describe('Bucket', function() {
callback(error);
}));

bucket.getFiles = function(query, callback) {
bucket.getFiles = function(callback) {
callback(null, successFiles.concat(errorFiles));
};

Expand Down