diff --git a/lib/storage/bucket.js b/lib/storage/bucket.js index ca427b0a9b7..124a0046d0e 100644 --- a/lib/storage/bucket.js +++ b/lib/storage/bucket.js @@ -20,6 +20,7 @@ 'use strict'; +var async = require('async'); var extend = require('extend'); var fs = require('fs'); var mime = require('mime-types'); @@ -409,6 +410,214 @@ Bucket.prototype.getMetadata = function(callback) { }.bind(this)); }; +/** + * Make the bucket listing private. + * + * You may also choose to make the contents of the bucket private by specifying + * `includeFiles: true`. This will automatically run + * {module:storage/file#makePrivate} for every file in the bucket. + * + * When specifying `includeFiles: true`, use `force: true` to delay execution of + * your callback until all files have been processed. By default, the callback + * is executed after the first error. Use `force` to queue such errors until all + * files have been procssed, after which they will be returned as an array as + * the first argument to your callback. + * + * NOTE: This may cause the process to be long-running and use a high number of + * requests. Use with caution. + * + * @param {object=} options - The configuration object. + * @param {boolean} options.includeFiles - Make each file in the bucket private. + * Default: `false`. + * @param {boolean} options.force - Queue errors occurred while making files + * private until all files have been processed. + * @param {function} callback - The callback function. + * + * @example + * //- + * // Make the bucket private. + * //- + * bucket.makePrivate(function(err) {}); + * + * //- + * // Make the bucket and its contents private. + * //- + * var opts = { + * includeFiles: true + * }; + * + * bucket.makePrivate(opts, function(err, files) { + * // `err`: + * // The first error to occur, otherwise null. + * // + * // `files`: + * // Array of files successfully made private in the bucket. + * }); + * + * //- + * // Make the bucket and its contents private, using force to suppress errors + * // until all files have been processed. + * //- + * var opts = { + * includeFiles: true, + * force: true + * }; + * + * bucket.makePrivate(opts, function(errors, files) { + * // `errors`: + * // Array of errors if any occurred, otherwise null. + * // + * // `files`: + * // Array of files successfully made private in the bucket. + * }); + */ +Bucket.prototype.makePrivate = function(options, callback) { + var self = this; + + if (util.is(options, 'function')) { + callback = options; + options = {}; + } + + options = options || {}; + options.private = true; + + async.parallel([setPredefinedAcl, makeFilesPrivate], callback); + + function setPredefinedAcl(done) { + var query = { + predefinedAcl: 'projectPrivate' + }; + + // You aren't allowed to set both predefinedAcl & acl properties on a bucket + // so acl must explicitly be nullified. + var metadata = { acl: null }; + + self.makeReq_('PATCH', '', query, metadata, function(err, resp) { + if (err) { + done(err); + return; + } + + self.metadata = resp; + + done(); + }); + } + + function makeFilesPrivate(done) { + if (!options.includeFiles) { + done(); + return; + } + + self.makeAllFilesPublicPrivate_(options, done); + } +}; + +/** + * Make the bucket publicly readable. + * + * You may also choose to make the contents of the bucket publicly readable by + * specifying `includeFiles: true`. This will automatically run + * {module:storage/file#makePublic} for every file in the bucket. + * + * When specifying `includeFiles: true`, use `force: true` to delay execution of + * your callback until all files have been processed. By default, the callback + * is executed after the first error. Use `force` to queue such errors until all + * files have been procssed, after which they will be returned as an array as + * the first argument to your callback. + * + * NOTE: This may cause the process to be long-running and use a high number of + * requests. Use with caution. + * + * @param {object=} options - The configuration object. + * @param {boolean} options.includeFiles - Make each file in the bucket publicly + * readable. Default: `false`. + * @param {boolean} options.force - Queue errors occurred while making files + * public until all files have been processed. + * @param {function} callback - The callback function. + * + * @example + * //- + * // Make the bucket publicly readable. + * //- + * bucket.makePublic(function(err) {}); + * + * //- + * // Make the bucket and its contents publicly readable. + * //- + * var opts = { + * includeFiles: true + * }; + * + * bucket.makePublic(opts, function(err, files) { + * // `err`: + * // The first error to occur, otherwise null. + * // + * // `files`: + * // Array of files successfully made public in the bucket. + * }); + * + * //- + * // Make the bucket and its contents publicly readable, using force to + * // suppress errors until all files have been processed. + * //- + * var opts = { + * includeFiles: true, + * force: true + * }; + * + * bucket.makePublic(opts, function(errors, files) { + * // `errors`: + * // Array of errors if any occurred, otherwise null. + * // + * // `files`: + * // Array of files successfully made public in the bucket. + * }); + */ +Bucket.prototype.makePublic = function(options, callback) { + var self = this; + + if (util.is(options, 'function')) { + callback = options; + options = {}; + } + + options = options || {}; + options.public = true; + + async.parallel([ + addAclPermissions, + addDefaultAclPermissions, + makeFilesPublic + ], callback); + + function addAclPermissions(done) { + // Allow reading bucket contents while preserving original permissions. + self.acl.add({ + entity: 'allUsers', + role: 'READER' + }, done); + } + + function addDefaultAclPermissions(done) { + self.acl.default.add({ + entity: 'allUsers', + role: 'READER' + }, done); + } + + function makeFilesPublic(done) { + if (!options.includeFiles) { + done(); + return; + } + + self.makeAllFilesPublicPrivate_(options, done); + } +}; + /** * Set the bucket's metadata. * @@ -576,6 +785,89 @@ Bucket.prototype.upload = function(localPath, options, callback) { } }; +/** + * Iterate over all of a bucket's files, calling `file.makePublic()` (public) + * or `file.makePrivate()` (private) on each. + * + * Operations are performed in parallel, up to 10 at once. The first error + * breaks the loop, and will execute the provided callback with it. Specify + * `{ force: true }` to suppress the errors. + * + * @private + * + * @param {object} options - Configuration object. + * @param {boolean} options.force - Supress errors until all files have been + * processed. + * @param {boolean} options.private - Make files private. + * @param {boolean} options.public - Make files public. + * @param {function} callback - The callback function. + */ +Bucket.prototype.makeAllFilesPublicPrivate_ = function(options, callback) { + var self = this; + + var MAX_PARALLEL_LIMIT = 10; + var errors = []; + var updatedFiles = []; + + // Start processing files, iteratively fetching more as necessary. + processFiles({}, function (err) { + if (err || errors.length > 0) { + callback(err || errors, updatedFiles); + return; + } + + callback(null, updatedFiles); + }); + + function processFiles(query, callback) { + self.getFiles(query, function(err, files, nextQuery) { + if (err) { + callback(err); + return; + } + + // Iterate through each file and make it public or private. + async.eachLimit(files, MAX_PARALLEL_LIMIT, processFile, function(err) { + if (err) { + callback(err); + return; + } + + if (nextQuery) { + processFiles(nextQuery, callback); + return; + } + + callback(); + }); + }); + } + + function processFile(file, callback) { + if (options.public) { + file.makePublic(processedCallback); + } else if (options.private) { + file.makePrivate(processedCallback); + } + + function processedCallback(err) { + if (err) { + if (options.force) { + errors.push(err); + callback(); + return; + } + + callback(err); + return; + } + + updatedFiles.push(file); + callback(); + } + } +}; + /** * Make a new request object from the provided arguments and wrap the callback * to intercept non-successful responses. diff --git a/regression/storage.js b/regression/storage.js index 64a84811fb8..e2947915710 100644 --- a/regression/storage.js +++ b/regression/storage.js @@ -175,6 +175,123 @@ describe('storage', function() { }); }); }); + + it('should make a bucket public', function(done) { + bucket.makePublic(function(err) { + assert.ifError(err); + bucket.acl.get({ entity: 'allUsers' }, function(err, aclObject) { + assert.ifError(err); + assert.deepEqual(aclObject, { entity: 'allUsers', role: 'READER' }); + bucket.acl.delete({ entity: 'allUsers' }, done); + }); + }); + }); + + it('should make files public', function(done) { + async.each(['a', 'b', 'c'], createFileWithContent, function(err) { + assert.ifError(err); + + bucket.makePublic({ includeFiles: true }, function(err) { + assert.ifError(err); + + bucket.getFiles(function(err, files) { + assert.ifError(err); + + async.each(files, isFilePublic, function(err) { + assert.ifError(err); + + async.parallel([ + function(next) { + bucket.acl.default.delete({ entity: 'allUsers' }, next); + }, + function(next) { + deleteFiles(bucket, next); + } + ], done); + }); + }); + }); + }); + + function createFileWithContent(content, callback) { + bucket.file(uuid() + '.txt').createWriteStream() + .on('error', callback) + .on('complete', function() { + callback(); + }) + .end(content); + } + + function isFilePublic(file, callback) { + file.acl.get({ entity: 'allUsers' }, function(err, aclObject) { + if (err) { + callback(err); + return; + } + + if (aclObject.entity === 'allUsers' && + aclObject.role === 'READER') { + callback(); + } else { + callback(new Error('File is not public.')); + } + }); + } + }); + + it('should make a bucket private', function(done) { + bucket.makePublic(function(err) { + assert.ifError(err); + bucket.makePrivate(function(err) { + assert.ifError(err); + bucket.acl.get({ entity: 'allUsers' }, function(err, aclObject) { + assert.equal(err.code, 404); + assert.equal(err.message, 'Not Found'); + assert.equal(aclObject, null); + done(); + }); + }); + }); + }); + + it('should make files private', function(done) { + async.each(['a', 'b', 'c'], createFileWithContent, function(err) { + assert.ifError(err); + + bucket.makePrivate({ includeFiles: true }, function(err) { + assert.ifError(err); + + bucket.getFiles(function(err, files) { + assert.ifError(err); + + async.each(files, isFilePrivate, function(err) { + assert.ifError(err); + + deleteFiles(bucket, done); + }); + }); + }); + }); + + function createFileWithContent(content, callback) { + bucket.file(uuid() + '.txt').createWriteStream() + .on('error', callback) + .on('complete', function() { + callback(); + }) + .end(content); + } + + function isFilePrivate(file, callback) { + file.acl.get({ entity: 'allUsers' }, function(err) { + if (err && err.code === 404) { + callback(); + } else { + callback(new Error('File is not private.')); + } + }); + } + }); }); describe('files', function() { @@ -245,7 +362,7 @@ describe('storage', function() { }); }); - it('should be made public', function(done) { + it('should make a file public', function(done) { file.makePublic(function(err) { assert.ifError(err); file.acl.get({ entity: 'allUsers' }, function(err, aclObject) { @@ -256,7 +373,7 @@ describe('storage', function() { }); }); - it('should be made private', function(done) { + it('should make a file private', function(done) { file.makePublic(function(err) { assert.ifError(err); file.makePrivate(function(err) { diff --git a/test/storage/bucket.js b/test/storage/bucket.js index 72235cc7128..9333db5c63a 100644 --- a/test/storage/bucket.js +++ b/test/storage/bucket.js @@ -20,6 +20,7 @@ var assert = require('assert'); var async = require('async'); +var extend = require('extend'); var mime = require('mime-types'); var mockery = require('mockery'); var request = require('request'); @@ -45,10 +46,16 @@ var request_Cached = request; var request_Override; function fakeRequest() { - var args = [].slice.apply(arguments); - return (request_Override || request_Cached).apply(null, args); + return (request_Override || request_Cached).apply(null, arguments); } +var eachLimit_Override; + +var fakeAsync = extend({}, async); +fakeAsync.eachLimit = function() { + (eachLimit_Override || async.eachLimit).apply(null, arguments); +}; + describe('Bucket', function() { var Bucket; var BUCKET_NAME = 'test-bucket'; @@ -61,6 +68,7 @@ describe('Bucket', function() { before(function() { mockery.registerMock('./file.js', FakeFile); + mockery.registerMock('async', fakeAsync); mockery.registerMock('request', fakeRequest); mockery.enable({ useCleanCache: true, @@ -76,6 +84,7 @@ describe('Bucket', function() { beforeEach(function() { request_Override = null; + eachLimit_Override = null; bucket = new Bucket(options, BUCKET_NAME); }); @@ -430,6 +439,138 @@ describe('Bucket', function() { }); }); + describe('makePublic', function() { + beforeEach(function() { + bucket.makeReq_ = function(method, path, query, body, callback) { + callback(); + }; + }); + + it('should set ACL, default ACL, and publicize files', function(done) { + var didSetAcl = false; + var didSetDefaultAcl = false; + var didMakeFilesPublic = false; + + bucket.acl.add = function(opts, callback) { + assert.equal(opts.entity, 'allUsers'); + assert.equal(opts.role, 'READER'); + didSetAcl = true; + callback(); + }; + + bucket.acl.default.add = function(opts, callback) { + assert.equal(opts.entity, 'allUsers'); + assert.equal(opts.role, 'READER'); + didSetDefaultAcl = true; + callback(); + }; + + bucket.makeAllFilesPublicPrivate_ = function(opts, callback) { + assert.strictEqual(opts.public, true); + assert.strictEqual(opts.force, true); + didMakeFilesPublic = true; + callback(); + }; + + bucket.makePublic({ + includeFiles: true, + force: true + }, function(err) { + assert.ifError(err); + assert(didSetAcl); + assert(didSetDefaultAcl); + assert(didMakeFilesPublic); + done(); + }); + }); + + it('should not make files public by default', function(done) { + bucket.acl.add = function(opts, callback) { + callback(); + }; + + bucket.acl.default.add = function(opts, callback) { + callback(); + }; + + bucket.makeAllFilesPublicPrivate_ = function() { + throw new Error('Please, no. I do not want to be called.'); + }; + + bucket.makePublic(done); + }); + + it('should execute callback with error', function(done) { + var error = new Error('Error.'); + + bucket.acl.add = function(opts, callback) { + callback(error); + }; + + bucket.makePublic(function(err) { + assert.equal(err, error); + done(); + }); + }); + }); + + describe('makePrivate', function() { + it('should set predefinedAcl & privatize files', function(done) { + var didSetPredefinedAcl = false; + var didMakeFilesPrivate = false; + + bucket.makeReq_ = function(method, path, query, body, callback) { + // Correct request. + assert.equal(method, 'PATCH'); + assert.equal(path, ''); + assert.deepEqual(query, { predefinedAcl: 'projectPrivate' }); + assert.deepEqual(body, { acl: null }); + + didSetPredefinedAcl = true; + callback(); + }; + + bucket.makeAllFilesPublicPrivate_ = function(opts, callback) { + assert.strictEqual(opts.private, true); + assert.strictEqual(opts.force, true); + didMakeFilesPrivate = true; + callback(); + }; + + bucket.makePrivate({ includeFiles: true, force: true }, function(err) { + assert.ifError(err); + assert(didSetPredefinedAcl); + assert(didMakeFilesPrivate); + done(); + }); + }); + + it('should not make files private by default', function(done) { + bucket.makeReq_ = function(method, path, query, body, callback) { + callback(); + }; + + bucket.makeAllFilesPublicPrivate_ = function() { + throw new Error('Please, no. I do not want to be called.'); + }; + + bucket.makePrivate(done); + }); + + it('should execute callback with error', function(done) { + var error = new Error('Error.'); + + bucket.makeReq_ = function(method, path, query, body, callback) { + callback(error); + }; + + bucket.makePrivate(function(err) { + assert.equal(err, error); + done(); + }); + }); + }); + describe('setMetadata', function() { var metadata = { fake: 'metadata' }; @@ -623,6 +764,177 @@ describe('Bucket', function() { }); }); + describe('makeAllFilesPublicPrivate_', function() { + it('should get all files from the bucket', function(done) { + bucket.getFiles = function() { + done(); + }; + + bucket.makeAllFilesPublicPrivate_({}, assert.ifError); + }); + + it('should process 10 files at a time', function(done) { + eachLimit_Override = function(arr, limit) { + assert.equal(limit, 10); + done(); + }; + + bucket.getFiles = function(query, callback) { + callback(null, []); + }; + + bucket.makeAllFilesPublicPrivate_({}, assert.ifError); + }); + + it('should make files public', function(done) { + var timesCalled = 0; + + var files = [ + bucket.file('1'), + bucket.file('2') + ].map(util.propAssign('makePublic', function(callback) { + timesCalled++; + callback(); + })); + + bucket.getFiles = function(query, callback) { + callback(null, files); + }; + + bucket.makeAllFilesPublicPrivate_({ public: true }, function(err) { + assert.ifError(err); + assert.equal(timesCalled, files.length); + done(); + }); + }); + + it('should make files private', function(done) { + var timesCalled = 0; + + var files = [ + bucket.file('1'), + bucket.file('2') + ].map(util.propAssign('makePrivate', function(callback) { + timesCalled++; + callback(); + })); + + bucket.getFiles = function(query, callback) { + callback(null, files); + }; + + bucket.makeAllFilesPublicPrivate_({ private: true }, function(err) { + assert.ifError(err); + assert.equal(timesCalled, files.length); + done(); + }); + }); + + it('should get more files if more exist', function(done) { + var fakeNextQuery = { a: 'b', c: 'd' }; + + bucket.getFiles = function(query, callback) { + if (Object.keys(query).length === 0) { + // First time through, return a `nextQuery` value. + callback(null, [], fakeNextQuery); + } else { + // Second time through. + assert.deepEqual(query, fakeNextQuery); + done(); + } + }; + + bucket.makeAllFilesPublicPrivate_({}, assert.ifError); + }); + + it('should execute callback with error from getting files', function(done) { + var error = new Error('Error.'); + + bucket.getFiles = function(query, callback) { + callback(error); + }; + + bucket.makeAllFilesPublicPrivate_({}, function(err) { + assert.equal(err, error); + done(); + }); + }); + + it('should execute callback with error from changing file', function(done) { + var error = new Error('Error.'); + + var files = [ + bucket.file('1'), + bucket.file('2') + ].map(util.propAssign('makePublic', function(callback) { + callback(error); + })); + + bucket.getFiles = function(query, callback) { + callback(null, files); + }; + + bucket.makeAllFilesPublicPrivate_({ public: true }, function(err) { + assert.equal(err, error); + done(); + }); + }); + + it('should execute callback with queued errors', function(done) { + var error = new Error('Error.'); + + var files = [ + bucket.file('1'), + bucket.file('2') + ].map(util.propAssign('makePublic', function(callback) { + callback(error); + })); + + bucket.getFiles = function(query, callback) { + callback(null, files); + }; + + bucket.makeAllFilesPublicPrivate_({ + public: true, + force: true + }, function(errs) { + assert.deepEqual(errs, [error, error]); + done(); + }); + }); + + it('should execute callback with files changed', function(done) { + var error = new Error('Error.'); + + var successFiles = [ + bucket.file('1'), + bucket.file('2') + ].map(util.propAssign('makePublic', function(callback) { + callback(); + })); + + var errorFiles = [ + bucket.file('3'), + bucket.file('4') + ].map(util.propAssign('makePublic', function(callback) { + callback(error); + })); + + bucket.getFiles = function(query, callback) { + callback(null, successFiles.concat(errorFiles)); + }; + + bucket.makeAllFilesPublicPrivate_({ + public: true, + force: true + }, function(errs, files) { + assert.deepEqual(errs, [error, error]); + assert.deepEqual(files, successFiles); + done(); + }); + }); + }); + describe('makeReq_', function() { var method = 'POST'; var path = '/path';