diff --git a/lib/storage/file.js b/lib/storage/file.js index eb28f97d9711..6c3dbe36a106 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -20,6 +20,7 @@ 'use strict'; +var crc = require('fast-crc32c'); var crypto = require('crypto'); var duplexify = require('duplexify'); var request = require('request'); @@ -213,6 +214,13 @@ File.prototype.copy = function(destination, callback) { * 911. If you receive this error, the best recourse is to try downloading the * file again. * + * @param {object=} options - Configuration object. + * @param {string|boolean} options.validation - Possible values: `"md5"`, + * `"crc32c"`, or `false`. By default, data integrity is validated with an + * MD5 checksum for maximum reliability. CRC32c will provide better + * performance with less reliability. You may also choose to skip validation + * completely, however this is **not recommended**. + * * @example * //- * //

Downloading a File

@@ -228,19 +236,45 @@ File.prototype.copy = function(destination, callback) { * .pipe(fs.createWriteStream('/Users/stephen/Photos/image.png')) * .on('error', function(err) {}); */ -File.prototype.createReadStream = function() { +File.prototype.createReadStream = function(options) { + options = options || {}; + var that = this; var throughStream = through(); - this.getMetadata(function(err, metadata) { - if (err) { - throughStream.emit('error', err); - throughStream.end(); - return; + var validations = ['crc32c', 'md5']; + var validation; + + if (util.is(options.validation, 'string')) { + options.validation = options.validation.toLowerCase(); + + if (validations.indexOf(options.validation) > -1) { + validation = options.validation; + } else { + validation = 'all'; } + } - createAuthorizedReq(metadata.mediaLink); - }); + if (util.is(options.validation, 'undefined')) { + validation = 'all'; + } + + var crc32c = validation === 'crc32c' || validation === 'all'; + var md5 = validation === 'md5' || validation === 'all'; + + if (this.metadata.mediaLink) { + createAuthorizedReq(this.metadata.mediaLink); + } else { + this.getMetadata(function(err, metadata) { + if (err) { + throughStream.emit('error', err); + throughStream.end(); + return; + } + + createAuthorizedReq(metadata.mediaLink); + }); + } return throughStream; @@ -261,6 +295,7 @@ File.prototype.createReadStream = function() { // For data integrity, hash the contents of the stream as we receive it // from the server. + var localCrc32cHash; var localMd5Hash = crypto.createHash('md5'); request(authorizedReqOpts) @@ -270,15 +305,46 @@ File.prototype.createReadStream = function() { }) .on('data', function(chunk) { - localMd5Hash.update(chunk); + if (crc32c) { + localCrc32cHash = crc.calculate(chunk, localCrc32cHash); + } + + if (md5) { + localMd5Hash.update(chunk); + } }) - .on('complete', function() { - localMd5Hash = localMd5Hash.digest('base64'); + .on('complete', function(res) { + var failed = false; - if (that.metadata.md5Hash === localMd5Hash) { - throughStream.emit('complete'); - } else { + if (crc32c) { + localCrc32cHash = new Buffer([localCrc32cHash]); + localCrc32cHash = localCrc32cHash.toString('base64'); + } + + if (md5) { + localMd5Hash = localMd5Hash.digest('base64'); + } + + var hashes = {}; + res.headers['x-goog-hash'].split(',').forEach(function (hash) { + var hashType = hash.split('=')[0]; + hashes[hashType] = hash.substr(hash.indexOf('=') + 1); + }); + + if (validation === 'all') { + if (hashes.md5) { + failed = localMd5Hash !== hashes.md5; + } else if (hashes.crc32) { + failed = localCrc32cHash !== hashes.crc32c.substr(4); + } + } else if (md5) { + failed = localMd5Hash !== hashes.md5; + } else if (crc32c) { + failed = localCrc32cHash !== hashes.crc32c.substr(4); + } + + if (failed) { var error = new Error({ code: 911, message: [ @@ -289,6 +355,8 @@ File.prototype.createReadStream = function() { }); throughStream.emit('error', error); + } else { + throughStream.emit('complete'); } throughStream.end(); diff --git a/package.json b/package.json index b1768d92b31b..ece602cbdadb 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "dependencies": { "duplexify": "^3.1.2", "extend": "^1.3.0", + "fast-crc32c": "^0.1.3", "google-service-account": "^1.0.3", "mime": "^1.2.11", "node-uuid": "^1.4.1",