diff --git a/lib/storage/file.js b/lib/storage/file.js
index eb28f97d9711..6c3dbe36a106 100644
--- a/lib/storage/file.js
+++ b/lib/storage/file.js
@@ -20,6 +20,7 @@
'use strict';
+var crc = require('fast-crc32c');
var crypto = require('crypto');
var duplexify = require('duplexify');
var request = require('request');
@@ -213,6 +214,13 @@ File.prototype.copy = function(destination, callback) {
* 911. If you receive this error, the best recourse is to try downloading the
* file again.
*
+ * @param {object=} options - Configuration object.
+ * @param {string|boolean} options.validation - Possible values: `"md5"`,
+ * `"crc32c"`, or `false`. By default, data integrity is validated with an
+ * MD5 checksum for maximum reliability. CRC32c will provide better
+ * performance with less reliability. You may also choose to skip validation
+ * completely, however this is **not recommended**.
+ *
* @example
* //-
* //
Downloading a File
@@ -228,19 +236,45 @@ File.prototype.copy = function(destination, callback) {
* .pipe(fs.createWriteStream('/Users/stephen/Photos/image.png'))
* .on('error', function(err) {});
*/
-File.prototype.createReadStream = function() {
+File.prototype.createReadStream = function(options) {
+ options = options || {};
+
var that = this;
var throughStream = through();
- this.getMetadata(function(err, metadata) {
- if (err) {
- throughStream.emit('error', err);
- throughStream.end();
- return;
+ var validations = ['crc32c', 'md5'];
+ var validation;
+
+ if (util.is(options.validation, 'string')) {
+ options.validation = options.validation.toLowerCase();
+
+ if (validations.indexOf(options.validation) > -1) {
+ validation = options.validation;
+ } else {
+ validation = 'all';
}
+ }
- createAuthorizedReq(metadata.mediaLink);
- });
+ if (util.is(options.validation, 'undefined')) {
+ validation = 'all';
+ }
+
+ var crc32c = validation === 'crc32c' || validation === 'all';
+ var md5 = validation === 'md5' || validation === 'all';
+
+ if (this.metadata.mediaLink) {
+ createAuthorizedReq(this.metadata.mediaLink);
+ } else {
+ this.getMetadata(function(err, metadata) {
+ if (err) {
+ throughStream.emit('error', err);
+ throughStream.end();
+ return;
+ }
+
+ createAuthorizedReq(metadata.mediaLink);
+ });
+ }
return throughStream;
@@ -261,6 +295,7 @@ File.prototype.createReadStream = function() {
// For data integrity, hash the contents of the stream as we receive it
// from the server.
+ var localCrc32cHash;
var localMd5Hash = crypto.createHash('md5');
request(authorizedReqOpts)
@@ -270,15 +305,46 @@ File.prototype.createReadStream = function() {
})
.on('data', function(chunk) {
- localMd5Hash.update(chunk);
+ if (crc32c) {
+ localCrc32cHash = crc.calculate(chunk, localCrc32cHash);
+ }
+
+ if (md5) {
+ localMd5Hash.update(chunk);
+ }
})
- .on('complete', function() {
- localMd5Hash = localMd5Hash.digest('base64');
+ .on('complete', function(res) {
+ var failed = false;
- if (that.metadata.md5Hash === localMd5Hash) {
- throughStream.emit('complete');
- } else {
+ if (crc32c) {
+ localCrc32cHash = new Buffer([localCrc32cHash]);
+ localCrc32cHash = localCrc32cHash.toString('base64');
+ }
+
+ if (md5) {
+ localMd5Hash = localMd5Hash.digest('base64');
+ }
+
+ var hashes = {};
+ res.headers['x-goog-hash'].split(',').forEach(function (hash) {
+ var hashType = hash.split('=')[0];
+ hashes[hashType] = hash.substr(hash.indexOf('=') + 1);
+ });
+
+ if (validation === 'all') {
+ if (hashes.md5) {
+ failed = localMd5Hash !== hashes.md5;
+ } else if (hashes.crc32) {
+ failed = localCrc32cHash !== hashes.crc32c.substr(4);
+ }
+ } else if (md5) {
+ failed = localMd5Hash !== hashes.md5;
+ } else if (crc32c) {
+ failed = localCrc32cHash !== hashes.crc32c.substr(4);
+ }
+
+ if (failed) {
var error = new Error({
code: 911,
message: [
@@ -289,6 +355,8 @@ File.prototype.createReadStream = function() {
});
throughStream.emit('error', error);
+ } else {
+ throughStream.emit('complete');
}
throughStream.end();
diff --git a/package.json b/package.json
index b1768d92b31b..ece602cbdadb 100644
--- a/package.json
+++ b/package.json
@@ -46,6 +46,7 @@
"dependencies": {
"duplexify": "^3.1.2",
"extend": "^1.3.0",
+ "fast-crc32c": "^0.1.3",
"google-service-account": "^1.0.3",
"mime": "^1.2.11",
"node-uuid": "^1.4.1",