Exponential backoff and retry certain requests

lib/common/util.js

@@ -409,8 +409,39 @@ function makeWritableStream(dup, options, onComplete) {
 
 module.exports.makeWritableStream = makeWritableStream;
 
+/**
+ * Returns an exponential distributed time to wait given the number of retries
+ * that have been previously been attempted on the request.
+ *
+ * @param  {number} retryNumber - The number of retries previously attempted.
+ * @return {number} An exponentially distributed time to wait E.g. for use with
+ *     exponential backoff.
+ */
+function getNextRetryWait(retryNumber) {
+  return (Math.pow(2, retryNumber) * 1000) + Math.floor(Math.random() * 1000);
+}
+
+module.exports.getNextRetryWait = getNextRetryWait;
+
+/**
+ * Returns true if the API request should be retried, given the error that was
+ * given the first time the request was attempted. This is used for rate limit
+ * related errors as well as intermittent server errors.
+ *
+ * @param  {error} err - The API error to check if it is appropriate to retry.
+ * @return {boolean} True if the API request should be retried, false otherwise.
+ */
+function shouldRetry(err) {
+  return !!err && [429, 500, 503].indexOf(err.code) !== -1;
+}
+
+module.exports.shouldRetryErr = shouldRetry;
+
 function makeAuthorizedRequest(config) {
   var GAE_OR_GCE = !config || (!config.credentials && !config.keyFile);
+  var MAX_RETRIES = config && config.maxRetries || 3;
+  var autoRetry = !config || config.autoRetry !== false ? true : false;
+  var attemptedRetries = 0;
 
   var missingCredentialsError = new Error();
   missingCredentialsError.message = [
@@ -475,12 +506,20 @@ function makeAuthorizedRequest(config) {
         return;
       }
 
+      function handleRateLimitResp(err, res, body) {
+        if (shouldRetry(err) && autoRetry && MAX_RETRIES > attemptedRetries) {
+          setTimeout(function() {
+            request(authorizedReqOpts, handleRateLimitResp);
+          }, getNextRetryWait(attemptedRetries++));
+        } else {
+          handleResp(err, res, body, callback);
+        }
+      }
+
       if (callback.onAuthorized) {
         callback.onAuthorized(null, authorizedReqOpts);
       } else {
-        request(authorizedReqOpts, function(err, res, body) {
-          handleResp(err, res, body, callback);
-        });
+        request(authorizedReqOpts, handleRateLimitResp);
       }
     }
 

lib/index.js

@@ -82,6 +82,12 @@ var util = require('./common/util.js');
  * @param {object=} config.credentials - Credentials object.
  * @param {string} config.credentials.client_email
  * @param {string} config.credentials.private_key
+ * @param {boolean} config.autoRetry - Automatically retry requests if the
+ *     response is related to rate limits or certain intermittent server errors.
+ *     (default: true). Recommended is true. We will exponentially backoff
+ *     subsequent requests by default.
+ * @param {number} config.maxRetries - Max number of auto retries to attempt
+ *     before returning the error. (default: 3).
  *
  * @example
  * var gcloud = require('gcloud')({
@@ -102,6 +108,13 @@ var util = require('./common/util.js');
  *   // properties may be overridden:
  *   keyFilename: '/path/to/other/keyfile.json'
  * });
+ *
+ * @example
+ * var gcloud = require('gcloud')({
+ *   keyFilename: '/path/to/keyfile.json',
+ *   projectId: 'my-project',
+ *   maxRetries: 5 // retry rate limited requests 5 times before giving up
+ * });
  */
 function gcloud(config) {
   return {

test/common/util.js

@@ -463,6 +463,190 @@ describe('common/util', function() {
         var makeRequest = util.makeAuthorizedRequest({});
         makeRequest({}, assert.ifError);
       });
+
+      it('should retry rate limit requests by default', function(done) {
+        var attemptedRetries = 0;
+        var error = new Error('Rate Limit Error.');
+        error.code = 429; // Rate limit error
+
+        var authorizedReqOpts = { a: 'b', c: 'd' };
+
+        var old_setTimeout = setTimeout;
+        setTimeout = function(callback, time) {
+          var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000);
+          var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000;
+          assert(time >= MIN_TIME && time <= MAX_TIME);
+          attemptedRetries++;
+          callback(); // make the request again
+        };
+
+        gsa_Override = function() {
+          return function authorize(reqOpts, callback) {
+            callback(null, authorizedReqOpts);
+          };
+        };
+
+        request_Override = function(reqOpts, callback) {
+          if (attemptedRetries === 3) {
+            setTimeout = old_setTimeout;
+            done();
+          } else {
+            callback(error); // this callback should check for rate limits
+          }
+        };
+
+        var makeRequest = util.makeAuthorizedRequest({});
+        makeRequest({}, assert.ifError);
+      });
+
+      it('should retry rate limits 3x on 429, 500, 503', function(done) {
+        var attemptedRetries = 0;
+        var codes = [429, 503, 500, 'done'];
+        var error = new Error('Rate Limit Error.');
+        error.code = codes[0]; // Rate limit error
+
+        var authorizedReqOpts = { a: 'b', c: 'd' };
+
+        var old_setTimeout = setTimeout;
+        setTimeout = function(callback, time) {
+          var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000);
+          var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000;
+          assert(time >= MIN_TIME && time <= MAX_TIME);
+          attemptedRetries++;
+          error.code = codes[attemptedRetries]; // test a new code
+          callback(); // make the request again
+        };
+
+        gsa_Override = function() {
+          return function authorize(reqOpts, callback) {
+            callback(null, authorizedReqOpts);
+          };
+        };
+
+        request_Override = function(reqOpts, callback) {
+          callback(error); // this callback should check for rate limits
+        };
+
+        var makeRequest = util.makeAuthorizedRequest({});
+        makeRequest({}, function(err) {
+          setTimeout = old_setTimeout;
+          assert.equal(err, error);
+          assert.equal(err.code, 'done');
+          done();
+        });
+      });
+
+      it('should retry rate limits 3x by default', function(done) {
+        var attemptedRetries = 0;
+        var error = new Error('Rate Limit Error.');
+        error.code = 429; // Rate limit error
+
+        var authorizedReqOpts = { a: 'b', c: 'd' };
+
+        var old_setTimeout = setTimeout;
+        setTimeout = function(callback, time) {
+          var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000);
+          var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000;
+          assert(time >= MIN_TIME && time <= MAX_TIME);
+          attemptedRetries++;
+          callback(); // make the request again
+        };
+
+        gsa_Override = function() {
+          return function authorize(reqOpts, callback) {
+            callback(null, authorizedReqOpts);
+          };
+        };
+
+        request_Override = function(reqOpts, callback) {
+          callback(error); // this callback should check for rate limits
+        };
+
+        var makeRequest = util.makeAuthorizedRequest({});
+        makeRequest({}, function(err) {
+          setTimeout = old_setTimeout;
+          assert.equal(attemptedRetries, 3);
+          assert.equal(err, error);
+          done();
+        });
+      });
+
+      it('should retry rate limits by maxRetries if provided', function(done) {
+        var MAX_RETRIES = 5;
+        var attemptedRetries = 0;
+        var error = new Error('Rate Limit Error.');
+        error.code = 429; // Rate limit error
+
+        var authorizedReqOpts = { a: 'b', c: 'd' };
+
+        var old_setTimeout = setTimeout;
+        setTimeout = function(callback, time) {
+          var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000);
+          var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000;
+          assert(time >= MIN_TIME && time <= MAX_TIME);
+          attemptedRetries++;
+          callback(); // make the request again
+        };
+
+        gsa_Override = function() {
+          return function authorize(reqOpts, callback) {
+            callback(null, authorizedReqOpts);
+          };
+        };
+
+        request_Override = function(reqOpts, callback) {
+          callback(error); // this callback should check for rate limits
+        };
+
+        var makeRequest = util.makeAuthorizedRequest({
+          maxRetries: MAX_RETRIES
+        });
+
+        makeRequest({}, function(err) {
+          setTimeout = old_setTimeout;
+          assert.equal(attemptedRetries, MAX_RETRIES);
+          assert.equal(err, error);
+          done();
+        });
+      });
+
+      it('should not retry rate limits if autoRetry is false', function(done) {
+        var attemptedRetries = 0;
+        var error = new Error('Rate Limit Error.');
+        error.code = 429; // Rate limit error
+
+        var authorizedReqOpts = { a: 'b', c: 'd' };
+
+        var old_setTimeout = setTimeout;
+        setTimeout = function(callback, time) {
+          var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000);
+          var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000;
+          assert(time >= MIN_TIME && time <= MAX_TIME);
+          attemptedRetries++;
+          callback(); // make the request again
+        };
+
+        gsa_Override = function() {
+          return function authorize(reqOpts, callback) {
+            callback(null, authorizedReqOpts);
+          };
+        };
+
+        request_Override = function(reqOpts, callback) {
+          callback(error); // this callback should check for rate limits
+        };
+
+        var makeRequest = util.makeAuthorizedRequest({
+          autoRetry: false
+        });
+
+        makeRequest({}, function(err) {
+          setTimeout = old_setTimeout;
+          assert.equal(attemptedRetries, 0);
+          assert.equal(err, error);
+          done();
+        });
+      });
     });
   });