bug: Trap BOTO Server exception as 503's

Closes #605

autopush/db.py

@@ -7,7 +7,7 @@
 import uuid
 from functools import wraps
 
-from boto.exception import JSONResponseError
+from boto.exception import JSONResponseError, BotoServerError
 from boto.dynamodb2.exceptions import (
     ConditionalCheckFailedException,
     ItemNotFound,
@@ -215,6 +215,9 @@ def wrapper(self, *args, **kwargs):
         except ProvisionedThroughputExceededException:
             self.metrics.increment("error.provisioned.%s" % func.__name__)
             raise
+        except BotoServerError:
+            self.metrics.increment("error.botoserver.%s" % func.__name__)
+            raise
     return wrapper
 
 

autopush/endpoint.py

@@ -34,6 +34,7 @@
     ItemNotFound,
     ProvisionedThroughputExceededException,
 )
+from boto.exception import BotoServerError
 from cryptography.fernet import InvalidToken
 from cryptography.hazmat.primitives import constant_time
 from jose import JOSEError
@@ -254,6 +255,14 @@ def _jws_err(self, fail):
         self._write_response(401, errno=109,
                              message="Invalid Authorization")
 
+    def _boto_err(self, fail):
+        """errBack for random boto exceptions"""
+        fail.trap(BotoServerError)
+        self.log.info(format="BOTO Error: %s" % str(fail.value),
+                      status_code=503, errno=202, **self._client_info)
+        self._write_response(503, errno=202,
+                             message="Communication error, please retry")
+
     def _router_response(self, response):
         for name, val in response.headers.items():
             self.set_header(name, val)
@@ -337,6 +346,7 @@ def _db_error_handling(self, d):
         """Tack on the common error handling for a dynamodb request and
         uncaught exceptions"""
         d.addErrback(self._overload_err)
+        d.addErrback(self._boto_err)
         d.addErrback(self._response_err)
         return d
 

autopush/tests/test_db.py

@@ -9,6 +9,7 @@
 )
 from boto.dynamodb2.layer1 import DynamoDBConnection
 from boto.dynamodb2.items import Item
+from boto.exception import BotoServerError
 from mock import Mock
 from nose.tools import eq_, assert_raises, ok_
 
@@ -153,6 +154,18 @@ def raise_error(*args, **kwargs):
         result = storage.save_notification(dummy_uaid, dummy_chid, 8)
         eq_(result, False)
 
+    def test_fetch_boto_err(self):
+        s = get_storage_table()
+        storage = Storage(s, SinkMetrics())
+        storage.table.connection = Mock()
+
+        def raise_error(*args, **kwargs):
+            raise BotoServerError(None, None)
+
+        storage.table.connection.put_item.side_effect = raise_error
+        with assert_raises(BotoServerError):
+            storage.save_notification(dummy_uaid, dummy_chid, 12)
+
     def test_fetch_over_provisioned(self):
         s = get_storage_table()
         storage = Storage(s, SinkMetrics())

autopush/tests/test_endpoint.py

@@ -10,6 +10,7 @@
 import ecdsa
 import jose
 import twisted.internet.base
+from boto.exception import BotoServerError
 from cryptography.fernet import Fernet, InvalidToken
 from cyclone.web import Application
 from jose import jws
@@ -595,6 +596,9 @@ def _throw_item_not_found(self, item):
     def _throw_provisioned_error(self, *args):
         raise ProvisionedThroughputExceededException(None, None)
 
+    def _throw_boto_err(self, *args):
+        raise BotoServerError(None, None)
+
     def test_process_token_client_unknown(self):
         self.router_mock.configure_mock(**{
             'get_uaid.side_effect': self._throw_item_not_found})
@@ -1210,6 +1214,18 @@ def handle_finish(result):
         self.endpoint.put(None, dummy_uaid)
         return self.finish_deferred
 
+    def test_put_boto_error(self):
+        self.fernet_mock.decrypt.return_value = dummy_token
+        self.router_mock.get_uaid.side_effect = self._throw_boto_err
+
+        def handle_finish(result):
+            self.assertTrue(result)
+            self.endpoint.set_status.assert_called_with(503, None)
+        self.finish_deferred.addCallback(handle_finish)
+
+        self.endpoint.put(None, dummy_uaid)
+        return self.finish_deferred
+
     def test_cors(self):
         ch1 = "Access-Control-Allow-Origin"
         ch2 = "Access-Control-Allow-Methods"

autopush/tests/test_web_base.py

@@ -1,6 +1,7 @@
 import sys
 import uuid
 
+from boto.exception import BotoServerError
 from cyclone.web import Application
 from mock import Mock, patch
 from moto import mock_dynamodb2
@@ -211,6 +212,14 @@ def test_overload_err(self):
         self.base._overload_err(fail)
         self.status_mock.assert_called_with(503)
 
+    def test_boto_err(self):
+        try:
+            raise BotoServerError(503, "derp")
+        except:
+            fail = Failure()
+        self.base._boto_err(fail)
+        self.status_mock.assert_called_with(503)
+
     def test_router_response(self):
         from autopush.router.interface import RouterResponse
         response = RouterResponse(headers=dict(Location="http://a.com/"))

autopush/tests/test_web_validation.py

@@ -134,6 +134,7 @@ def get(self):
         vr.finish = lambda: d.callback(True)
         vr.write = Mock()
         vr._overload_err = Mock()
+        vr._boto_err = Mock()
         vr._validation_err = Mock()
         vr._response_err = Mock()
         vr.ap_settings = Mock()

autopush/web/base.py

@@ -7,6 +7,7 @@
 from boto.dynamodb2.exceptions import (
     ProvisionedThroughputExceededException,
 )
+from boto.exception import BotoServerError
 from twisted.logger import Logger
 from twisted.python import failure
 
@@ -196,6 +197,14 @@ def _overload_err(self, fail):
         self._write_response(503, 201,
                              message="Please slow message send rate")
 
+    def _boto_err(self, fail):
+        """errBack for random boto exceptions"""
+        fail.trap(BotoServerError)
+        self.log.info(format="BOTO Error: %s" % str(fail.value),
+                      status_code=503, errno=202, **self._client_info)
+        self._write_response(503, errno=202,
+                             message="Communication error, please retry")
+
     def _router_response(self, response):
         for name, val in response.headers.items():
             self.set_header(name, val)

autopush/web/validation.py

@@ -79,6 +79,7 @@ def wrapper(request_handler, *args, **kwargs):
             d.addCallback(self._call_func, func, request_handler, *args,
                           **kwargs)
             d.addErrback(request_handler._overload_err)
+            d.addErrback(request_handler._boto_err)
             d.addErrback(request_handler._validation_err)
             d.addErrback(request_handler._response_err)
         return wrapper
@@ -321,8 +322,8 @@ def validate_auth(self, d):
 
         try:
             jwt = extract_jwt(token, public_key)
-        except (ValueError, JOSEError) as ex:
-            raise InvalidRequest("Invalid Authorization Header: %s" % str(ex),
+        except (ValueError, JOSEError):
+            raise InvalidRequest("Invalid Authorization Header",
                                  status_code=401, errno=109,
                                  headers={"www-authenticate": PREF_SCHEME})
         if jwt.get('exp', 0) < time.time():