From 75a8889d64c095db74db6cda458413181822d4e5 Mon Sep 17 00:00:00 2001 From: jrconlin Date: Fri, 19 Aug 2016 10:49:37 -0700 Subject: [PATCH] bug: Trap BOTO Server exception as 503's Closes #605 --- autopush/db.py | 5 ++++- autopush/endpoint.py | 10 ++++++++++ autopush/tests/test_db.py | 13 +++++++++++++ autopush/tests/test_endpoint.py | 16 ++++++++++++++++ autopush/tests/test_web_base.py | 9 +++++++++ autopush/tests/test_web_validation.py | 1 + autopush/web/base.py | 9 +++++++++ autopush/web/validation.py | 5 +++-- 8 files changed, 65 insertions(+), 3 deletions(-) diff --git a/autopush/db.py b/autopush/db.py index 40db5443..de3ddd0a 100644 --- a/autopush/db.py +++ b/autopush/db.py @@ -7,7 +7,7 @@ import uuid from functools import wraps -from boto.exception import JSONResponseError +from boto.exception import JSONResponseError, BotoServerError from boto.dynamodb2.exceptions import ( ConditionalCheckFailedException, ItemNotFound, @@ -215,6 +215,9 @@ def wrapper(self, *args, **kwargs): except ProvisionedThroughputExceededException: self.metrics.increment("error.provisioned.%s" % func.__name__) raise + except BotoServerError: + self.metrics.increment("error.botoserver.%s" % func.__name__) + raise return wrapper diff --git a/autopush/endpoint.py b/autopush/endpoint.py index e8a0cdf9..18c70882 100644 --- a/autopush/endpoint.py +++ b/autopush/endpoint.py @@ -34,6 +34,7 @@ ItemNotFound, ProvisionedThroughputExceededException, ) +from boto.exception import BotoServerError from cryptography.fernet import InvalidToken from cryptography.hazmat.primitives import constant_time from jose import JOSEError @@ -254,6 +255,14 @@ def _jws_err(self, fail): self._write_response(401, errno=109, message="Invalid Authorization") + def _boto_err(self, fail): + """errBack for random boto exceptions""" + fail.trap(BotoServerError) + self.log.info(format="BOTO Error: %s" % str(fail.value), + status_code=503, errno=202, **self._client_info) + self._write_response(503, errno=202, + message="Communication error, please retry") + def _router_response(self, response): for name, val in response.headers.items(): self.set_header(name, val) @@ -337,6 +346,7 @@ def _db_error_handling(self, d): """Tack on the common error handling for a dynamodb request and uncaught exceptions""" d.addErrback(self._overload_err) + d.addErrback(self._boto_err) d.addErrback(self._response_err) return d diff --git a/autopush/tests/test_db.py b/autopush/tests/test_db.py index 4ece6c8b..76ebaa12 100644 --- a/autopush/tests/test_db.py +++ b/autopush/tests/test_db.py @@ -9,6 +9,7 @@ ) from boto.dynamodb2.layer1 import DynamoDBConnection from boto.dynamodb2.items import Item +from boto.exception import BotoServerError from mock import Mock from nose.tools import eq_, assert_raises, ok_ @@ -153,6 +154,18 @@ def raise_error(*args, **kwargs): result = storage.save_notification(dummy_uaid, dummy_chid, 8) eq_(result, False) + def test_fetch_boto_err(self): + s = get_storage_table() + storage = Storage(s, SinkMetrics()) + storage.table.connection = Mock() + + def raise_error(*args, **kwargs): + raise BotoServerError(None, None) + + storage.table.connection.put_item.side_effect = raise_error + with assert_raises(BotoServerError): + storage.save_notification(dummy_uaid, dummy_chid, 12) + def test_fetch_over_provisioned(self): s = get_storage_table() storage = Storage(s, SinkMetrics()) diff --git a/autopush/tests/test_endpoint.py b/autopush/tests/test_endpoint.py index 077355e0..56dbbb0b 100644 --- a/autopush/tests/test_endpoint.py +++ b/autopush/tests/test_endpoint.py @@ -10,6 +10,7 @@ import ecdsa import jose import twisted.internet.base +from boto.exception import BotoServerError from cryptography.fernet import Fernet, InvalidToken from cyclone.web import Application from jose import jws @@ -595,6 +596,9 @@ def _throw_item_not_found(self, item): def _throw_provisioned_error(self, *args): raise ProvisionedThroughputExceededException(None, None) + def _throw_boto_err(self, *args): + raise BotoServerError(None, None) + def test_process_token_client_unknown(self): self.router_mock.configure_mock(**{ 'get_uaid.side_effect': self._throw_item_not_found}) @@ -1210,6 +1214,18 @@ def handle_finish(result): self.endpoint.put(None, dummy_uaid) return self.finish_deferred + def test_put_boto_error(self): + self.fernet_mock.decrypt.return_value = dummy_token + self.router_mock.get_uaid.side_effect = self._throw_boto_err + + def handle_finish(result): + self.assertTrue(result) + self.endpoint.set_status.assert_called_with(503, None) + self.finish_deferred.addCallback(handle_finish) + + self.endpoint.put(None, dummy_uaid) + return self.finish_deferred + def test_cors(self): ch1 = "Access-Control-Allow-Origin" ch2 = "Access-Control-Allow-Methods" diff --git a/autopush/tests/test_web_base.py b/autopush/tests/test_web_base.py index ee49e07a..7e48cb5f 100644 --- a/autopush/tests/test_web_base.py +++ b/autopush/tests/test_web_base.py @@ -1,6 +1,7 @@ import sys import uuid +from boto.exception import BotoServerError from cyclone.web import Application from mock import Mock, patch from moto import mock_dynamodb2 @@ -211,6 +212,14 @@ def test_overload_err(self): self.base._overload_err(fail) self.status_mock.assert_called_with(503) + def test_boto_err(self): + try: + raise BotoServerError(503, "derp") + except: + fail = Failure() + self.base._boto_err(fail) + self.status_mock.assert_called_with(503) + def test_router_response(self): from autopush.router.interface import RouterResponse response = RouterResponse(headers=dict(Location="http://a.com/")) diff --git a/autopush/tests/test_web_validation.py b/autopush/tests/test_web_validation.py index c49cc6aa..0413bba8 100644 --- a/autopush/tests/test_web_validation.py +++ b/autopush/tests/test_web_validation.py @@ -134,6 +134,7 @@ def get(self): vr.finish = lambda: d.callback(True) vr.write = Mock() vr._overload_err = Mock() + vr._boto_err = Mock() vr._validation_err = Mock() vr._response_err = Mock() vr.ap_settings = Mock() diff --git a/autopush/web/base.py b/autopush/web/base.py index d9877b9c..27d96643 100644 --- a/autopush/web/base.py +++ b/autopush/web/base.py @@ -7,6 +7,7 @@ from boto.dynamodb2.exceptions import ( ProvisionedThroughputExceededException, ) +from boto.exception import BotoServerError from twisted.logger import Logger from twisted.python import failure @@ -196,6 +197,14 @@ def _overload_err(self, fail): self._write_response(503, 201, message="Please slow message send rate") + def _boto_err(self, fail): + """errBack for random boto exceptions""" + fail.trap(BotoServerError) + self.log.info(format="BOTO Error: %s" % str(fail.value), + status_code=503, errno=202, **self._client_info) + self._write_response(503, errno=202, + message="Communication error, please retry") + def _router_response(self, response): for name, val in response.headers.items(): self.set_header(name, val) diff --git a/autopush/web/validation.py b/autopush/web/validation.py index 368a15c0..de555df6 100644 --- a/autopush/web/validation.py +++ b/autopush/web/validation.py @@ -79,6 +79,7 @@ def wrapper(request_handler, *args, **kwargs): d.addCallback(self._call_func, func, request_handler, *args, **kwargs) d.addErrback(request_handler._overload_err) + d.addErrback(request_handler._boto_err) d.addErrback(request_handler._validation_err) d.addErrback(request_handler._response_err) return wrapper @@ -321,8 +322,8 @@ def validate_auth(self, d): try: jwt = extract_jwt(token, public_key) - except (ValueError, JOSEError) as ex: - raise InvalidRequest("Invalid Authorization Header: %s" % str(ex), + except (ValueError, JOSEError): + raise InvalidRequest("Invalid Authorization Header", status_code=401, errno=109, headers={"www-authenticate": PREF_SCHEME}) if jwt.get('exp', 0) < time.time():