Merge pull request #132 from prkumar/io

Add Retry and Ratelimit decorators

.coveragerc

@@ -12,6 +12,7 @@ omit =
     uplink/__about__.py
     uplink/interfaces.py
     uplink/clients/interfaces.py
+    uplink/clients/io/interfaces.py
     uplink/converters/interfaces.py
 
 [report]

docs/source/dev/decorators.rst

@@ -66,3 +66,15 @@ data serialization format for any consumer method.
 
 .. automodule:: uplink.returns
     :members:
+
+
+retry
+=====
+
+.. autoclass:: uplink.retry
+    :members:
+
+ratelimit
+=========
+
+.. autoclass:: uplink.ratelimit

setup.py

@@ -24,7 +24,7 @@ def read(filename):
     "twisted:python_version != '3.3'": "twisted>=17.1.0",
     "twisted:python_version == '3.3'": "twisted<=17.9.0",
     "typing": ["typing>=3.6.4"],
-    "tests": ["pytest", "pytest-mock", "pytest-cov"],
+    "tests": ["pytest", "pytest-mock", "pytest-cov", "pytest-twisted"],
 }
 
 metadata = {

tests/__init__.py

@@ -0,0 +1,9 @@
+# Standard library imports
+import sys
+
+# Third-party import
+import pytest
+
+requires_python34 = pytest.mark.skipif(
+    sys.version_info < (3, 4), reason="Requires Python 3.4 or above."
+)

tests/integration/__init__.py

@@ -1,13 +1,14 @@
 # Local imports
 from uplink import utils, clients
-from uplink.clients import helpers, exceptions as client_exceptions
+from uplink.clients import helpers, io, exceptions as client_exceptions
 
 
 class MockClient(clients.interfaces.HttpClientAdapter):
     def __init__(self, request):
         self._mocked_request = request
         self._request = _HistoryMaintainingRequest(_MockRequest(request))
         self._exceptions = client_exceptions.Exceptions()
+        self._io = io.BlockingStrategy()
 
     def create_request(self):
         return self._request
@@ -16,8 +17,8 @@ def with_response(self, response):
         self._mocked_request.send.return_value = response
         return self
 
-    def with_side_effect(self, error):
-        self._mocked_request.send.side_effect = error
+    def with_side_effect(self, side_effect):
+        self._mocked_request.send.side_effect = side_effect
         return self
 
     @property
@@ -28,6 +29,13 @@ def exceptions(self):
     def history(self):
         return self._request.history
 
+    def with_io(self, io_):
+        self._io = io_
+        return self
+
+    def io(self):
+        return self._io
+
 
 class MockResponse(object):
     def __init__(self, response):

tests/integration/test_ratelimit.py

@@ -0,0 +1,43 @@
+# Local imports.
+import uplink
+from uplink.ratelimit import now
+
+# Constants
+BASE_URL = "https://api.github.com/"
+
+
+class GitHub(uplink.Consumer):
+    @uplink.ratelimit(calls=1, period=1)
+    @uplink.get("/users/{user}")
+    def get_user(self, user):
+        pass
+
+
+# Tests
+
+
+def test_limit_exceeded_by_1(mock_client):
+    # Setup
+    github = GitHub(base_url=BASE_URL, client=mock_client)
+
+    # Run
+    start_time = now()
+    github.get_user("prkumar")
+    github.get_user("prkumar")
+    elapsed_time = now() - start_time
+
+    # Verify
+    assert elapsed_time >= 1
+
+
+def test_exact_limit(mock_client):
+    # Setup
+    github = GitHub(base_url=BASE_URL, client=mock_client)
+
+    # Run
+    start_time = now()
+    github.get_user("prkumar")
+    elapsed_time = now() - start_time
+
+    # Verify
+    assert elapsed_time <= 1

tests/integration/test_retry.py

@@ -0,0 +1,175 @@
+# Third-party imports
+import pytest
+import pytest_twisted
+
+# Local imports.
+import uplink
+from uplink.clients import io
+from tests import requires_python34
+
+# Constants
+BASE_URL = "https://api.github.com/"
+
+
+def wait_once():
+    yield 0.1
+
+
+wait_default = uplink.retry.exponential_backoff(multiplier=0.1, minimum=0.1)
+
+
+class GitHub(uplink.Consumer):
+    @uplink.retry(max_attempts=2, wait=wait_default)
+    @uplink.get("/users/{user}")
+    def get_user(self, user):
+        pass
+
+    @uplink.retry(max_attempts=3, wait=wait_once)
+    @uplink.get("/{user}/{repo}/{issue}")
+    def get_issue(self, user, repo, issue):
+        pass
+
+    @uplink.retry(max_attempts=3, when_raises=uplink.retry.CONNECTION_TIMEOUT)
+    @uplink.get("/{user}/{repo}/{project}")
+    def get_project(self, user, repo, project):
+        pass
+
+
+# Tests
+
+
+def test_retry(mock_client, mock_response):
+    # Setup
+    mock_response.with_json({"id": 123, "name": "prkumar"})
+    mock_client.with_side_effect([Exception, mock_response])
+    github = GitHub(base_url=BASE_URL, client=mock_client)
+
+    # Run
+    response = github.get_user("prkumar")
+
+    # Verify
+    assert len(mock_client.history) == 2
+    assert response.json() == {"id": 123, "name": "prkumar"}
+
+
+def test_retry_fail(mock_client, mock_response):
+    # Setup
+    mock_response.with_json({"id": 123, "name": "prkumar"})
+    mock_client.with_side_effect([Exception, Exception, mock_response])
+    github = GitHub(base_url=BASE_URL, client=mock_client)
+
+    # Run
+    with pytest.raises(Exception):
+        github.get_issue("prkumar", "uplink", "#1")
+
+    # Verify
+    assert len(mock_client.history) == 2
+
+
+def test_retry_fail_with_client_exception(mock_client, mock_response):
+    # Setup
+    mock_response.with_json({"id": 123, "name": "prkumar"})
+    mock_client.exceptions.ConnectionTimeout = type(
+        "ConnectionTimeout", (Exception,), {}
+    )
+    CustomException = type("CustomException", (Exception,), {})
+    mock_client.with_side_effect(
+        [
+            mock_client.exceptions.ConnectionTimeout,
+            CustomException,
+            mock_response,
+        ]
+    )
+    github = GitHub(base_url=BASE_URL, client=mock_client)
+
+    # Run
+    with pytest.raises(CustomException):
+        github.get_project("prkumar", "uplink", "1")
+
+    # Verify
+    assert len(mock_client.history) == 2
+
+
+def test_retry_fail_because_of_wait(mock_client, mock_response):
+    # Setup
+    mock_response.with_json({"id": 123, "name": "prkumar"})
+    mock_client.with_side_effect([Exception, Exception, mock_response])
+    github = GitHub(base_url=BASE_URL, client=mock_client)
+
+    # Run
+    with pytest.raises(Exception):
+        github.get_issue("prkumar", "uplink", "#1")
+
+    # Verify
+    assert len(mock_client.history) == 2
+
+
+@requires_python34
+def test_retry_with_asyncio(mock_client, mock_response):
+    import asyncio
+
+    @asyncio.coroutine
+    def coroutine():
+        return mock_response
+
+    # Setup
+    mock_response.with_json({"id": 123, "name": "prkumar"})
+    mock_client.with_side_effect([Exception, coroutine()])
+    mock_client.with_io(io.AsyncioStrategy())
+    github = GitHub(base_url=BASE_URL, client=mock_client)
+
+    # Run
+    awaitable = github.get_user("prkumar")
+    loop = asyncio.get_event_loop()
+    response = loop.run_until_complete(asyncio.ensure_future(awaitable))
+
+    # Verify
+    assert len(mock_client.history) == 2
+    assert response.json() == {"id": 123, "name": "prkumar"}
+
+
+@pytest_twisted.inlineCallbacks
+def test_retry_with_twisted(mock_client, mock_response):
+    from twisted.internet import defer
+
+    @defer.inlineCallbacks
+    def return_response():
+        yield
+        defer.returnValue(mock_response)
+
+    # Setup
+    mock_response.with_json({"id": 123, "name": "prkumar"})
+    mock_client.with_side_effect([Exception, return_response()])
+    mock_client.with_io(io.TwistedStrategy())
+    github = GitHub(base_url=BASE_URL, client=mock_client)
+
+    # Run
+    response = yield github.get_user("prkumar")
+
+    assert len(mock_client.history) == 2
+    assert response.json() == {"id": 123, "name": "prkumar"}
+
+
+@pytest_twisted.inlineCallbacks
+def test_retry_fail_with_twisted(mock_client, mock_response):
+    from twisted.internet import defer
+
+    @defer.inlineCallbacks
+    def return_response():
+        yield
+        defer.returnValue(mock_response)
+
+    # Setup
+    CustomException = type("CustomException", (Exception,), {})
+    mock_response.with_json({"id": 123, "name": "prkumar"})
+    mock_client.with_side_effect(
+        [Exception, CustomException, return_response()]
+    )
+    mock_client.with_io(io.TwistedStrategy())
+    github = GitHub(base_url=BASE_URL, client=mock_client)
+
+    # Run
+    with pytest.raises(CustomException):
+        yield github.get_user("prkumar")
+
+    assert len(mock_client.history) == 2

tests/unit/conftest.py

@@ -6,6 +6,7 @@
 
 # Local imports
 from uplink import clients, converters, hooks, interfaces, helpers
+from uplink.clients.exceptions import Exceptions
 
 
 @pytest.fixture
@@ -70,4 +71,5 @@ def request_builder(mocker):
     builder = mocker.MagicMock(spec=helpers.RequestBuilder)
     builder.info = collections.defaultdict(dict)
     builder.get_converter.return_value = lambda x: x
+    builder.client.exceptions = Exceptions()
     return builder

tests/unit/test_builder.py

@@ -27,29 +27,35 @@ def test_prepare_request(self, mocker, request_builder):
         request_builder.url = "/example/path"
         request_builder.return_type = None
         request_builder.transaction_hooks = ()
+        request_builder.request_template = "request_template"
         uplink_builder = mocker.Mock(spec=builder.Builder)
         uplink_builder.converters = ()
         uplink_builder.hooks = ()
         request_preparer = builder.RequestPreparer(uplink_builder)
         request_preparer.prepare_request(request_builder)
-        uplink_builder.client.create_request().send.assert_called_with(
-            request_builder.method, request_builder.url, request_builder.info
+        uplink_builder.client.send.assert_called_with(
+            uplink_builder.client.create_request(),
+            request_builder.request_template,
+            (request_builder.method, request_builder.url, request_builder.info),
         )
 
     def test_prepare_request_with_transaction_hook(
         self, uplink_builder, request_builder, transaction_hook_mock
     ):
         request_builder.method = "METHOD"
         request_builder.url = "/example/path"
+        request_builder.request_template = "request_template"
         uplink_builder.base_url = "https://example.com"
         uplink_builder.add_hook(transaction_hook_mock)
         request_preparer = builder.RequestPreparer(uplink_builder)
         request_preparer.prepare_request(request_builder)
         transaction_hook_mock.audit_request.assert_called_with(
             None, request_builder
         )
-        uplink_builder.client.create_request().send.assert_called_with(
-            request_builder.method, request_builder.url, request_builder.info
+        uplink_builder.client.send.assert_called_with(
+            uplink_builder.client.create_request(),
+            request_builder.request_template,
+            (request_builder.method, request_builder.url, request_builder.info),
         )
 
     def test_create_request_builder(self, uplink_builder, request_definition):

tests/unit/test_clients.py

@@ -11,6 +11,7 @@
     requests_,
     twisted_,
     register,
+    io,
 )
 
 try:
@@ -125,6 +126,9 @@ def test_exceptions(self):
         with pytest.raises(exceptions.InvalidURL):
             raise requests.exceptions.InvalidURL()
 
+    def test_io(self):
+        assert isinstance(requests_.RequestsClient.io(), io.BlockingStrategy)
+
 
 class TestTwisted(object):
     def test_init_without_client(self):
@@ -183,6 +187,13 @@ def test_handle_failure(self, mocker, request_mock):
             failure.type, failure.value, failure.getTracebackObject()
         )
 
+    def test_exceptions(self, http_client_mock):
+        twisted_client = twisted_.TwistedClient(http_client_mock)
+        assert http_client_mock.exceptions == twisted_client.exceptions
+
+    def test_io(self):
+        assert isinstance(twisted_.TwistedClient.io(), io.TwistedStrategy)
+
 
 @pytest.fixture
 def aiohttp_session_mock(mocker):
@@ -404,3 +415,7 @@ def test_exceptions(self):
 
         with pytest.raises(exceptions.InvalidURL):
             raise aiohttp.InvalidURL("invalid")
+
+    @requires_python34
+    def test_io(self):
+        assert isinstance(aiohttp_.AiohttpClient.io(), io.AsyncioStrategy)