feat: update to plugin reconnect mechanics

.jenkins

@@ -20,12 +20,16 @@ pipeline {
     }
 
     stage('Test') {
+      agent {
+        label 'python-3.8'
+      }
       environment {
         CODECOV_TOKEN = credentials('codecov-token')
       }
       steps {
-        sh 'tox tests/unit'
-        sh 'codecov'
+        container('python') {
+          sh 'tox tests/unit'
+        }
       }
     }
 

requirements-test.txt

@@ -4,27 +4,28 @@
 #
 #    pip-compile --output-file=requirements-test.txt test-requirements.in
 #
-aiohttp==3.6.2
+aiohttp==3.6.2            # via -r test-requirements.in
 async-timeout==3.0.1      # via aiohttp
-asynctest==0.13.0
+asynctest==0.13.0         # via -r test-requirements.in
 attrs==19.3.0             # via aiohttp, pytest
 chardet==3.0.4            # via aiohttp
 coverage==5.0.3           # via pytest-cov
 idna-ssl==1.1.0           # via aiohttp
 idna==2.9                 # via idna-ssl, yarl
 importlib-metadata==1.5.0  # via pluggy, pytest
+mock==4.0.1               # via -r test-requirements.in
 more-itertools==8.2.0     # via pytest
-multidict==4.7.4          # via aiohttp, yarl
-packaging==20.1           # via pytest
+multidict==4.7.5          # via aiohttp, yarl
+packaging==20.3           # via pytest
 pluggy==0.13.1            # via pytest
 py==1.8.1                 # via pytest
 pyparsing==2.4.6          # via packaging
-pytest-asyncio==0.10.0
-pytest-cov==2.8.1
-pytest-mock==2.0.0
-pytest==5.3.5
+pytest-asyncio==0.10.0    # via -r test-requirements.in
+pytest-cov==2.8.1         # via -r test-requirements.in
+pytest-mock==2.0.0        # via -r test-requirements.in
+pytest==5.3.5             # via -r test-requirements.in, pytest-asyncio, pytest-cov, pytest-mock
 six==1.14.0               # via packaging
 typing-extensions==3.7.4.1  # via aiohttp
 wcwidth==0.1.8            # via pytest
 yarl==1.4.2               # via aiohttp
-zipp==3.0.0               # via importlib-metadata
+zipp==3.1.0               # via importlib-metadata

requirements.txt

@@ -4,48 +4,48 @@
 #
 #    pip-compile --output-file=requirements.txt setup.py
 #
-aiocache==0.11.1
+aiocache==0.11.1          # via synse_server (setup.py)
 aiofiles==0.4.0           # via sanic
-bison==0.1.2
+bison==0.1.2              # via synse_server (setup.py)
 cachetools==4.0.0         # via google-auth
 certifi==2019.11.28       # via httpx, kubernetes, requests
 chardet==3.0.4            # via httpx, requests
 contextvars==2.4          # via sniffio
 google-auth==1.11.2       # via kubernetes
-grpcio==1.27.2
+grpcio==1.27.2            # via synse-grpc, synse_server (setup.py)
 h11==0.8.1                # via httpx
 h2==3.2.0                 # via httpx
 hpack==3.0.0              # via h2
-hstspreload==2020.2.20    # via httpx
+hstspreload==2020.3.4     # via httpx
 httptools==0.1.1          # via sanic
 httpx==0.9.3              # via sanic
 hyperframe==5.2.0         # via h2
 idna==2.9                 # via httpx, requests
 immutables==0.11          # via contextvars
-kubernetes==10.0.1
-multidict==4.7.4          # via sanic
+kubernetes==10.0.1        # via synse_server (setup.py)
+multidict==4.7.5          # via sanic
 oauthlib==3.1.0           # via requests-oauthlib
-prometheus-client==0.7.1
+prometheus-client==0.7.1  # via synse_server (setup.py)
 protobuf==3.11.3          # via synse-grpc
 pyasn1-modules==0.2.8     # via google-auth
 pyasn1==0.4.8             # via pyasn1-modules, rsa
 python-dateutil==2.8.1    # via kubernetes
-pyyaml==5.3
+pyyaml==5.3               # via bison, kubernetes, synse_server (setup.py)
 requests-oauthlib==1.3.0  # via kubernetes
 requests==2.23.0          # via kubernetes, requests-oauthlib
 rfc3986==1.3.2            # via httpx
 rsa==4.0                  # via google-auth
-sanic==19.12.2
-shortuuid==0.5.0
+sanic==19.12.2            # via synse_server (setup.py)
+shortuuid==1.0.1          # via synse_server (setup.py)
 six==1.14.0               # via google-auth, grpcio, kubernetes, protobuf, python-dateutil, structlog, websocket-client
 sniffio==1.1.0            # via httpx
-structlog==20.1.0
-synse-grpc==3.0.0a4
-ujson==1.35               # via sanic
+structlog==20.1.0         # via synse_server (setup.py)
+synse-grpc==3.0.0a4       # via synse_server (setup.py)
+ujson==2.0.1              # via sanic
 urllib3==1.25.8           # via kubernetes, requests
 uvloop==0.14.0            # via sanic
 websocket-client==0.57.0  # via kubernetes
-websockets==8.1
+websockets==8.1           # via sanic, synse_server (setup.py)
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools==45.2.0        # via google-auth, kubernetes, protobuf
+# setuptools

synse_server/backoff.py

@@ -0,0 +1,56 @@
+"""Retry backoff strategies."""
+
+import random
+import time
+from typing import Union
+
+__all__ = ['ExponentialBackoff']
+
+
+class ExponentialBackoff:
+    """An implementation of the exponential backoff strategy.
+
+    This is useful for getting an exponentially backed-off delay for
+    reconnection or retry actions.
+
+    Each call to ``delay`` will return the next exponentially backed-off
+    value, in seconds, to use for waiting. The backoff will continue for
+    each call, up to a maximum of 2^10 * base.
+
+    Args:
+        base: The base delay, in seconds. This is the starting point for
+            the returned exponentially backed off time values.
+        cap: The cap on the exponent, after which the backoff will not
+            grow exponentially. This is 9 by default (2^9 = 512 ~= 8.5 minutes)
+    """
+
+    def __init__(self, base: int = 1, cap: int = 9) -> None:
+        self._base = base
+        self._exp = 0
+        self._max = cap
+
+        self._reset_time = base * 2 ** (self._max + 1)
+        self._last_invocation = time.monotonic()
+
+        self.rand = random.Random()
+        self.rand.seed()
+
+    def delay(self) -> Union[int, float]:
+        """Get the next exponentially backed off time delay, in seconds.
+
+        The delay value is incremented exponentially with every call, up
+        to the defined max. If a period of time greater than 2^(max+1) * base
+        has passed, the backoff is reset.
+
+        Returns:
+            The time, in seconds, to be used as the next delay interval.
+        """
+        invocation = time.monotonic()
+        interval = invocation - self._last_invocation
+        self._last_invocation = invocation
+
+        if interval > self._reset_time:
+            self._exp = 0
+
+        self._exp = min(self._exp + 1, self._max)
+        return self.rand.uniform(0, self._base * 2 ** self._exp)

synse_server/cache.py

@@ -149,7 +149,7 @@ async def update_device_cache() -> None:
     # marked inactive, attempt to refresh all plugins. This can be the case when
     # Synse Server is first starting up, being restarted, or is recovering from a
     # networking error.
-    if not plugin.manager.has_plugins() or not plugin.manager.all_active():
+    if not plugin.manager.has_plugins() or not plugin.manager.all_ready():
         logger.debug('refreshing plugins prior to updating device cache')
         plugin.manager.refresh()