bugfix: update plugin refresh triggers, gate actions on plugin status…

…, add/update tests

requirements-test.txt

@@ -7,25 +7,24 @@
 aiohttp==3.6.2
 async-timeout==3.0.1      # via aiohttp
 asynctest==0.13.0
-atomicwrites==1.3.0       # via pytest
 attrs==19.3.0             # via aiohttp, pytest
 chardet==3.0.4            # via aiohttp
-coverage==4.5.4           # via pytest-cov
+coverage==5.0.3           # via pytest-cov
 idna-ssl==1.1.0           # via aiohttp
 idna==2.8                 # via idna-ssl, yarl
-importlib-metadata==0.23  # via pluggy, pytest
-more-itertools==7.2.0     # via pytest, zipp
-multidict==4.5.2          # via aiohttp, yarl
-packaging==19.2           # via pytest
-pluggy==0.13.0            # via pytest
-py==1.8.0                 # via pytest
-pyparsing==2.4.2          # via packaging
+importlib-metadata==1.4.0  # via pluggy, pytest
+more-itertools==8.1.0     # via pytest
+multidict==4.7.4          # via aiohttp, yarl
+packaging==20.1           # via pytest
+pluggy==0.13.1            # via pytest
+py==1.8.1                 # via pytest
+pyparsing==2.4.6          # via packaging
 pytest-asyncio==0.10.0
 pytest-cov==2.8.1
-pytest-mock==1.11.2
-pytest==5.2.2
-six==1.12.0               # via packaging
+pytest-mock==2.0.0
+pytest==5.3.4
+six==1.14.0               # via packaging
 typing-extensions==3.7.4.1  # via aiohttp
-wcwidth==0.1.7            # via pytest
-yarl==1.3.0               # via aiohttp
-zipp==0.6.0               # via importlib-metadata
+wcwidth==0.1.8            # via pytest
+yarl==1.4.2               # via aiohttp
+zipp==2.1.0               # via importlib-metadata

requirements.txt

@@ -7,41 +7,44 @@
 aiocache==0.11.1
 aiofiles==0.4.0           # via sanic
 bison==0.1.2
-cachetools==3.1.1         # via google-auth
-certifi==2019.9.11        # via httpcore, kubernetes, requests
-chardet==3.0.4            # via httpcore, requests
-google-auth==1.6.3        # via kubernetes
-grpcio==1.24.3
-h11==0.8.1                # via httpcore
-h2==3.1.1                 # via httpcore
+cachetools==4.0.0         # via google-auth
+certifi==2019.11.28       # via httpx, kubernetes, requests
+chardet==3.0.4            # via httpx, requests
+contextvars==2.4          # via sniffio
+google-auth==1.11.0       # via kubernetes
+grpcio==1.26.0
+h11==0.8.1                # via httpx
+h2==3.1.1                 # via httpx
 hpack==3.0.0              # via h2
-httpcore==0.3.0           # via requests-async
+hstspreload==2020.1.22    # via httpx
 httptools==0.0.13         # via sanic
+httpx==0.9.3              # via sanic
 hyperframe==5.2.0         # via h2
-idna==2.8                 # via httpcore, requests
+idna==2.8                 # via httpx, requests
+immutables==0.11          # via contextvars
 kubernetes==10.0.1
-multidict==4.5.2          # via sanic
+multidict==4.7.4          # via sanic
 oauthlib==3.1.0           # via requests-oauthlib
 prometheus-client==0.7.1
-protobuf==3.10.0          # via synse-grpc
-pyasn1-modules==0.2.7     # via google-auth
-pyasn1==0.4.7             # via pyasn1-modules, rsa
-python-dateutil==2.8.0    # via kubernetes
-pyyaml==5.1.2
-requests-async==0.5.0     # via sanic
-requests-oauthlib==1.2.0  # via kubernetes
-requests==2.22.0
-rfc3986==1.3.2            # via httpcore
+protobuf==3.11.2          # via synse-grpc
+pyasn1-modules==0.2.8     # via google-auth
+pyasn1==0.4.8             # via pyasn1-modules, rsa
+python-dateutil==2.8.1    # via kubernetes
+pyyaml==5.3
+requests-oauthlib==1.3.0  # via kubernetes
+requests==2.22.0          # via kubernetes, requests-oauthlib
+rfc3986==1.3.2            # via httpx
 rsa==4.0                  # via google-auth
-sanic==19.9.0
-six==1.12.0               # via google-auth, grpcio, kubernetes, protobuf, python-dateutil, structlog, websocket-client
-structlog==19.2.0
+sanic==19.12.2
+six==1.14.0               # via google-auth, grpcio, kubernetes, protobuf, python-dateutil, structlog, websocket-client
+sniffio==1.1.0            # via httpx
+structlog==20.1.0
 synse-grpc==3.0.0a4
 ujson==1.35               # via sanic
-urllib3==1.25.6           # via kubernetes, requests
-uvloop==0.13.0            # via sanic
-websocket-client==0.56.0  # via kubernetes
-websockets==8.0.2
+urllib3==1.25.8           # via kubernetes, requests
+uvloop==0.14.0            # via sanic
+websocket-client==0.57.0  # via kubernetes
+websockets==8.1
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools==41.6.0        # via kubernetes, protobuf
+# setuptools==45.1.0        # via google-auth, kubernetes, protobuf

setup.py

@@ -46,7 +46,6 @@
         'grpcio',
         'kubernetes',
         'pyyaml>=4.2b1',
-        'requests>=2.20.0',  # used by 'kubernetes'
         'sanic>=0.8.0',
         'prometheus-client',
         'structlog',

synse_server/api/http.py

@@ -100,8 +100,12 @@ async def plugins(request: Request) -> HTTPResponse:
     """
     log_request(request)
 
+    refresh = request.args.get('refresh', 'false').lower() == 'true'
+
     return utils.http_json_response(
-        await cmd.plugins(),
+        await cmd.plugins(
+            refresh=refresh,
+        ),
     )
 
 

synse_server/api/websocket.py

@@ -271,10 +271,12 @@ async def handle_request_plugins(self, payload: Payload) -> None:
         Args:
             payload: The message payload received from the WebSocket.
         """
+        refresh = payload.data.get('refresh', False)
+
         await self.send(
             id=payload.id,
             event='response/plugin_summary',
-            data=await cmd.plugins(),
+            data=await cmd.plugins(refresh=refresh),
         )
 
     async def handle_request_plugin_health(self, payload: Payload) -> None:

synse_server/cache.py

@@ -156,6 +156,12 @@ async def update_device_cache() -> None:
         await device_cache.clear()
 
         for p in plugin.manager:
+            if not p.active:
+                logger.debug(
+                    _('plugin not active, will not get its devices'),
+                    plugin=p.tag, plugin_id=p.id,
+                )
+                continue
             logger.debug(_('getting devices from plugin'), plugin=p.tag, plugin_id=p.id)
             try:
                 with p as client:
@@ -175,6 +181,7 @@ async def update_device_cache() -> None:
 
             except grpc.RpcError as e:
                 logger.warning(_('failed to get device(s)'), plugin=p.tag, plugin_id=p.id, error=e)
+                continue
 
 
 async def get_device(device_id: str) -> Union[api.V3Device, None]:

synse_server/cmd/plugin.py

@@ -52,17 +52,20 @@ async def plugin(plugin_id: str) -> Dict[str, Any]:
     return response
 
 
-async def plugins() -> List[Dict[str, Any]]:
+async def plugins(refresh: bool) -> List[Dict[str, Any]]:
     """Generate the plugin summary response data.
 
+    Args:
+        refresh: Force a refresh of the registered plugins.
+
     Returns:
         A list of dictionary representations of the plugin summary response(s).
     """
     logger.info(_('issuing command'), command='PLUGINS')
 
     # If there are no plugins registered, re-registering to ensure
     # the most up-to-date plugin state.
-    if not manager.has_plugins():
+    if refresh or not manager.has_plugins():
         manager.refresh()
 
     summaries = []

synse_server/cmd/read.py

@@ -67,6 +67,13 @@ async def read(ns: str, tag_groups: Union[List[str], List[List[str]]]) -> List[D
         logger.debug(_('no tags specified, reading with no tag filter'), command='READ')
         readings = []
         for p in plugin.manager:
+            if not p.active:
+                logger.debug(
+                    _('plugin not active, will not read its devices'),
+                    plugin=p.tag, plugin_id=p.id,
+                )
+                continue
+
             try:
                 with p as client:
                     data = client.read()
@@ -96,6 +103,13 @@ async def read(ns: str, tag_groups: Union[List[str], List[List[str]]]) -> List[D
                 group[i] = f'{ns}/{tag}'
 
         for p in plugin.manager:
+            if not p.active:
+                logger.debug(
+                    _('plugin not active, will not read its devices'),
+                    plugin=p.tag, plugin_id=p.id,
+                )
+                continue
+
             try:
                 with p as client:
                     data = client.read(tags=group)
@@ -163,6 +177,13 @@ async def read_cache(start: str = None, end: str = None) -> AsyncIterable:
 
     # FIXME: this could benefit from being async
     for p in plugin.manager:
+        if not p.active:
+            logger.debug(
+                _('plugin not active, will not read its devices'),
+                plugin=p.tag, plugin_id=p.id,
+            )
+            continue
+
         logger.debug(_('getting cached readings for plugin'), plugin=p.tag, command='READ CACHE')
         try:
             with p as client:
@@ -263,6 +284,13 @@ async def read_stream(
 
     threads = []
     for p in plugin.manager:
+        if not p.active:
+            logger.debug(
+                _('plugin not active, will not read its devices'),
+                plugin=p.tag, plugin_id=p.id,
+            )
+            continue
+
         t = Stream(p, ids, tag_groups, q)
         t.start()
         threads.append(t)