diff --git a/requirements-test.txt b/requirements-test.txt index 8dceaf7b..4798db47 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -7,25 +7,24 @@ aiohttp==3.6.2 async-timeout==3.0.1 # via aiohttp asynctest==0.13.0 -atomicwrites==1.3.0 # via pytest attrs==19.3.0 # via aiohttp, pytest chardet==3.0.4 # via aiohttp -coverage==4.5.4 # via pytest-cov +coverage==5.0.3 # via pytest-cov idna-ssl==1.1.0 # via aiohttp idna==2.8 # via idna-ssl, yarl -importlib-metadata==0.23 # via pluggy, pytest -more-itertools==7.2.0 # via pytest, zipp -multidict==4.5.2 # via aiohttp, yarl -packaging==19.2 # via pytest -pluggy==0.13.0 # via pytest -py==1.8.0 # via pytest -pyparsing==2.4.2 # via packaging +importlib-metadata==1.4.0 # via pluggy, pytest +more-itertools==8.1.0 # via pytest +multidict==4.7.4 # via aiohttp, yarl +packaging==20.1 # via pytest +pluggy==0.13.1 # via pytest +py==1.8.1 # via pytest +pyparsing==2.4.6 # via packaging pytest-asyncio==0.10.0 pytest-cov==2.8.1 -pytest-mock==1.11.2 -pytest==5.2.2 -six==1.12.0 # via packaging +pytest-mock==2.0.0 +pytest==5.3.4 +six==1.14.0 # via packaging typing-extensions==3.7.4.1 # via aiohttp -wcwidth==0.1.7 # via pytest -yarl==1.3.0 # via aiohttp -zipp==0.6.0 # via importlib-metadata +wcwidth==0.1.8 # via pytest +yarl==1.4.2 # via aiohttp +zipp==2.1.0 # via importlib-metadata diff --git a/requirements.txt b/requirements.txt index 77133060..8bf3ea1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,41 +7,44 @@ aiocache==0.11.1 aiofiles==0.4.0 # via sanic bison==0.1.2 -cachetools==3.1.1 # via google-auth -certifi==2019.9.11 # via httpcore, kubernetes, requests -chardet==3.0.4 # via httpcore, requests -google-auth==1.6.3 # via kubernetes -grpcio==1.24.3 -h11==0.8.1 # via httpcore -h2==3.1.1 # via httpcore +cachetools==4.0.0 # via google-auth +certifi==2019.11.28 # via httpx, kubernetes, requests +chardet==3.0.4 # via httpx, requests +contextvars==2.4 # via sniffio +google-auth==1.11.0 # via kubernetes +grpcio==1.26.0 +h11==0.8.1 # via httpx +h2==3.1.1 # via httpx hpack==3.0.0 # via h2 -httpcore==0.3.0 # via requests-async +hstspreload==2020.1.22 # via httpx httptools==0.0.13 # via sanic +httpx==0.9.3 # via sanic hyperframe==5.2.0 # via h2 -idna==2.8 # via httpcore, requests +idna==2.8 # via httpx, requests +immutables==0.11 # via contextvars kubernetes==10.0.1 -multidict==4.5.2 # via sanic +multidict==4.7.4 # via sanic oauthlib==3.1.0 # via requests-oauthlib prometheus-client==0.7.1 -protobuf==3.10.0 # via synse-grpc -pyasn1-modules==0.2.7 # via google-auth -pyasn1==0.4.7 # via pyasn1-modules, rsa -python-dateutil==2.8.0 # via kubernetes -pyyaml==5.1.2 -requests-async==0.5.0 # via sanic -requests-oauthlib==1.2.0 # via kubernetes -requests==2.22.0 -rfc3986==1.3.2 # via httpcore +protobuf==3.11.2 # via synse-grpc +pyasn1-modules==0.2.8 # via google-auth +pyasn1==0.4.8 # via pyasn1-modules, rsa +python-dateutil==2.8.1 # via kubernetes +pyyaml==5.3 +requests-oauthlib==1.3.0 # via kubernetes +requests==2.22.0 # via kubernetes, requests-oauthlib +rfc3986==1.3.2 # via httpx rsa==4.0 # via google-auth -sanic==19.9.0 -six==1.12.0 # via google-auth, grpcio, kubernetes, protobuf, python-dateutil, structlog, websocket-client -structlog==19.2.0 +sanic==19.12.2 +six==1.14.0 # via google-auth, grpcio, kubernetes, protobuf, python-dateutil, structlog, websocket-client +sniffio==1.1.0 # via httpx +structlog==20.1.0 synse-grpc==3.0.0a4 ujson==1.35 # via sanic -urllib3==1.25.6 # via kubernetes, requests -uvloop==0.13.0 # via sanic -websocket-client==0.56.0 # via kubernetes -websockets==8.0.2 +urllib3==1.25.8 # via kubernetes, requests +uvloop==0.14.0 # via sanic +websocket-client==0.57.0 # via kubernetes +websockets==8.1 # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.6.0 # via kubernetes, protobuf +# setuptools==45.1.0 # via google-auth, kubernetes, protobuf diff --git a/setup.py b/setup.py index 6b894b6e..bb4360a8 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,6 @@ 'grpcio', 'kubernetes', 'pyyaml>=4.2b1', - 'requests>=2.20.0', # used by 'kubernetes' 'sanic>=0.8.0', 'prometheus-client', 'structlog', diff --git a/synse_server/api/http.py b/synse_server/api/http.py index 1e8a4bd3..94ed52d8 100644 --- a/synse_server/api/http.py +++ b/synse_server/api/http.py @@ -100,8 +100,12 @@ async def plugins(request: Request) -> HTTPResponse: """ log_request(request) + refresh = request.args.get('refresh', 'false').lower() == 'true' + return utils.http_json_response( - await cmd.plugins(), + await cmd.plugins( + refresh=refresh, + ), ) diff --git a/synse_server/api/websocket.py b/synse_server/api/websocket.py index 0204a3be..9f80c497 100644 --- a/synse_server/api/websocket.py +++ b/synse_server/api/websocket.py @@ -271,10 +271,12 @@ async def handle_request_plugins(self, payload: Payload) -> None: Args: payload: The message payload received from the WebSocket. """ + refresh = payload.data.get('refresh', False) + await self.send( id=payload.id, event='response/plugin_summary', - data=await cmd.plugins(), + data=await cmd.plugins(refresh=refresh), ) async def handle_request_plugin_health(self, payload: Payload) -> None: diff --git a/synse_server/cache.py b/synse_server/cache.py index 6c6b5e63..16f93bb0 100644 --- a/synse_server/cache.py +++ b/synse_server/cache.py @@ -156,6 +156,12 @@ async def update_device_cache() -> None: await device_cache.clear() for p in plugin.manager: + if not p.active: + logger.debug( + _('plugin not active, will not get its devices'), + plugin=p.tag, plugin_id=p.id, + ) + continue logger.debug(_('getting devices from plugin'), plugin=p.tag, plugin_id=p.id) try: with p as client: @@ -175,6 +181,7 @@ async def update_device_cache() -> None: except grpc.RpcError as e: logger.warning(_('failed to get device(s)'), plugin=p.tag, plugin_id=p.id, error=e) + continue async def get_device(device_id: str) -> Union[api.V3Device, None]: diff --git a/synse_server/cmd/plugin.py b/synse_server/cmd/plugin.py index 5276518e..fb3ffad6 100644 --- a/synse_server/cmd/plugin.py +++ b/synse_server/cmd/plugin.py @@ -52,9 +52,12 @@ async def plugin(plugin_id: str) -> Dict[str, Any]: return response -async def plugins() -> List[Dict[str, Any]]: +async def plugins(refresh: bool) -> List[Dict[str, Any]]: """Generate the plugin summary response data. + Args: + refresh: Force a refresh of the registered plugins. + Returns: A list of dictionary representations of the plugin summary response(s). """ @@ -62,7 +65,7 @@ async def plugins() -> List[Dict[str, Any]]: # If there are no plugins registered, re-registering to ensure # the most up-to-date plugin state. - if not manager.has_plugins(): + if refresh or not manager.has_plugins(): manager.refresh() summaries = [] diff --git a/synse_server/cmd/read.py b/synse_server/cmd/read.py index 5501d9db..8640d85e 100644 --- a/synse_server/cmd/read.py +++ b/synse_server/cmd/read.py @@ -67,6 +67,13 @@ async def read(ns: str, tag_groups: Union[List[str], List[List[str]]]) -> List[D logger.debug(_('no tags specified, reading with no tag filter'), command='READ') readings = [] for p in plugin.manager: + if not p.active: + logger.debug( + _('plugin not active, will not read its devices'), + plugin=p.tag, plugin_id=p.id, + ) + continue + try: with p as client: data = client.read() @@ -96,6 +103,13 @@ async def read(ns: str, tag_groups: Union[List[str], List[List[str]]]) -> List[D group[i] = f'{ns}/{tag}' for p in plugin.manager: + if not p.active: + logger.debug( + _('plugin not active, will not read its devices'), + plugin=p.tag, plugin_id=p.id, + ) + continue + try: with p as client: data = client.read(tags=group) @@ -163,6 +177,13 @@ async def read_cache(start: str = None, end: str = None) -> AsyncIterable: # FIXME: this could benefit from being async for p in plugin.manager: + if not p.active: + logger.debug( + _('plugin not active, will not read its devices'), + plugin=p.tag, plugin_id=p.id, + ) + continue + logger.debug(_('getting cached readings for plugin'), plugin=p.tag, command='READ CACHE') try: with p as client: @@ -263,6 +284,13 @@ async def read_stream( threads = [] for p in plugin.manager: + if not p.active: + logger.debug( + _('plugin not active, will not read its devices'), + plugin=p.tag, plugin_id=p.id, + ) + continue + t = Stream(p, ids, tag_groups, q) t.start() threads.append(t) diff --git a/synse_server/locale/en_US/LC_MESSAGES/synse_server.po b/synse_server/locale/en_US/LC_MESSAGES/synse_server.po index f1841305..bcc002f0 100644 --- a/synse_server/locale/en_US/LC_MESSAGES/synse_server.po +++ b/synse_server/locale/en_US/LC_MESSAGES/synse_server.po @@ -1,15 +1,15 @@ # English (United States) translations for Synse Server. -# Copyright (C) 2019 Vapor IO +# Copyright (C) 2020 Vapor IO # This file is distributed under the same license as the Synse Server # project. -# FIRST AUTHOR , 2019. +# FIRST AUTHOR , 2020. # msgid "" msgstr "" "Project-Id-Version: Synse Server VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2019-11-15 14:09-0500\n" -"PO-Revision-Date: 2019-11-15 14:09-0500\n" +"POT-Creation-Date: 2020-01-28 11:17-0500\n" +"PO-Revision-Date: 2020-01-28 11:17-0500\n" "Last-Translator: FULL NAME \n" "Language: en_US\n" "Language-Team: en_US \n" @@ -19,7 +19,7 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.7.0\n" -#: synse_server/cache.py:84 +#: synse_server/cache.py:85 msgid "caching transaction" msgstr "" @@ -35,91 +35,95 @@ msgstr "" msgid "no plugins found when updating device cache" msgstr "" -#: synse_server/cache.py:159 +#: synse_server/cache.py:161 +msgid "plugin not active, will not get its devices" +msgstr "" + +#: synse_server/cache.py:165 msgid "getting devices from plugin" msgstr "" -#: synse_server/cache.py:177 +#: synse_server/cache.py:183 msgid "failed to get device(s)" msgstr "" -#: synse_server/cache.py:200 +#: synse_server/cache.py:207 msgid "looking up device ID in cache" msgstr "" -#: synse_server/cache.py:214 +#: synse_server/cache.py:221 msgid "device ID not found in cache - checking for alias" msgstr "" -#: synse_server/errors.py:52 +#: synse_server/errors.py:54 msgid "an unexpected error occurred" msgstr "" -#: synse_server/errors.py:76 +#: synse_server/errors.py:78 msgid "invalid user input" msgstr "" -#: synse_server/errors.py:88 +#: synse_server/errors.py:90 msgid "resource not found" msgstr "" -#: synse_server/errors.py:100 +#: synse_server/errors.py:102 msgid "device action not supported" msgstr "" -#: synse_server/errors.py:112 +#: synse_server/errors.py:114 msgid "error processing the request" msgstr "" -#: synse_server/plugin.py:76 +#: synse_server/plugin.py:77 msgid "registering new plugin" msgstr "" -#: synse_server/plugin.py:80 +#: synse_server/plugin.py:81 msgid "application metrics enabled: registering gRPC interceptor" msgstr "" -#: synse_server/plugin.py:106 +#: synse_server/plugin.py:107 msgid "plugin with id already registered - skipping" msgstr "" -#: synse_server/plugin.py:109 +#: synse_server/plugin.py:110 msgid "successfully registered plugin" msgstr "" -#: synse_server/plugin.py:124 +#: synse_server/plugin.py:125 msgid "loading plugins from configuration" msgstr "" -#: synse_server/plugin.py:131 synse_server/plugin.py:137 +#: synse_server/plugin.py:132 synse_server/plugin.py:138 msgid "loading plugin from config" msgstr "" -#: synse_server/plugin.py:157 +#: synse_server/plugin.py:158 msgid "failed plugin discovery via Kubernetes" msgstr "" -#: synse_server/plugin.py:174 +#: synse_server/plugin.py:175 msgid "refreshing plugin manager" msgstr "" -#: synse_server/plugin.py:186 +#: synse_server/plugin.py:187 msgid "failed to register configured plugin - will attempt re-registering later" msgstr "" -#: synse_server/plugin.py:200 +#: synse_server/plugin.py:201 msgid "failed to register discovered plugin - will attempt re-registering later" msgstr "" -#: synse_server/plugin.py:206 +#: synse_server/plugin.py:207 msgid "plugin manager refresh complete" msgstr "" -#: synse_server/plugin.py:283 +#: synse_server/plugin.py:282 msgid "marking plugin as active" msgstr "" -#: synse_server/plugin.py:290 +#: synse_server/plugin.py:289 msgid "marking plugin as inactive" msgstr "" @@ -163,180 +167,193 @@ msgstr "" msgid "serving API endpoints" msgstr "" -#: synse_server/tasks.py:18 +#: synse_server/tasks.py:20 synse_server/tasks.py:22 msgid "adding task" msgstr "" -#: synse_server/tasks.py:28 +#: synse_server/tasks.py:32 msgid "task: rebuilding device cache" msgstr "" -#: synse_server/tasks.py:36 +#: synse_server/tasks.py:40 msgid "task: failed to rebuild device cache" msgstr "" -#: synse_server/api/http.py:26 +#: synse_server/tasks.py:53 +msgid "task: refreshing plugins" +msgstr "" + +#: synse_server/tasks.py:61 +msgid "task: failed to refresh plugins" +msgstr "" + +#: synse_server/api/http.py:27 msgid "processing request" msgstr "" -#: synse_server/api/websocket.py:23 +#: synse_server/api/websocket.py:25 msgid "new websocket connection" msgstr "" -#: synse_server/api/websocket.py:40 +#: synse_server/api/websocket.py:42 msgid "failed to load payload" msgstr "" -#: synse_server/api/websocket.py:121 +#: synse_server/api/websocket.py:122 msgid "running message handler for websocket" msgstr "" -#: synse_server/api/websocket.py:127 +#: synse_server/api/websocket.py:128 msgid "error loading websocket message" msgstr "" -#: synse_server/api/websocket.py:131 +#: synse_server/api/websocket.py:132 msgid "websocket handler: got message" msgstr "" -#: synse_server/api/websocket.py:135 +#: synse_server/api/websocket.py:136 msgid "error generating websocket response" msgstr "" -#: synse_server/api/websocket.py:199 +#: synse_server/api/websocket.py:200 msgid "processing websocket request" msgstr "" -#: synse_server/api/websocket.py:204 +#: synse_server/api/websocket.py:205 msgid "websocket request cancelled" msgstr "" -#: synse_server/api/websocket.py:207 +#: synse_server/api/websocket.py:208 msgid "error processing websocket request" msgstr "" -#: synse_server/api/websocket.py:422 +#: synse_server/api/websocket.py:425 msgid "read stream stop request received - terminating stream tasks" msgstr "" -#: synse_server/api/websocket.py:436 +#: synse_server/api/websocket.py:439 msgid "websocket raised ConnectionClosed - terminating read stream" msgstr "" -#: synse_server/cmd/config.py:13 synse_server/cmd/info.py:18 -#: synse_server/cmd/plugin.py:20 synse_server/cmd/plugin.py:60 -#: synse_server/cmd/plugin.py:84 synse_server/cmd/read.py:62 -#: synse_server/cmd/read.py:119 synse_server/cmd/read.py:157 -#: synse_server/cmd/read.py:247 synse_server/cmd/scan.py:28 -#: synse_server/cmd/tags.py:18 synse_server/cmd/test.py:13 -#: synse_server/cmd/transaction.py:18 synse_server/cmd/transaction.py:65 -#: synse_server/cmd/version.py:13 synse_server/cmd/write.py:21 +#: synse_server/cmd/config.py:15 synse_server/cmd/info.py:20 +#: synse_server/cmd/plugin.py:22 synse_server/cmd/plugin.py:64 +#: synse_server/cmd/plugin.py:88 synse_server/cmd/read.py:63 +#: synse_server/cmd/read.py:138 synse_server/cmd/read.py:176 +#: synse_server/cmd/read.py:281 synse_server/cmd/scan.py:32 +#: synse_server/cmd/tags.py:20 synse_server/cmd/test.py:15 +#: synse_server/cmd/transaction.py:20 synse_server/cmd/transaction.py:67 +#: synse_server/cmd/version.py:15 synse_server/cmd/write.py:22 #: synse_server/cmd/write.py:61 msgid "issuing command" msgstr "" -#: synse_server/cmd/plugin.py:101 +#: synse_server/cmd/plugin.py:105 msgid "failed to get plugin health" msgstr "" -#: synse_server/cmd/read.py:66 +#: synse_server/cmd/read.py:67 msgid "no tags specified, reading with no tag filter" msgstr "" -#: synse_server/cmd/read.py:74 synse_server/cmd/read.py:98 +#: synse_server/cmd/read.py:72 synse_server/cmd/read.py:108 +#: synse_server/cmd/read.py:182 synse_server/cmd/read.py:289 +msgid "plugin not active, will not read its devices" +msgstr "" + +#: synse_server/cmd/read.py:82 synse_server/cmd/read.py:118 msgid "error while issuing gRPC request: read" msgstr "" -#: synse_server/cmd/read.py:78 synse_server/cmd/read.py:105 +#: synse_server/cmd/read.py:86 synse_server/cmd/read.py:125 msgid "got readings" msgstr "" -#: synse_server/cmd/read.py:85 synse_server/cmd/scan.py:55 +#: synse_server/cmd/read.py:98 synse_server/cmd/scan.py:59 msgid "parsing tag groups" msgstr "" -#: synse_server/cmd/read.py:124 synse_server/cmd/write.py:28 +#: synse_server/cmd/read.py:143 synse_server/cmd/write.py:29 #: synse_server/cmd/write.py:68 msgid "plugin not found for device {}" msgstr "" -#: synse_server/cmd/read.py:133 +#: synse_server/cmd/read.py:152 msgid "error while issuing gRPC request: read device" msgstr "" -#: synse_server/cmd/read.py:161 +#: synse_server/cmd/read.py:187 msgid "getting cached readings for plugin" msgstr "" -#: synse_server/cmd/read.py:168 +#: synse_server/cmd/read.py:194 msgid "error while issuing gRPC request: read cache" msgstr "" -#: synse_server/cmd/read.py:201 +#: synse_server/cmd/read.py:232 msgid "running Stream thread" msgstr "" -#: synse_server/cmd/read.py:210 +#: synse_server/cmd/read.py:241 msgid "stream thread cancelled" msgstr "" -#: synse_server/cmd/read.py:215 +#: synse_server/cmd/read.py:246 msgid "error while issuing gRPC request: read stream" msgstr "" -#: synse_server/cmd/read.py:220 +#: synse_server/cmd/read.py:251 msgid "cancelling reading stream" msgstr "" -#: synse_server/cmd/read.py:258 +#: synse_server/cmd/read.py:299 msgid "executing callback to cancel read stream threads" msgstr "" -#: synse_server/cmd/read.py:270 +#: synse_server/cmd/read.py:311 msgid "collecting streamed readings..." msgstr "" -#: synse_server/cmd/scan.py:36 +#: synse_server/cmd/scan.py:40 msgid "forced scan: rebuilding device cache" msgstr "" -#: synse_server/cmd/scan.py:40 +#: synse_server/cmd/scan.py:44 msgid "failed to rebuild device cache" msgstr "" -#: synse_server/cmd/scan.py:44 +#: synse_server/cmd/scan.py:48 msgid "getting devices with no tag filter" msgstr "" -#: synse_server/cmd/scan.py:49 +#: synse_server/cmd/scan.py:53 msgid "failed to get all devices from cache" msgstr "" -#: synse_server/cmd/scan.py:67 +#: synse_server/cmd/scan.py:71 msgid "failed to get devices from cache" msgstr "" -#: synse_server/cmd/scan.py:80 +#: synse_server/cmd/scan.py:84 msgid "sorting devices" msgstr "" -#: synse_server/cmd/scan.py:99 +#: synse_server/cmd/scan.py:103 msgid "got devices" msgstr "" -#: synse_server/cmd/tags.py:32 +#: synse_server/cmd/tags.py:34 msgid "filtering ID tags" msgstr "" -#: synse_server/cmd/tags.py:36 +#: synse_server/cmd/tags.py:38 msgid "filtering tags by namespace" msgstr "" -#: synse_server/cmd/tags.py:39 +#: synse_server/cmd/tags.py:41 msgid "got tags" msgstr "" -#: synse_server/cmd/write.py:43 +#: synse_server/cmd/write.py:44 msgid "error while issuing gRPC request: async write" msgstr "" @@ -344,95 +361,95 @@ msgstr "" msgid "error while issuing gRPC request: sync write" msgstr "" -#: synse_server/discovery/kubernetes.py:23 +#: synse_server/discovery/kubernetes.py:25 msgid "plugin discovery via Kubernetes is disabled" msgstr "" -#: synse_server/discovery/kubernetes.py:35 +#: synse_server/discovery/kubernetes.py:37 msgid "plugin discovery via Kubernetes is enabled" msgstr "" -#: synse_server/discovery/kubernetes.py:49 +#: synse_server/discovery/kubernetes.py:51 msgid "found no configured endpoints for plugin discovery via Kubernetes" msgstr "" -#: synse_server/discovery/kubernetes.py:83 +#: synse_server/discovery/kubernetes.py:85 msgid "found no configured labels for plugin discovery via Kubernetes Endpoints" msgstr "" -#: synse_server/discovery/kubernetes.py:100 +#: synse_server/discovery/kubernetes.py:102 msgid "listing Kubernetes endpoint" msgstr "" -#: synse_server/discovery/kubernetes.py:108 +#: synse_server/discovery/kubernetes.py:110 msgid "discovered matching Endpoint" msgstr "" -#: synse_server/discovery/kubernetes.py:111 +#: synse_server/discovery/kubernetes.py:113 msgid "parsing EndpointSubset" msgstr "" -#: synse_server/discovery/kubernetes.py:117 +#: synse_server/discovery/kubernetes.py:119 msgid "no addresses for EndpointSubset - skipping" msgstr "" -#: synse_server/discovery/kubernetes.py:123 +#: synse_server/discovery/kubernetes.py:125 msgid "collecting available addresses for EndpointSubset" msgstr "" -#: synse_server/discovery/kubernetes.py:127 +#: synse_server/discovery/kubernetes.py:129 msgid "parsing EndpointAddress" msgstr "" -#: synse_server/discovery/kubernetes.py:133 +#: synse_server/discovery/kubernetes.py:135 msgid "address has no target_ref - skipping" msgstr "" -#: synse_server/discovery/kubernetes.py:138 +#: synse_server/discovery/kubernetes.py:140 msgid "address is not a Pod address - skipping" msgstr "" -#: synse_server/discovery/kubernetes.py:147 +#: synse_server/discovery/kubernetes.py:149 msgid "no IPs found for EndpointSubset - skipping" msgstr "" -#: synse_server/discovery/kubernetes.py:150 +#: synse_server/discovery/kubernetes.py:152 msgid "found IPs for EndpointSubset" msgstr "" -#: synse_server/discovery/kubernetes.py:156 +#: synse_server/discovery/kubernetes.py:158 msgid "no ports for EndpointSubset - skipping" msgstr "" -#: synse_server/discovery/kubernetes.py:162 +#: synse_server/discovery/kubernetes.py:164 msgid "found single port for EndpointSubset" msgstr "" -#: synse_server/discovery/kubernetes.py:167 +#: synse_server/discovery/kubernetes.py:169 msgid "found multiple ports - search for port named \"http\"" msgstr "" -#: synse_server/discovery/kubernetes.py:169 +#: synse_server/discovery/kubernetes.py:171 msgid "found port" msgstr "" -#: synse_server/discovery/kubernetes.py:171 +#: synse_server/discovery/kubernetes.py:173 msgid "skipping port - does not match" msgstr "" -#: synse_server/discovery/kubernetes.py:175 +#: synse_server/discovery/kubernetes.py:177 msgid "found port name \"http\"" msgstr "" -#: synse_server/discovery/kubernetes.py:185 +#: synse_server/discovery/kubernetes.py:187 msgid "discovered plugin via Endpoint" msgstr "" -#: synse_server/discovery/kubernetes.py:189 +#: synse_server/discovery/kubernetes.py:191 msgid "no plugins found via Kubernetes Endpoints" msgstr "" -#: synse_server/discovery/kubernetes.py:191 +#: synse_server/discovery/kubernetes.py:193 msgid "found plugins via Kubernetes Endpoints" msgstr "" diff --git a/synse_server/tasks.py b/synse_server/tasks.py index 0f262d8e..00131d3f 100644 --- a/synse_server/tasks.py +++ b/synse_server/tasks.py @@ -4,7 +4,7 @@ import sanic -from synse_server import config +from synse_server import config, plugin from synse_server.cache import update_device_cache from synse_server.i18n import _ from synse_server.log import logger @@ -19,6 +19,8 @@ def register_with_app(app: sanic.Sanic) -> None: # Periodically invalidate caches logger.info(_('adding task'), task='periodic device cache rebuild') app.add_task(_rebuild_device_cache) + logger.info(_('adding task'), task='periodic plugin refresh') + app.add_task(_refresh_plugins) async def _rebuild_device_cache() -> None: @@ -40,3 +42,24 @@ async def _rebuild_device_cache() -> None: ) await asyncio.sleep(interval) + + +async def _refresh_plugins() -> None: + """Periodically refresh the plugin manager.""" + interval = 2 * 60 # 2 minutes + + while True: + logger.info( + _('task: refreshing plugins'), + task='periodic plugin refresh', interval=interval, + ) + + try: + plugin.manager.refresh() + except Exception as e: + logger.error( + _('task: failed to refresh plugins'), + task='periodic plugin refresh', interval=interval, error=e, + ) + + await asyncio.sleep(interval) diff --git a/tests/unit/api/test_websocket.py b/tests/unit/api/test_websocket.py index 4041894d..3d3568ae 100644 --- a/tests/unit/api/test_websocket.py +++ b/tests/unit/api/test_websocket.py @@ -251,7 +251,9 @@ async def test_request_plugins(self): await m.handle_request_plugins(p) mock_cmd.assert_called_once() - mock_cmd.assert_called_with() + mock_cmd.assert_called_with( + refresh=False, + ) mock_send.assert_called_once() mock_send.assert_called_with(json.dumps({ 'id': 'testing', diff --git a/tests/unit/cmd/test_plugin.py b/tests/unit/cmd/test_plugin.py index aecf308a..5685632d 100644 --- a/tests/unit/cmd/test_plugin.py +++ b/tests/unit/cmd/test_plugin.py @@ -146,7 +146,7 @@ async def test_plugins_no_plugin(mocker): ) # --- Test case ----------------------------- - resp = await cmd.plugins() + resp = await cmd.plugins(refresh=False) assert resp == [] mock_refresh.assert_called_once() @@ -163,18 +163,41 @@ async def test_plugins_ok(mocker, simple_plugin): ) # --- Test case ----------------------------- - resp = await cmd.plugins() + resp = await cmd.plugins(refresh=False) assert resp == [ { # from simple_plugin fixture 'id': '123', 'tag': 'test/foo', - 'active': False, + 'active': True, }, ] mock_refresh.assert_not_called() +@pytest.mark.asyncio +async def test_plugins_ok_with_refresh(mocker, simple_plugin): + # Mock test data + mocker.patch.dict('synse_server.plugin.manager.plugins', { + '123': simple_plugin, + }) + mock_refresh = mocker.patch( + 'synse_server.plugin.manager.refresh', + ) + + # --- Test case ----------------------------- + resp = await cmd.plugins(refresh=True) + assert resp == [ + { # from simple_plugin fixture + 'id': '123', + 'tag': 'test/foo', + 'active': True, + }, + ] + + mock_refresh.assert_called() + + @pytest.mark.asyncio @pytest.mark.usefixtures('patch_utils_rfc3339now') async def test_plugin_health_no_plugins(mocker): diff --git a/tests/unit/cmd/test_read.py b/tests/unit/cmd/test_read.py index 6ee5fd81..9dfe53b8 100644 --- a/tests/unit/cmd/test_read.py +++ b/tests/unit/cmd/test_read.py @@ -95,6 +95,32 @@ async def test_read_fails_read_multiple_one_fail(mocker, simple_plugin, temperat # ensure that it was actually called prior to the error read +@pytest.mark.asyncio +async def test_read_ok_inactive_plugin(mocker, simple_plugin, state_reading): + # Mock test data + mocker.patch.dict('synse_server.plugin.PluginManager.plugins', { + '123': simple_plugin, + }) + + mock_read = mocker.patch( + 'synse_grpc.client.PluginClientV3.read', + return_value=[ + state_reading, + ], + ) + + # --- Test case ----------------------------- + # Set the simple_plugin to inactive to start. + simple_plugin.active = False + + resp = await cmd.read('default', [['foo/bar', 'vapor/ware']]) + # No readings - the one plugin set is inactive so should not be read from. + assert resp == [] + assert simple_plugin.active is False + + mock_read.assert_not_called() + + @pytest.mark.asyncio async def test_read_ok_no_tags(mocker, simple_plugin, temperature_reading, humidity_reading): # Mock test data @@ -459,6 +485,30 @@ def patchreadcache(*args, **kwargs): mock_read.assert_called_with(start='2019-04-22T13:30:00Z', end='2019-04-22T13:35:00Z') +@pytest.mark.asyncio +async def test_read_cache_inactive_plugin(mocker, simple_plugin, humidity_reading): + # Mock test data + simple_plugin.active = False + mocker.patch.dict('synse_server.plugin.PluginManager.plugins', { + '123': simple_plugin, + }) + + def patchreadcache(*args, **kwargs): + yield humidity_reading + + mock_read = mocker.patch( + 'synse_grpc.client.PluginClientV3.read_cache', + side_effect=patchreadcache, + ) + + # --- Test case ----------------------------- + resp = [r async for r in cmd.read_cache('2019-04-22T13:30:00Z', '2019-04-22T13:35:00Z')] + assert len(resp) == 0 # no readings since plugin not active + assert simple_plugin.active is False + + mock_read.assert_not_called() + + def test_reading_to_dict_1(temperature_reading): actual = reading_to_dict(temperature_reading) assert actual == { diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index dbd85cfc..cac09360 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -104,7 +104,7 @@ def simple_plugin(): configured minimally. """ - return plugin.Plugin( + p = plugin.Plugin( client=client.PluginClientV3('localhost:5432', 'tcp'), info={ 'tag': 'test/foo', @@ -113,6 +113,8 @@ def simple_plugin(): }, version={}, ) + p.active = True + return p @pytest.fixture() diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index be6cfe17..4328f5fa 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -2,9 +2,9 @@ import asynctest import grpc import pytest -from synse_grpc import api +from synse_grpc import api, client -from synse_server import cache +from synse_server import cache, plugin @pytest.mark.usefixtures('clear_txn_cache') @@ -189,10 +189,22 @@ async def test_update_device_cache_no_devices(self, mocker, simple_plugin): @pytest.mark.asyncio async def test_update_device_cache_devices_rpc_error(self, mocker, simple_plugin): + # Need to define a plugin different than simple_plugin so we have different instances. + p = plugin.Plugin( + client=client.PluginClientV3('localhost:5432', 'tcp'), + info={ + 'tag': 'test/bar', + 'id': '456', + 'vcs': 'https://github.com/vapor-ware/synse-server', + }, + version={}, + ) + p.active = True + # Mock test data mocker.patch.dict('synse_server.plugin.PluginManager.plugins', { '123': simple_plugin, - '456': simple_plugin, + '456': p, }) mock_devices = mocker.patch( @@ -213,6 +225,38 @@ async def test_update_device_cache_devices_rpc_error(self, mocker, simple_plugin mocker.call(), ]) + @pytest.mark.asyncio + async def test_update_device_cache_devices_rpc_error_2(self, mocker, simple_plugin): + """This test is similar to the one above, but it tests that upon failing via RPCError, + the plugin is marked inactive and will not have its devices collected the next time + around. This is done in a bit of a tricky manner by using the same object (simple_plugin) + for both patched plugins. The update on failure on the first will be reflected in the + state of the second iteration (since they are the same object. + """ + + # Mock test data + mocker.patch.dict('synse_server.plugin.PluginManager.plugins', { + '123': simple_plugin, + '456': simple_plugin, + }) + + mock_devices = mocker.patch( + 'synse_grpc.client.PluginClientV3.devices', + side_effect=grpc.RpcError(), + ) + + # --- Test case ----------------------------- + assert len(cache.device_cache._cache) == 0 + + await cache.update_device_cache() + + assert len(cache.device_cache._cache) == 0 + + mock_devices.assert_called() + mock_devices.assert_has_calls([ + mocker.call(), # should only be called once, second is skipped because inactive + ]) + @pytest.mark.asyncio async def test_update_device_cache_devices_error(self, mocker, simple_plugin): # Mock test data @@ -427,8 +471,8 @@ async def test_get_plugin_no_device(self): with asynctest.patch('synse_server.cache.get_device') as mock_get: mock_get.return_value = None - plugin = await cache.get_plugin('device-1') - assert plugin is None + p = await cache.get_plugin('device-1') + assert p is None mock_get.assert_called_once() mock_get.assert_called_with('device-1') @@ -438,8 +482,8 @@ async def test_get_plugin_no_plugin(self, simple_device): with asynctest.patch('synse_server.cache.get_device') as mock_get: mock_get.return_value = simple_device - plugin = await cache.get_plugin('device-1') - assert plugin is None + p = await cache.get_plugin('device-1') + assert p is None mock_get.assert_called_once() mock_get.assert_called_with('device-1') @@ -455,8 +499,8 @@ async def test_get_plugin_ok(self, mocker, simple_plugin, simple_device): with asynctest.patch('synse_server.cache.get_device') as mock_get: mock_get.return_value = simple_device - plugin = await cache.get_plugin('device-1') - assert plugin == simple_plugin + p = await cache.get_plugin('device-1') + assert p == simple_plugin mock_get.assert_called_once() mock_get.assert_called_with('device-1') diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py index b8678c31..d1df1324 100644 --- a/tests/unit/test_plugin.py +++ b/tests/unit/test_plugin.py @@ -476,6 +476,7 @@ def test_str(self, simple_plugin): assert str(simple_plugin) == '' def test_context_no_error(self, simple_plugin): + simple_plugin.active = False assert simple_plugin.active is False with simple_plugin as cli: @@ -485,7 +486,7 @@ def test_context_no_error(self, simple_plugin): assert simple_plugin.active is True def test_context_unexpected_error(self, simple_plugin): - assert simple_plugin.active is False + assert simple_plugin.active is True with pytest.raises(ValueError): with simple_plugin: @@ -494,6 +495,7 @@ def test_context_unexpected_error(self, simple_plugin): assert simple_plugin.active is False def test_context_plugin_error(self, simple_plugin): + simple_plugin.active = False assert simple_plugin.active is False with pytest.raises(errors.PluginError): diff --git a/tests/unit/test_tasks.py b/tests/unit/test_tasks.py index 76e1aba7..92906e96 100644 --- a/tests/unit/test_tasks.py +++ b/tests/unit/test_tasks.py @@ -7,8 +7,11 @@ def test_register_with_app(): - app = Sanic() + app = Sanic('test-app') app.add_task = mock.MagicMock() tasks.register_with_app(app) - app.add_task.assert_called_once() + app.add_task.assert_has_calls([ + mock.call(tasks._rebuild_device_cache), + mock.call(tasks._refresh_plugins), + ])