From 2a39689d63065f452680f4b2a3146c5505f6e0ca Mon Sep 17 00:00:00 2001 From: Olivier Desenfans Date: Sat, 7 May 2022 00:48:57 +0200 Subject: [PATCH] [Tests] Make HTTP endpoints testable It is now possible to write API tests with the test client of aiohttp. Added an example test that checks that the /version endpoint returns the expected version. Refactored the declaration of endpoints in the initialization of the app. We now have a `register_routes` function centralized in a single module. All references to the global app object inside the implementation of endpoints were replaced by accesses to `request.app`. This allows to import the app object, configured properly, without importing other files. --- setup.cfg | 3 +- src/aleph/web/__init__.py | 3 ++ src/aleph/web/controllers/__init__.py | 63 ++++++++++++++++++++++--- src/aleph/web/controllers/aggregates.py | 4 -- src/aleph/web/controllers/channels.py | 4 -- src/aleph/web/controllers/info.py | 10 ++-- src/aleph/web/controllers/ipfs.py | 4 -- src/aleph/web/controllers/main.py | 32 ------------- src/aleph/web/controllers/messages.py | 8 ---- src/aleph/web/controllers/p2p.py | 7 +-- src/aleph/web/controllers/posts.py | 5 -- src/aleph/web/controllers/stats.py | 4 -- src/aleph/web/controllers/storage.py | 19 -------- src/aleph/web/controllers/version.py | 9 ++++ tests/api/test_version.py | 15 ++++++ 15 files changed, 89 insertions(+), 101 deletions(-) create mode 100644 src/aleph/web/controllers/version.py create mode 100644 tests/api/test_version.py diff --git a/setup.cfg b/setup.cfg index c9589befc..3d27240f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -91,8 +91,9 @@ testing = mypy==0.950 pytest pytest-cov - pytest-mock + pytest-aiohttp pytest-asyncio + pytest-mock types-requests types-setuptools nuls2 = diff --git a/src/aleph/web/__init__.py b/src/aleph/web/__init__.py index bd0afc83e..a985c40c9 100644 --- a/src/aleph/web/__init__.py +++ b/src/aleph/web/__init__.py @@ -8,6 +8,7 @@ import pkg_resources import socketio from aiohttp import web +from aleph.web.controllers import register_routes app = web.Application(client_max_size=1024 ** 2 * 64) sio = socketio.AsyncServer(async_mode="aiohttp", cors_allowed_origins="*") @@ -49,6 +50,8 @@ } ) +register_routes(app) + def init_cors(): # Configure CORS on all routes. diff --git a/src/aleph/web/controllers/__init__.py b/src/aleph/web/controllers/__init__.py index 152d37fcb..f8cce0079 100644 --- a/src/aleph/web/controllers/__init__.py +++ b/src/aleph/web/controllers/__init__.py @@ -1,13 +1,62 @@ +from aiohttp import web +import pkg_resources + from aleph.web.controllers import ( - main, + aggregates, + channels, + info, ipfs, - storage, - p2p, + main, messages, - aggregates, + p2p, posts, stats, - channels, - listener, - info, + storage, + version, ) + + +def register_routes(app: web.Application): + app.router.add_static( + "/static/", + path=pkg_resources.resource_filename("aleph.web", "static/"), + name="static", + ) + app.router.add_get("/", main.index) + app.router.add_get("/api/ws0/status", main.status_ws) + app.router.add_get("/metrics", main.metrics) + app.router.add_get("/metrics.json", main.metrics_json) + + app.router.add_get( + "/api/v0/aggregates/{address}.json", aggregates.address_aggregate + ) + + app.router.add_get("/api/v0/channels/list.json", channels.used_channels) + + app.router.add_get("/api/v0/info/public.json", info.public_multiaddress) + + app.router.add_post("/api/v0/ipfs/add_file", ipfs.ipfs_add_file) + + app.router.add_get("/api/v0/messages.json", messages.view_messages_list) + app.router.add_get("/api/v0/messages/page/{page}.json", messages.view_messages_list) + app.router.add_get("/api/ws0/messages", messages.messages_ws) + + app.router.add_post("/api/v0/ipfs/pubsub/pub", p2p.pub_json) + app.router.add_post("/api/v0/p2p/pubsub/pub", p2p.pub_json) + + app.router.add_get("/api/v0/posts.json", posts.view_posts_list) + app.router.add_get("/api/v0/posts/page/{page}.json", posts.view_posts_list) + + app.router.add_get("/api/v0/addresses/stats.json", stats.addresses_stats_view) + + app.router.add_post("/api/v0/ipfs/add_json", storage.add_ipfs_json_controller) + app.router.add_post("/api/v0/storage/add_json", storage.add_storage_json_controller) + app.router.add_post("/api/v0/storage/add_file", storage.storage_add_file) + app.router.add_get("/api/v0/storage/{hash}", storage.get_hash) + app.router.add_get("/api/v0/storage/raw/{hash}", storage.get_raw_hash) + app.router.add_get( + "/api/v0/storage/count/{hash}", storage.get_file_references_count + ) + + app.router.add_get("/version", version.version) + app.router.add_get("/api/v0/version", version.version) diff --git a/src/aleph/web/controllers/aggregates.py b/src/aleph/web/controllers/aggregates.py index 3ea785aca..6f83a59d1 100644 --- a/src/aleph/web/controllers/aggregates.py +++ b/src/aleph/web/controllers/aggregates.py @@ -1,7 +1,6 @@ from aiohttp import web from aleph.model.messages import get_computed_address_aggregates -from aleph.web import app async def address_aggregate(request): @@ -27,6 +26,3 @@ async def address_aggregate(request): output = {"address": address, "data": aggregates.get(address, {})} return web.json_response(output) - - -app.router.add_get("/api/v0/aggregates/{address}.json", address_aggregate) diff --git a/src/aleph/web/controllers/channels.py b/src/aleph/web/controllers/channels.py index b3fb2f6a4..1ae61f0f3 100644 --- a/src/aleph/web/controllers/channels.py +++ b/src/aleph/web/controllers/channels.py @@ -2,7 +2,6 @@ from aiohttp import web from aleph.model.messages import Message -from aleph.web import app @cached(ttl=60 * 120, cache=SimpleMemoryCache, timeout=120) @@ -20,6 +19,3 @@ async def used_channels(request): response = web.json_response({"channels": await get_channels()}) response.enable_compression() return response - - -app.router.add_get("/api/v0/channels/list.json", used_channels) diff --git a/src/aleph/web/controllers/info.py b/src/aleph/web/controllers/info.py index 566bb3e70..b4c2da28f 100644 --- a/src/aleph/web/controllers/info.py +++ b/src/aleph/web/controllers/info.py @@ -1,17 +1,13 @@ from aiohttp import web -from aleph.web import app -async def public_multriaddres(request): - """Broadcast public node adresses +async def public_multiaddress(request): + """Broadcast public node addresses - According to multriaddr spec https://multiformats.io/multiaddr/ + According to multiaddr spec https://multiformats.io/multiaddr/ """ output = { "node_multi_addresses": request.config_dict["extra_config"]["public_adresses"], } return web.json_response(output) - - -app.router.add_get("/api/v0/info/public.json", public_multriaddres) diff --git a/src/aleph/web/controllers/ipfs.py b/src/aleph/web/controllers/ipfs.py index 4bff20455..697e6f71b 100644 --- a/src/aleph/web/controllers/ipfs.py +++ b/src/aleph/web/controllers/ipfs.py @@ -1,7 +1,6 @@ from aiohttp import web from aleph.services.ipfs.storage import add_file -from aleph.web import app async def ipfs_add_file(request): @@ -17,6 +16,3 @@ async def ipfs_add_file(request): "size": output["Size"], } return web.json_response(output) - - -app.router.add_post("/api/v0/ipfs/add_file", ipfs_add_file) diff --git a/src/aleph/web/controllers/main.py b/src/aleph/web/controllers/main.py index fa37ad99e..eeb48438a 100644 --- a/src/aleph/web/controllers/main.py +++ b/src/aleph/web/controllers/main.py @@ -7,20 +7,11 @@ import pkg_resources from aiohttp import web -from aleph import __version__ -from aleph.web import app from aleph.web.controllers.metrics import format_dataclass_for_prometheus, get_metrics logger = logging.getLogger(__name__) -app.router.add_static( - "/static/", - path=pkg_resources.resource_filename("aleph.web", "static/"), - name="static", -) - - @aiohttp_jinja2.template("index.html") async def index(request) -> Dict: """Index of aleph.""" @@ -28,20 +19,6 @@ async def index(request) -> Dict: return asdict(await get_metrics(shared_stats)) -app.router.add_get("/", index) - - -async def version(request): - """Version endpoint.""" - - response = web.json_response({"version": __version__}) - return response - - -app.router.add_get("/version", version) -app.router.add_get("/api/v0/version", version) - - async def status_ws(request): ws = web.WebSocketResponse() await ws.prepare(request) @@ -63,9 +40,6 @@ async def status_ws(request): await asyncio.sleep(2) -app.router.add_get("/api/ws0/status", status_ws) - - async def metrics(request): """Prometheus compatible metrics. @@ -78,9 +52,6 @@ async def metrics(request): ) -app.router.add_get("/metrics", metrics) - - async def metrics_json(request): """JSON version of the Prometheus metrics.""" shared_stats = request.config_dict["shared_stats"] @@ -88,6 +59,3 @@ async def metrics_json(request): text=(await get_metrics(shared_stats)).to_json(), content_type="application/json", ) - - -app.router.add_get("/metrics.json", metrics_json) diff --git a/src/aleph/web/controllers/messages.py b/src/aleph/web/controllers/messages.py index 7bf5e912e..06db4bba8 100644 --- a/src/aleph/web/controllers/messages.py +++ b/src/aleph/web/controllers/messages.py @@ -1,7 +1,6 @@ from typing import Any, Dict, List, Optional, Set from aleph.model.messages import CappedMessage, Message -from aleph.web import app from aiohttp import web from aiohttp.web_exceptions import HTTPBadRequest import asyncio @@ -156,10 +155,6 @@ async def view_messages_list(request): return cond_output(request, context, "TODO.html") -app.router.add_get("/api/v0/messages.json", view_messages_list) -app.router.add_get("/api/v0/messages/page/{page}.json", view_messages_list) - - async def messages_ws(request: web.Request): ws = web.WebSocketResponse() await ws.prepare(request) @@ -219,6 +214,3 @@ async def messages_ws(request: web.Request): LOGGER.exception("Error processing") await asyncio.sleep(1) - - -app.router.add_get("/api/ws0/messages", messages_ws) diff --git a/src/aleph/web/controllers/p2p.py b/src/aleph/web/controllers/p2p.py index d34a5c966..012c08004 100644 --- a/src/aleph/web/controllers/p2p.py +++ b/src/aleph/web/controllers/p2p.py @@ -7,7 +7,6 @@ from aleph.services.ipfs.pubsub import pub as pub_ipfs from aleph.services.p2p.pubsub import publish as pub_p2p from aleph.types import Protocol -from aleph.web import app LOGGER = logging.getLogger("web.controllers.p2p") @@ -18,7 +17,7 @@ async def pub_json(request): failed_publications = [] try: - if app["config"].ipfs.enabled.value: + if request.app["config"].ipfs.enabled.value: await asyncio.wait_for(pub_ipfs(data.get("topic"), data.get("data")), 1) except Exception: LOGGER.exception("Can't publish on ipfs") @@ -41,7 +40,3 @@ async def pub_json(request): {"status": status, "failed": failed_publications}, status=500 if status == "error" else 200, ) - - -app.router.add_post("/api/v0/ipfs/pubsub/pub", pub_json) -app.router.add_post("/api/v0/p2p/pubsub/pub", pub_json) diff --git a/src/aleph/web/controllers/posts.py b/src/aleph/web/controllers/posts.py index 955623dbb..7f4a806ac 100644 --- a/src/aleph/web/controllers/posts.py +++ b/src/aleph/web/controllers/posts.py @@ -1,5 +1,4 @@ from aleph.model.messages import Message, get_merged_posts -from aleph.web import app from aleph.web.controllers.utils import Pagination, cond_output, prepare_date_filters @@ -107,7 +106,3 @@ async def view_posts_list(request): ) return cond_output(request, context, "TODO.html") - - -app.router.add_get("/api/v0/posts.json", view_posts_list) -app.router.add_get("/api/v0/posts/page/{page}.json", view_posts_list) diff --git a/src/aleph/web/controllers/stats.py b/src/aleph/web/controllers/stats.py index ba2092896..6c09fcfa9 100644 --- a/src/aleph/web/controllers/stats.py +++ b/src/aleph/web/controllers/stats.py @@ -6,7 +6,6 @@ from bson import json_util from aleph.model.messages import Message -from aleph.web import app # WARNING: we are storing this in memory... memcached or similar would @@ -108,6 +107,3 @@ async def addresses_stats_view(request): return web.json_response( output, dumps=lambda v: json.dumps(v, default=json_util.default) ) - - -app.router.add_get("/api/v0/addresses/stats.json", addresses_stats_view) diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index becc98d45..f195a3f71 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -8,7 +8,6 @@ from aleph.storage import add_json, get_hash_content, add_file from aleph.types import ItemType from aleph.utils import run_in_executor -from aleph.web import app logger = logging.getLogger(__name__) @@ -21,9 +20,6 @@ async def add_ipfs_json_controller(request): return web.json_response(output) -app.router.add_post("/api/v0/ipfs/add_json", add_ipfs_json_controller) - - async def add_storage_json_controller(request): """Forward the json content to IPFS server and return an hash""" data = await request.json() @@ -32,9 +28,6 @@ async def add_storage_json_controller(request): return web.json_response(output) -app.router.add_post("/api/v0/storage/add_json", add_storage_json_controller) - - async def storage_add_file(request): # No need to pin it here anymore. # TODO: find a way to specify linked ipfs hashes in posts/aggr. @@ -47,9 +40,6 @@ async def storage_add_file(request): return web.json_response(output) -app.router.add_post("/api/v0/storage/add_file", storage_add_file) - - def prepare_content(content): return base64.encodebytes(content).decode("utf-8") @@ -88,9 +78,6 @@ async def get_hash(request): return response -app.router.add_get("/api/v0/storage/{hash}", get_hash) - - async def get_raw_hash(request): item_hash = request.match_info.get("hash", None) @@ -118,13 +105,7 @@ async def get_raw_hash(request): return response -app.router.add_get("/api/v0/storage/raw/{hash}", get_raw_hash) - - async def get_file_references_count(request): item_hash = request.match_info.get("hash", None) count = await count_file_references(storage_hash=item_hash) return web.json_response(data=count) - - -app.router.add_get("/api/v0/storage/count/{hash}", get_file_references_count) diff --git a/src/aleph/web/controllers/version.py b/src/aleph/web/controllers/version.py new file mode 100644 index 000000000..5b41c82c8 --- /dev/null +++ b/src/aleph/web/controllers/version.py @@ -0,0 +1,9 @@ +from aiohttp import web +from aleph import __version__ + + +async def version(request): + """Version endpoint.""" + + response = web.json_response({"version": __version__}) + return response diff --git a/tests/api/test_version.py b/tests/api/test_version.py new file mode 100644 index 000000000..4a1d91f73 --- /dev/null +++ b/tests/api/test_version.py @@ -0,0 +1,15 @@ +import pytest + +from aleph.web import app +from aleph import get_git_version + + +@pytest.mark.asyncio +async def test_get_version(aiohttp_client): + client = await aiohttp_client(app) + + response = await client.get("/api/v0/version") + assert response.status == 200, await response.text() + + data = await response.json() + assert data["version"] == get_git_version()