[Tests] Make API endpoints testable (#260)

It is now possible to write API tests with the test client of aiohttp.
Added an example test that checks that the /version endpoint returns
the expected version.

Refactored the declaration of endpoints in the initialization of
the app. We now have a `register_routes` function centralized in
a single module. All references to the global app object inside the
implementation of endpoints were replaced by accesses to `request.app`.
This allows to import the app object, configured properly, without
importing other files.

setup.cfg

@@ -91,8 +91,9 @@ testing =
     mypy==0.950
     pytest
     pytest-cov
-    pytest-mock
+    pytest-aiohttp
     pytest-asyncio
+    pytest-mock
     types-requests
     types-setuptools
 nuls2 =

src/aleph/__init__.py

@@ -4,7 +4,7 @@
 from pkg_resources import get_distribution, DistributionNotFound
 
 
-def get_git_version() -> str:
+def _get_git_version() -> str:
     output = subprocess.check_output(("git", "describe", "--tags"))
     return output.decode().strip()
 
@@ -14,6 +14,6 @@ def get_git_version() -> str:
     dist_name = __name__
     __version__ = get_distribution(dist_name).version
 except DistributionNotFound:
-    __version__ = get_git_version()
+    __version__ = _get_git_version()
 finally:
     del get_distribution, DistributionNotFound

src/aleph/web/__init__.py

@@ -8,6 +8,7 @@
 import pkg_resources
 import socketio
 from aiohttp import web
+from aleph.web.controllers.routes import register_routes
 
 app = web.Application(client_max_size=1024 ** 2 * 64)
 sio = socketio.AsyncServer(async_mode="aiohttp", cors_allowed_origins="*")
@@ -49,6 +50,8 @@
     }
 )
 
+register_routes(app)
+
 
 def init_cors():
     # Configure CORS on all routes.

src/aleph/web/controllers/__init__.py

@@ -1,13 +0,0 @@
-from aleph.web.controllers import (
-    main,
-    ipfs,
-    storage,
-    p2p,
-    messages,
-    aggregates,
-    posts,
-    stats,
-    channels,
-    listener,
-    info,
-)

src/aleph/web/controllers/aggregates.py

@@ -1,7 +1,6 @@
 from aiohttp import web
 
 from aleph.model.messages import get_computed_address_aggregates
-from aleph.web import app
 
 
 async def address_aggregate(request):
@@ -27,6 +26,3 @@ async def address_aggregate(request):
 
     output = {"address": address, "data": aggregates.get(address, {})}
     return web.json_response(output)
-
-
-app.router.add_get("/api/v0/aggregates/{address}.json", address_aggregate)

src/aleph/web/controllers/channels.py

@@ -2,7 +2,6 @@
 from aiohttp import web
 
 from aleph.model.messages import Message
-from aleph.web import app
 
 
 @cached(ttl=60 * 120, cache=SimpleMemoryCache, timeout=120)
@@ -20,6 +19,3 @@ async def used_channels(request):
     response = web.json_response({"channels": await get_channels()})
     response.enable_compression()
     return response
-
-
-app.router.add_get("/api/v0/channels/list.json", used_channels)

src/aleph/web/controllers/info.py

@@ -1,17 +1,13 @@
 from aiohttp import web
-from aleph.web import app
 
 
-async def public_multriaddres(request):
-    """Broadcast public node adresses
+async def public_multiaddress(request):
+    """Broadcast public node addresses
 
-    According to multriaddr spec https://multiformats.io/multiaddr/
+    According to multiaddr spec https://multiformats.io/multiaddr/
     """
 
     output = {
         "node_multi_addresses": request.config_dict["extra_config"]["public_adresses"],
     }
     return web.json_response(output)
-
-
-app.router.add_get("/api/v0/info/public.json", public_multriaddres)

src/aleph/web/controllers/ipfs.py

@@ -1,7 +1,6 @@
 from aiohttp import web
 
 from aleph.services.ipfs.storage import add_file
-from aleph.web import app
 
 
 async def ipfs_add_file(request):
@@ -17,6 +16,3 @@ async def ipfs_add_file(request):
         "size": output["Size"],
     }
     return web.json_response(output)
-
-
-app.router.add_post("/api/v0/ipfs/add_file", ipfs_add_file)

src/aleph/web/controllers/main.py

@@ -7,41 +7,18 @@
 import pkg_resources
 from aiohttp import web
 
-from aleph import __version__
-from aleph.web import app
 from aleph.web.controllers.metrics import format_dataclass_for_prometheus, get_metrics
 
 logger = logging.getLogger(__name__)
 
 
-app.router.add_static(
-    "/static/",
-    path=pkg_resources.resource_filename("aleph.web", "static/"),
-    name="static",
-)
-
-
 @aiohttp_jinja2.template("index.html")
 async def index(request) -> Dict:
     """Index of aleph."""
     shared_stats = request.config_dict["shared_stats"]
     return asdict(await get_metrics(shared_stats))
 
 
-app.router.add_get("/", index)
-
-
-async def version(request):
-    """Version endpoint."""
-
-    response = web.json_response({"version": __version__})
-    return response
-
-
-app.router.add_get("/version", version)
-app.router.add_get("/api/v0/version", version)
-
-
 async def status_ws(request):
     ws = web.WebSocketResponse()
     await ws.prepare(request)
@@ -63,9 +40,6 @@ async def status_ws(request):
         await asyncio.sleep(2)
 
 
-app.router.add_get("/api/ws0/status", status_ws)
-
-
 async def metrics(request):
     """Prometheus compatible metrics.
 
@@ -78,16 +52,10 @@ async def metrics(request):
     )
 
 
-app.router.add_get("/metrics", metrics)
-
-
 async def metrics_json(request):
     """JSON version of the Prometheus metrics."""
     shared_stats = request.config_dict["shared_stats"]
     return web.Response(
         text=(await get_metrics(shared_stats)).to_json(),
         content_type="application/json",
     )
-
-
-app.router.add_get("/metrics.json", metrics_json)

src/aleph/web/controllers/messages.py

@@ -1,7 +1,6 @@
 from typing import Any, Dict, List, Optional, Set
 
 from aleph.model.messages import CappedMessage, Message
-from aleph.web import app
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPBadRequest
 import asyncio
@@ -156,10 +155,6 @@ async def view_messages_list(request):
     return cond_output(request, context, "TODO.html")
 
 
-app.router.add_get("/api/v0/messages.json", view_messages_list)
-app.router.add_get("/api/v0/messages/page/{page}.json", view_messages_list)
-
-
 async def messages_ws(request: web.Request):
     ws = web.WebSocketResponse()
     await ws.prepare(request)
@@ -219,6 +214,3 @@ async def messages_ws(request: web.Request):
 
             LOGGER.exception("Error processing")
             await asyncio.sleep(1)
-
-
-app.router.add_get("/api/ws0/messages", messages_ws)

src/aleph/web/controllers/p2p.py

@@ -7,7 +7,6 @@
 from aleph.services.ipfs.pubsub import pub as pub_ipfs
 from aleph.services.p2p.pubsub import publish as pub_p2p
 from aleph.types import Protocol
-from aleph.web import app
 
 LOGGER = logging.getLogger("web.controllers.p2p")
 
@@ -18,7 +17,7 @@ async def pub_json(request):
     failed_publications = []
 
     try:
-        if app["config"].ipfs.enabled.value:
+        if request.app["config"].ipfs.enabled.value:
             await asyncio.wait_for(pub_ipfs(data.get("topic"), data.get("data")), 1)
     except Exception:
         LOGGER.exception("Can't publish on ipfs")
@@ -41,7 +40,3 @@ async def pub_json(request):
         {"status": status, "failed": failed_publications},
         status=500 if status == "error" else 200,
     )
-
-
-app.router.add_post("/api/v0/ipfs/pubsub/pub", pub_json)
-app.router.add_post("/api/v0/p2p/pubsub/pub", pub_json)

src/aleph/web/controllers/posts.py

@@ -1,5 +1,4 @@
 from aleph.model.messages import Message, get_merged_posts
-from aleph.web import app
 from aleph.web.controllers.utils import Pagination, cond_output, prepare_date_filters
 
 
@@ -107,7 +106,3 @@ async def view_posts_list(request):
         )
 
     return cond_output(request, context, "TODO.html")
-
-
-app.router.add_get("/api/v0/posts.json", view_posts_list)
-app.router.add_get("/api/v0/posts/page/{page}.json", view_posts_list)

src/aleph/web/controllers/routes.py

@@ -0,0 +1,62 @@
+from aiohttp import web
+import pkg_resources
+
+from aleph.web.controllers import (
+    aggregates,
+    channels,
+    info,
+    ipfs,
+    main,
+    messages,
+    p2p,
+    posts,
+    stats,
+    storage,
+    version,
+)
+
+
+def register_routes(app: web.Application):
+    app.router.add_static(
+        "/static/",
+        path=pkg_resources.resource_filename("aleph.web", "static/"),
+        name="static",
+    )
+    app.router.add_get("/", main.index)
+    app.router.add_get("/api/ws0/status", main.status_ws)
+    app.router.add_get("/metrics", main.metrics)
+    app.router.add_get("/metrics.json", main.metrics_json)
+
+    app.router.add_get(
+        "/api/v0/aggregates/{address}.json", aggregates.address_aggregate
+    )
+
+    app.router.add_get("/api/v0/channels/list.json", channels.used_channels)
+
+    app.router.add_get("/api/v0/info/public.json", info.public_multiaddress)
+
+    app.router.add_post("/api/v0/ipfs/add_file", ipfs.ipfs_add_file)
+
+    app.router.add_get("/api/v0/messages.json", messages.view_messages_list)
+    app.router.add_get("/api/v0/messages/page/{page}.json", messages.view_messages_list)
+    app.router.add_get("/api/ws0/messages", messages.messages_ws)
+
+    app.router.add_post("/api/v0/ipfs/pubsub/pub", p2p.pub_json)
+    app.router.add_post("/api/v0/p2p/pubsub/pub", p2p.pub_json)
+
+    app.router.add_get("/api/v0/posts.json", posts.view_posts_list)
+    app.router.add_get("/api/v0/posts/page/{page}.json", posts.view_posts_list)
+
+    app.router.add_get("/api/v0/addresses/stats.json", stats.addresses_stats_view)
+
+    app.router.add_post("/api/v0/ipfs/add_json", storage.add_ipfs_json_controller)
+    app.router.add_post("/api/v0/storage/add_json", storage.add_storage_json_controller)
+    app.router.add_post("/api/v0/storage/add_file", storage.storage_add_file)
+    app.router.add_get("/api/v0/storage/{hash}", storage.get_hash)
+    app.router.add_get("/api/v0/storage/raw/{hash}", storage.get_raw_hash)
+    app.router.add_get(
+        "/api/v0/storage/count/{hash}", storage.get_file_references_count
+    )
+
+    app.router.add_get("/version", version.version)
+    app.router.add_get("/api/v0/version", version.version)

src/aleph/web/controllers/stats.py

@@ -6,7 +6,6 @@
 from bson import json_util
 
 from aleph.model.messages import Message
-from aleph.web import app
 
 
 # WARNING: we are storing this in memory... memcached or similar would
@@ -108,6 +107,3 @@ async def addresses_stats_view(request):
     return web.json_response(
         output, dumps=lambda v: json.dumps(v, default=json_util.default)
     )
-
-
-app.router.add_get("/api/v0/addresses/stats.json", addresses_stats_view)

src/aleph/web/controllers/storage.py

@@ -8,7 +8,6 @@
 from aleph.storage import add_json, get_hash_content, add_file
 from aleph.types import ItemType
 from aleph.utils import run_in_executor
-from aleph.web import app
 
 logger = logging.getLogger(__name__)
 
@@ -21,9 +20,6 @@ async def add_ipfs_json_controller(request):
     return web.json_response(output)
 
 
-app.router.add_post("/api/v0/ipfs/add_json", add_ipfs_json_controller)
-
-
 async def add_storage_json_controller(request):
     """Forward the json content to IPFS server and return an hash"""
     data = await request.json()
@@ -32,9 +28,6 @@ async def add_storage_json_controller(request):
     return web.json_response(output)
 
 
-app.router.add_post("/api/v0/storage/add_json", add_storage_json_controller)
-
-
 async def storage_add_file(request):
     # No need to pin it here anymore.
     # TODO: find a way to specify linked ipfs hashes in posts/aggr.
@@ -47,9 +40,6 @@ async def storage_add_file(request):
     return web.json_response(output)
 
 
-app.router.add_post("/api/v0/storage/add_file", storage_add_file)
-
-
 def prepare_content(content):
     return base64.encodebytes(content).decode("utf-8")
 
@@ -88,9 +78,6 @@ async def get_hash(request):
     return response
 
 
-app.router.add_get("/api/v0/storage/{hash}", get_hash)
-
-
 async def get_raw_hash(request):
     item_hash = request.match_info.get("hash", None)
 
@@ -118,13 +105,7 @@ async def get_raw_hash(request):
     return response
 
 
-app.router.add_get("/api/v0/storage/raw/{hash}", get_raw_hash)
-
-
 async def get_file_references_count(request):
     item_hash = request.match_info.get("hash", None)
     count = await count_file_references(storage_hash=item_hash)
     return web.json_response(data=count)
-
-
-app.router.add_get("/api/v0/storage/count/{hash}", get_file_references_count)