remove the additional web server and fix remaining problems

Backend/bungio/client.py

@@ -26,8 +26,10 @@ async def on_token_update(self, before: AuthData | None, after: AuthData) -> Non
                 token_expiry=after.token_expiry,
                 refresh_token_expiry=after.refresh_token_expiry,
             )
-
-        self.logger.debug(f"Updated token for {before.membership_id=}: {before.token=} -> {after.token=}")
+        if before:
+            self.logger.debug(f"Updated token for {before.membership_id=}: {before.token=} -> {after.token=}")
+        else:
+            self.logger.debug(f"Inserted token for {after.membership_id=} -> {after.token=}")
 
     async def on_manifest_update(self) -> None:
         from Backend.bungio.manifest import destiny_manifest

Backend/core/errors.py

@@ -4,6 +4,7 @@
     BungieDead,
     BungieException,
     BungIOException,
+    HttpException,
     InvalidAuthentication,
     NotFound,
     TimeoutException,
@@ -34,7 +35,7 @@ async def handle_custom_exception(request: Request, exception: CustomException):
     )
 
 
-async def handle_bungio_exception(request: Request, exception: BungieException):
+async def handle_bungio_exception(request: Request, exception: HttpException):
     content = {}
     if isinstance(exception, InvalidAuthentication):
         error = "NoToken"

Backend/database/base.py

@@ -2,7 +2,7 @@
 
 import logging
 import os
-from contextlib import asynccontextmanager
+from contextlib import aclosing, asynccontextmanager
 from typing import AsyncContextManager, Optional
 
 import orjson
@@ -86,7 +86,6 @@ def is_test_mode(set_test_mode: Optional[bool] = None) -> bool:
 @asynccontextmanager
 async def acquire_db_session() -> AsyncContextManager[AsyncSession]:
     """Get a database session"""
-
     db = get_async_sessionmaker()()
     logger = logging.getLogger("db")
 

Backend/endpoints/auth.py

@@ -1,6 +1,7 @@
 import logging
 from datetime import timedelta
 
+from bungio.error import BungieException, HttpException, NotFound
 from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status
 from fastapi.security import OAuth2PasswordRequestForm
 
@@ -23,14 +24,15 @@
 async def save_bungie_token(bungie_input: BungieRegistrationInput, background_tasks: BackgroundTasks):
     """Saves a bungie token"""
 
-    async with acquire_db_session() as db:
-        user = None
+    try:
+        async with acquire_db_session() as db:
+            user = None
+
+            # since this is a prerequisite for everything really, the test for this is called in insert_dummy_data()
+            # it is not tested directly, since we don't want to update the activities in the background
 
-        # since this is a prerequisite for everything really, the test for this is called in insert_dummy_data()
-        # it is not tested directly, since we don't want to update the activities in the background
+            # get the initial token from the authentication
 
-        # get the initial token from the authentication
-        try:
             auth = await get_bungio_client().generate_auth(code=bungie_input.code)
 
             # save in db
@@ -42,39 +44,41 @@ async def save_bungie_token(bungie_input: BungieRegistrationInput, background_ta
             logger.info(
                 f"User with discord ID `{user.discord_id}` has registered successfully with destiny ID `{user.destiny_id}`, system `{user.system}`, and bungie name `{user.bungie_name}`"
             )
-        except Exception as error:
-            # catch bungie errors, no need to log them
-            if isinstance(error, CustomException):
-                raise error
-
-            logger = logging.getLogger("registration")
-            logger.exception(
-                f"Registration for ID `{user.destiny_id if user else bungie_input.state}` failed", exc_info=error
-            )
-
-        # get users activities in background
-        background_tasks.add_task(update_activities_in_background, user)
-
-        # send a msg to Elevator and get the mutual guild ids
-        elevator_api = ElevatorApi()
-
-        try:
-            response = await elevator_api.post(
-                route="/registration",
-                json={
-                    "discord_id": discord_id,
-                },
-            )
-        except Exception as error:
-            response = None
-
-            # it is bad if this fails, since it disrupts the user flow
-            logger = logging.getLogger("elevatorApiExceptions")
-            logger.exception("Registration Error", exc_info=error)
-
-        # see if we could connect
-        if response is not None:
-            # assign the role in all guilds
+    except Exception as error:
+        # catch bungie errors, no need to log them
+        if isinstance(error, CustomException) or isinstance(error, HttpException):
+            raise error
+
+        logger = logging.getLogger("registration")
+        logger.exception(
+            f"Registration for ID `{user.destiny_id if user else bungie_input.state}` failed", exc_info=error
+        )
+        raise error
+
+    # get users activities in background
+    background_tasks.add_task(update_activities_in_background, user)
+
+    # send a msg to Elevator and get the mutual guild ids
+    elevator_api = ElevatorApi()
+
+    try:
+        response = await elevator_api.post(
+            route="/registration",
+            json={
+                "discord_id": discord_id,
+            },
+        )
+    except Exception as error:
+        response = None
+
+        # it is bad if this fails, since it disrupts the user flow
+        logger = logging.getLogger("elevatorApiExceptions")
+        logger.exception("Registration Error", exc_info=error)
+
+    # see if we could connect
+    if response is not None:
+        # assign the role in all guilds
+        async with acquire_db_session() as db:
             await discord_users.add_registration_roles(
                 db=db, discord_id=discord_id, guild_ids=response.content["guild_ids"]
             )

Backend/main.py

@@ -3,7 +3,7 @@
 import os
 import time
 
-from bungio.error import BungieException
+import bungio.error as bungio_errors
 from fastapi import Depends, FastAPI, Request
 from prometheus_client import CONTENT_TYPE_LATEST, REGISTRY, generate_latest
 from rich.console import Console
@@ -130,7 +130,7 @@ async def log_requests(request: Request, call_next):
 
     else:
         # do not log health check spam
-        if "health_check" not in request.url.path:
+        if not any(forbidden in request.url.path for forbidden in ["health_check", "metrics"]):
             process_time = round(time.time() - start_time, 2)
 
             # log that
@@ -157,7 +157,12 @@ async def log_requests(request: Request, call_next):
 # add exception handlers
 default_logger.debug("Adding Exception Handlers...")
 app.add_exception_handler(CustomException, handle_custom_exception)
-app.add_exception_handler(BungieException, handle_bungio_exception)
+for exc_class in bungio_errors.__dict__.values():
+    try:
+        if issubclass(exc_class, bungio_errors.HttpException):
+            app.add_exception_handler(exc_class, handle_bungio_exception)
+    except TypeError:
+        pass
 startup_progress.update(startup_task, advance=1)
 
 
@@ -175,7 +180,7 @@ async def startup():
 
     # Update the Destiny 2 manifest
     default_logger.debug("Updating and Caching Destiny 2 Manifest...")
-    await destiny_manifest.reset()
+    # await destiny_manifest.reset()
     startup_progress.update(startup_task, advance=1)
 
     # register background events
@@ -193,6 +198,3 @@ async def startup():
     prom_endpoints_registered.set(len(app.router.routes))
 
     startup_progress.stop()
-
-
-# todo bungio exceptions need to issue 429 responses

Backend/requirements.txt

@@ -6,7 +6,7 @@ alembic==1.8.1
 anyio==3.6.1
 apscheduler==3.9.1
 asyncpg==0.26.0
-bungio==0.6.0
+bungio==0.6.3
 fastapi==0.81.0
 feedparser==6.0.10
 ics==0.7.2

Backend/startup/initLogging.py

@@ -13,12 +13,12 @@ def init_logging() -> None:
     )
     ElevatorLogger.make_console_logger(
         logger=logging.getLogger("uvicorn"),
-        level=logging.WARNING if not get_setting("ENABLE_DEBUG_MODE") else logging.NOTSET,
+        level=logging.WARNING,
         highlighter=ColourHighlighter(name="uvicorn", colour="yellow"),
     )
     ElevatorLogger.make_console_logger(
         logger=logging.getLogger("uvicorn.access"),
-        level=logging.WARNING if not get_setting("ENABLE_DEBUG_MODE") else logging.NOTSET,
+        level=logging.WARNING,
         highlighter=ColourHighlighter(name="uvicorn", colour="yellow"),
     )
     ElevatorLogger.make_console_logger(

ElevatorBot/Dockerfile

@@ -26,4 +26,3 @@ COPY ./ElevatorBot /app/ElevatorBot
 WORKDIR /app
 CMD ["python3.10", "ElevatorBot/elevator.py"]
 EXPOSE $ELEVATOR_PORT
-EXPOSE 8877

ElevatorBot/prometheus/extension.py

@@ -35,58 +35,10 @@
 vc_lock = asyncio.Lock()
 
 
-class StatsServer(uvicorn.Server):
-    async def serve(self, sockets: Optional[List[socket.socket]] = None) -> None:
-        config = self.config
-        if not config.loaded:
-            config.load()
-
-        self.lifespan = config.lifespan_class(config)
-        self.force_exit = True  # to disable waiting for ctrl+c on exit in some cases
-
-        logger.info("Started metrics endpoint server")
-
-        await self.startup(sockets=sockets)
-        if self.should_exit:
-            return
-        try:
-            await self.main_loop()
-            # set self.should_exit = True to force main loop to exit. Maybe use in some events?
-        finally:
-            await self.shutdown(sockets=sockets)
-            logger.info("Finished metrics endpoint server")
-
-    def _log_started_message(self, listeners: Sequence[socket.SocketType]) -> None:
-        config = self.config
-
-        addr_format = "%s://%s:%d"
-        host = "0.0.0.0" if config.host is None else config.host
-        if ":" in host:
-            # It's an IPv6 address.
-            addr_format = "%s://[%s]:%d"
-
-        port = config.port
-        if port == 0:
-            port = listeners[0].getsockname()[1]
-
-        protocol_name = "https" if config.ssl else "http"
-        message = f"Uvicorn is running metrics endpoint on {addr_format}"
-        logger.info(
-            message,
-            protocol_name,
-            host,
-            port,
-        )
-
-
 class Stats(naff.Extension):
-    host = "0.0.0.0"
-    port = 8877
     interval = 15
 
     def __init__(self, bot):
-        self.server: Optional[StatsServer] = None
-
         self.bot_caches = {
             name.removesuffix("_cache"): cache
             for name, cache in inspect.getmembers(self.bot.cache, predicate=lambda x: isinstance(x, dict))
@@ -95,19 +47,9 @@ def __init__(self, bot):
 
     @naff.listen()
     async def on_startup(self) -> None:
-        app = prometheus_client.make_asgi_app()
-        cfg = uvicorn.Config(app=app, host=self.host, port=self.port, access_log=False)
-        self.server = StatsServer(cfg)
-
-        loop = asyncio.get_running_loop()
-        loop.create_task(self.server.serve())
-
         lib_info.info({"version": naff.const.__version__})
-
         elevator_version_info.info({"version": __version__})
-
         start_time_info.info({"datetime": f"{self.bot.start_time:%d.%m.%Y, %H:%M} UTC"})
-
         guilds_gauge.set(len(self.bot.user._guild_ids))
 
         stats_task = naff.Task(self.collect_stats, naff.triggers.IntervalTrigger(seconds=self.interval))

ElevatorBot/requirements.txt

@@ -4,7 +4,7 @@ aioredis==1.3.1     # aiohttp_client_cache currently requires this version
 # aiohttp_client_cache[redis]==0.7.0
 anyio==3.6.1
 apscheduler==3.9.1
-bungio==0.6.0
+bungio==0.6.3
 cachetools==5.2.0
 dateparser==1.1.1
 gtts==2.2.4

ElevatorBot/startup/initLogging.py

@@ -11,16 +11,6 @@ def init_logging() -> None:
         level=logging.NOTSET,
         highlighter=ColourHighlighter(name="nafftrack", colour="green"),
     )
-    ElevatorLogger.make_console_logger(
-        logger=logging.getLogger("uvicorn"),
-        level=logging.WARNING if not get_setting("ENABLE_DEBUG_MODE") else logging.NOTSET,
-        highlighter=ColourHighlighter(name="uvicorn", colour="yellow"),
-    )
-    ElevatorLogger.make_console_logger(
-        logger=logging.getLogger("uvicorn.access"),
-        level=logging.WARNING if not get_setting("ENABLE_DEBUG_MODE") else logging.NOTSET,
-        highlighter=ColourHighlighter(name="uvicorn", colour="yellow"),
-    )
 
     # Initialize formatter
     logger = ElevatorLogger("ElevatorBot")

ElevatorBot/webserver/routes/metrics.py

@@ -0,0 +1,13 @@
+from aiohttp import web
+from prometheus_client import CONTENT_TYPE_LATEST, REGISTRY, generate_latest
+
+
+async def metrics(request: web.Request):
+    """
+    Prometheus metric endpoint
+    """
+
+    resp = web.Response(body=generate_latest(REGISTRY))
+    resp.content_type = CONTENT_TYPE_LATEST
+
+    return resp

ElevatorBot/webserver/server.py

@@ -4,6 +4,7 @@
 
 from ElevatorBot.webserver.routes.manifestUpdate import manifest_update
 from ElevatorBot.webserver.routes.messages import messages
+from ElevatorBot.webserver.routes.metrics import metrics
 from ElevatorBot.webserver.routes.registration import registration
 from ElevatorBot.webserver.routes.roles import roles
 from ElevatorBot.webserver.routes.statusUpdate import status_update
@@ -15,8 +16,9 @@ async def log_requests(request: web.Request, handler):
         response = await handler(request)
 
         # log the successful request
-        logger = logging.getLogger("webServer")
-        logger.info(f"`{response.status}`: `{request.path_qs}`")
+        if "metrics" not in request.path:
+            logger = logging.getLogger("webServer")
+            logger.info(f"`{response.status}`: `{request.path_qs}`")
 
         return response
 
@@ -41,6 +43,7 @@ async def run_webserver(client):
             web.post("/messages", messages),
             web.post("/manifest_update", manifest_update),
             web.post("/status_update", status_update),
+            web.get("/metrics", metrics),
         ]
     )