finalise the grafana model

Backend/core/destiny/activities.py

@@ -11,6 +11,7 @@
 
 from Backend.bungio.client import get_bungio_client
 from Backend.bungio.manifest import destiny_manifest
+from Backend.core.destiny.clan import DestinyClan
 from Backend.core.errors import CustomException
 from Backend.crud import crud_activities, crud_activities_fail_to_get, discord_users
 from Backend.database.base import acquire_db_session
@@ -95,7 +96,15 @@ async def update_missing_pgcr(self):
         """Insert the missing pgcr"""
 
         async with update_missing_pgcr_lock:
-            for activity in await crud_activities_fail_to_get.get_all(db=self.db):
+            if not (missing := await crud_activities_fail_to_get.get_all(db=self.db)):
+                return
+
+            # get the destiny clan members for descend
+            async with acquire_db_session() as session:
+                clan = DestinyClan(db=session, guild_id=-1)
+                descend_clan_members = await clan.get_descend_clan_members()
+
+            for activity in missing:
                 # check if info is already in DB, delete and skip if so
                 result = await crud_activities.get(db=self.db, instance_id=activity.instance_id)
                 if result:
@@ -110,7 +119,9 @@ async def update_missing_pgcr(self):
                     continue
 
                 # add info to DB
-                await crud_activities.insert(data=[(activity.instance_id, activity.period, pgcr)])
+                await crud_activities.insert(
+                    data=[(activity.instance_id, activity.period, pgcr)], descend_clan_members=descend_clan_members
+                )
 
                 # delete from to-do DB
                 await crud_activities_fail_to_get.delete(db=self.db, obj=activity)
@@ -217,12 +228,17 @@ async def input_data(gather_instances: dict[int, datetime.datetime]):
                     tg.start_soon(lambda: handle(results=results, i=i, t=t))
 
             # insert information to DB
-            await crud_activities.insert(data=results)
+            await crud_activities.insert(data=results, descend_clan_members=descend_clan_members)
 
         # get the logger
         logger = logging.getLogger("updateActivityDb")
         logger_exceptions = logging.getLogger("updateActivityDbExceptions")
 
+        # get the destiny clan members for descend
+        async with acquire_db_session() as session:
+            clan = DestinyClan(db=session, guild_id=-1)
+            descend_clan_members = await clan.get_descend_clan_members()
+
         try:
             async with input_data_lock:
                 # ignore this if the same user is currently running

Backend/core/destiny/clan.py

@@ -7,8 +7,7 @@
 
 from Backend.bungio.client import get_bungio_client
 from Backend.core.errors import CustomException
-from Backend.crud import destiny_clan_links, discord_users, persistent_messages
-from Backend.database import acquire_db_session
+from Backend.crud import destiny_clan_links, discord_users
 from Backend.database.models import DiscordUsers
 from Shared.functions.helperFunctions import get_min_with_tz, get_now_with_tz, localize_datetime
 from Shared.functions.readSettingsFile import get_setting
@@ -90,8 +89,11 @@ async def get_descend_clan_members(self) -> dict[int, DestinyClanMemberModel]:
         """Get all descend clan members"""
 
         if self.descend_clan_id is None:
-            link = await destiny_clan_links.get_link(db=self.db, discord_guild_id=get_setting("DESCEND_GUILD_ID"))
-            self.descend_clan_id = link.destiny_clan_id
+            try:
+                link = await destiny_clan_links.get_link(db=self.db, discord_guild_id=get_setting("DESCEND_GUILD_ID"))
+                self.descend_clan_id = link.destiny_clan_id
+            except CustomException:
+                return {}
 
         if self.descend_clan_members_updated + datetime.timedelta(hours=1) < get_now_with_tz():
             members = await self.get_clan_members(clan_id=self.descend_clan_id)

Backend/crud/destiny/activities.py

@@ -7,12 +7,12 @@
 from sqlalchemy import distinct, func, not_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from Backend.core.destiny.clan import DestinyClan
 from Backend.core.errors import CustomException
 from Backend.crud.base import CRUDBase
 from Backend.database import acquire_db_session
 from Backend.database.models import Activities, ActivitiesFailToGet, ActivitiesUsers, ActivitiesUsersWeapons
 from Backend.prometheus.stats import prom_clan_activities
+from Shared.networkingSchemas import DestinyClanMemberModel
 from Shared.networkingSchemas.destiny.roles import TimePeriodModel
 
 fail_to_get_insert_lock = asyncio.Lock()
@@ -49,17 +49,18 @@ async def get(self, db: AsyncSession, instance_id: int) -> Optional[Activities]:
 
         return await self._get_with_key(db=db, primary_key=instance_id)
 
-    async def insert(self, data: list[tuple[int, datetime.datetime, DestinyPostGameCarnageReportData]]):
+    async def insert(
+        self,
+        data: list[tuple[int, datetime.datetime, DestinyPostGameCarnageReportData]],
+        descend_clan_members: dict[int, DestinyClanMemberModel],
+    ):
         """Get the activity with the instance_id"""
 
         to_create = []
         for instance_id, activity_time, pgcr in data:
             to_create.append(self._convert_to_model(instance_id=instance_id, activity_time=activity_time, pgcr=pgcr))
 
         async with acquire_db_session() as session:
-            clan = DestinyClan(db=session, guild_id=-1)
-            descend_clan_members = await clan.get_descend_clan_members()
-
             await self._insert_multi(db=session, to_create=to_create)
 
         # save the prometheus stats

Backend/prometheus/collecting.py

@@ -20,7 +20,7 @@ async def collect_prometheus_stats():
                 pass
 
     # registered users
-    query = select(func.count(DiscordUsers)).filter_by(token != None)
+    query = select(func.count()).select_from(DiscordUsers).filter(DiscordUsers.token.is_not(None))
     async with acquire_db_session() as db:
         result = await db.execute(query)
         result = result.scalar()

Backend/startup/initBackgroundEvents.py

@@ -37,7 +37,9 @@ def event_added(scheduler_event: JobEvent):
 
         # log the execution
         logger = logging.getLogger("backgroundEvents")
-        logger.info(f"Event `{job_name}` with ID `{scheduler_event.job_id}` has been added")
+        logger.info(
+            f"Event `{event_name_by_id[scheduler_event.job_id]}` with ID `{scheduler_event.job_id}` has been added"
+        )
 
     backgroundEvents.scheduler.add_listener(event_added, EVENT_JOB_ADDED)
 
@@ -51,6 +53,10 @@ def event_removed(scheduler_event: JobEvent):
     backgroundEvents.scheduler.add_listener(event_removed, EVENT_JOB_REMOVED)
 
     def event_submitted(scheduler_event: JobSubmissionEvent):
+        event_name = event_name_by_id[scheduler_event.job_id]
+        if event_name == "collect_prometheus_stats":
+            return
+
         logger = logging.getLogger("backgroundEvents")
         logger.debug(
             f"Running event `{event_name_by_id[scheduler_event.job_id]}` with ID `{scheduler_event.job_id}`..."

Backend/startup/initLogging.py

@@ -1,6 +1,7 @@
 import logging
 
 from Shared.functions.logging import ColourHighlighter, ElevatorLogger
+from Shared.functions.readSettingsFile import get_setting
 
 
 def init_logging() -> None:
@@ -12,12 +13,12 @@ def init_logging() -> None:
     )
     ElevatorLogger.make_console_logger(
         logger=logging.getLogger("uvicorn"),
-        level=logging.NOTSET,
+        level=logging.WARNING if not get_setting("ENABLE_DEBUG_MODE") else logging.NOTSET,
         highlighter=ColourHighlighter(name="uvicorn", colour="yellow"),
     )
     ElevatorLogger.make_console_logger(
         logger=logging.getLogger("uvicorn.access"),
-        level=logging.NOTSET,
+        level=logging.WARNING if not get_setting("ENABLE_DEBUG_MODE") else logging.NOTSET,
         highlighter=ColourHighlighter(name="uvicorn", colour="yellow"),
     )
     ElevatorLogger.make_console_logger(

ElevatorBot/backgroundEvents/clanRoleAssigment.py

@@ -6,6 +6,7 @@
 from ElevatorBot.core.misc.persistentMessages import PersistentMessages
 from ElevatorBot.discordEvents.base import ElevatorClient
 from ElevatorBot.networking.destiny.clan import DestinyClan
+from ElevatorBot.networking.errors import BackendException
 
 logger = logging.getLogger("backgroundEvents")
 
@@ -21,26 +22,25 @@ async def run(self, client: ElevatorClient):
         for guild in client.guilds:
             # get the linked clan role
             persistent_messages = PersistentMessages(ctx=None, guild=guild, message_name="clan_role")
-            result = await persistent_messages.get()
-
-            persistent_messages = PersistentMessages(ctx=None, guild=guild, message_name="clan_role")
-            if result := await persistent_messages.get():
-                if clan_role := await guild.fetch_role(result.channel_id):
-                    # get clan members
-                    destiny_clan = DestinyClan(ctx=None, discord_guild=guild)
-                    clan_members = await destiny_clan.get_clan_members()
-                    clan_members_discord_ids = [
-                        member.discord_id for member in clan_members.members if member.discord_id
-                    ]
-
-                    # check all members
-                    member: Member
-                    for member in guild.humans:
-                        if member.id not in clan_members_discord_ids:
-                            if member.has_role(clan_role):
-                                await member.remove_role(role=clan_role, reason="Destiny2 Clan Membership Update")
-                                logger.info(f"Removed clan role from {member}")
-                        else:
-                            if not member.has_role(clan_role):
-                                await member.add_role(role=clan_role, reason="Destiny2 Clan Membership Update")
-                                logger.info(f"Added clan role to {member}")
+            try:
+                result = await persistent_messages.get()
+            except BackendException:
+                return
+
+            if clan_role := await guild.fetch_role(result.channel_id):
+                # get clan members
+                destiny_clan = DestinyClan(ctx=None, discord_guild=guild)
+                clan_members = await destiny_clan.get_clan_members()
+                clan_members_discord_ids = [member.discord_id for member in clan_members.members if member.discord_id]
+
+                # check all members
+                member: Member
+                for member in guild.humans:
+                    if member.id not in clan_members_discord_ids:
+                        if member.has_role(clan_role):
+                            await member.remove_role(role=clan_role, reason="Destiny2 Clan Membership Update")
+                            logger.info(f"Removed clan role from {member}")
+                    else:
+                        if not member.has_role(clan_role):
+                            await member.add_role(role=clan_role, reason="Destiny2 Clan Membership Update")
+                            logger.info(f"Added clan role to {member}")

ElevatorBot/docs/commands.json

@@ -425,9 +425,6 @@
                                     "Orbs of Power Dropped",
                                     "Orbs of Power Gathered",
                                     "Time Played (in seconds)",
-                                    "Activities Cleared",
-                                    "Public Events Completed",
-                                    "Heroic Public Events Completed",
                                     "Kills with: Super",
                                     "Kills with: Melee",
                                     "Kills with: Grenade",
@@ -1614,12 +1611,6 @@
                 "default_required_permissions": null,
                 "enabled_in_dm": false
             },
-            {
-                "name": "metrics",
-                "description": "Shows interesting ElevatorBot metrics",
-                "default_required_permissions": "Administrator",
-                "enabled_in_dm": false
-            },
             {
                 "name": "mute",
                 "description": "Mutes the specified user",
@@ -1998,6 +1989,20 @@
                         "default_required_permissions": "Administrator",
                         "enabled_in_dm": false
                     },
+                    {
+                        "name": "setup clan role",
+                        "description": "Designate the role that is given to people that are in the linked clan",
+                        "default_required_permissions": "Administrator",
+                        "enabled_in_dm": false,
+                        "options": [
+                            {
+                                "name": "role",
+                                "description": "The role to link",
+                                "required": true,
+                                "autocomplete": false
+                            }
+                        ]
+                    },
                     {
                         "name": "setup clan unlink",
                         "description": "Unlinks the current Destiny 2 clan and the discord server this was executed in",

ElevatorBot/prometheus/extension.py

@@ -82,7 +82,7 @@ def _log_started_message(self, listeners: Sequence[socket.SocketType]) -> None:
 class Stats(naff.Extension):
     host = "0.0.0.0"
     port = 8877
-    interval = 5
+    interval = 15
 
     def __init__(self, bot):
         self.server: Optional[StatsServer] = None
@@ -136,16 +136,33 @@ async def on_ready(self) -> None:
     async def on_message_create(self, event: naff.events.MessageCreate):
         if guild := event.message.guild:
             counter = messages_counter.labels(
-                guild_id=guild.id, guild_name=guild.name, dm=0, user_id=event.message.author.id
+                guild_id=guild.id,
+                guild_name=guild.name,
+                dm=0,
+                user_id=event.message.author.id,
+                channel_id=event.message.channel.id,
+                channel_name=event.message.channel.name,
             )
         else:
-            counter = messages_counter.labels(guild_id=None, guild_name=None, dm=1, user_id=event.message.author.id)
+            counter = messages_counter.labels(
+                guild_id=None,
+                guild_name=None,
+                channel_id=None,
+                channel_name=None,
+                dm=1,
+                user_id=event.message.author.id,
+            )
 
         counter.inc()
 
     @naff.listen()
     async def on_voice_state_update(self, event: naff.events.VoiceStateUpdate):
-        if event.guild != descend_channels.guild:
+        if event.after:
+            guild = event.after.guild
+        else:
+            guild = event.before.guild
+
+        if guild != descend_channels.guild:
             return
 
         async with vc_lock:
@@ -154,7 +171,9 @@ async def on_voice_state_update(self, event: naff.events.VoiceStateUpdate):
                 user_id = event.before.member.id
                 if user_id in users_in_vs:
                     join_date = users_in_vs.pop(user_id)
-                    metric = descend_voice_channel_activity.labels(channel_id=event.before.channel.id, user_id=user_id)
+                    metric = descend_voice_channel_activity.labels(
+                        channel_id=event.before.channel.id, channel_name=event.before.channel.name, user_id=user_id
+                    )
                     metric.observe((get_now_with_tz() - join_date).seconds)
 
             # save user join time

ElevatorBot/prometheus/stats.py

@@ -25,7 +25,7 @@
 messages_counter = Counter(
     "naff_received_messages",
     "Amount of received messages",
-    labelnames=["guild_id", "guild_name", "dm", "user_id"],
+    labelnames=["guild_id", "guild_name", "channel_id", "channel_name", "dm", "user_id"],
 )
 
 guilds_gauge = Gauge("naff_guilds", "Amount of guilds this bot is in")
@@ -68,6 +68,6 @@
 descend_voice_channel_activity = Histogram(
     "naff_descend_voice_channel_activity",
     "How long users are in voice channels",
-    labelnames=["channel_id", "user_id"],
+    labelnames=["channel_id", "channel_name", "user_id"],
     buckets=BUCKETS,
 )

ElevatorBot/requirements.txt

@@ -10,7 +10,7 @@ dateparser==1.1.1
 gtts==2.2.4
 ics==0.7.2
 matplotlib==3.5.3
-naff[voice]==1.9.0
+naff[voice]==1.10.0
 orjson==3.8.0
 pandas==1.4.3
 pillow==9.2.0

ElevatorBot/startup/initLogging.py

@@ -1,6 +1,7 @@
 import logging
 
 from Shared.functions.logging import ColourHighlighter, ElevatorLogger
+from Shared.functions.readSettingsFile import get_setting
 
 
 def init_logging() -> None:
@@ -12,7 +13,12 @@ def init_logging() -> None:
     )
     ElevatorLogger.make_console_logger(
         logger=logging.getLogger("uvicorn"),
-        level=logging.NOTSET,
+        level=logging.WARNING if not get_setting("ENABLE_DEBUG_MODE") else logging.NOTSET,
+        highlighter=ColourHighlighter(name="uvicorn", colour="yellow"),
+    )
+    ElevatorLogger.make_console_logger(
+        logger=logging.getLogger("uvicorn.access"),
+        level=logging.WARNING if not get_setting("ENABLE_DEBUG_MODE") else logging.NOTSET,
         highlighter=ColourHighlighter(name="uvicorn", colour="yellow"),
     )