Audit after join, Audit Queueing, Swagger UI

.github/workflows/lint.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions/checkout@v2.3.3
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.x' # Version range or exact version of a Python version to use, using SemVer's version range syntax
+          python-version: '3.8' # Version range or exact version of a Python version to use, using SemVer's version range syntax
           architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
       - name: Install flake8
         run: pip install flake8
@@ -34,7 +34,7 @@ jobs:
       - uses: actions/checkout@v2.3.3
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.x' # Version range or exact version of a Python version to use, using SemVer's version range syntax
+          python-version: '3.8' # Version range or exact version of a Python version to use, using SemVer's version range syntax
           architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
       - name: Intall Pylint
         run: pip install pylint pylint-exit

.github/workflows/test.yml

@@ -7,7 +7,7 @@ jobs:
       - uses: actions/checkout@v2.3.3
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.x' # Version range or exact version of a Python version to use, using SemVer's version range syntax
+          python-version: '3.8' # Version range or exact version of a Python version to use, using SemVer's version range syntax
           architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
       - name: Install pytest
         run: pip install pytest

.gitignore

@@ -62,3 +62,5 @@ venv
 #Jetbrains IDE Settings
 .idea
 *.egg-info
+
+known_hosts

README.md

@@ -35,6 +35,12 @@ optional arguments:
                         Config file location
 ```
 
+#API
+This project has an [OpenAPI Specification](https://swagger.io/specification/), that is available under `http://localhost:10137/v2/api-docs` once the service started.
+A corresponding [Swagger UI](https://swagger.io/tools/swagger-ui/) is available under `http://localhost:10137/`.
+
+There are no api security mechanisms in place. It is assumed, that either a network based access policy will be enforced, or a reverse proxy handles authentication. 
+
 # Docker Compose
 ```
 version: "3.8"

bot.conf.example

@@ -8,15 +8,27 @@
 
 [teamspeak connection settings]
 
+# can be "telnet" or "ssh". Default is "telnet"
+protocol = telnet
+
 #can be IP or FQDN
 host = 127.0.0.1
 
-# Default port for ServerQuery access. Shouldn't need to change this.
+# 10011 = Default port for ServerQuery (telnet)
+# 10022 = Default port for ServerQuery (ssh)
 port = 10011
 
 user = serveradmin
 passwd = xxxxxxx
 
+# Only for SSH.
+# A standard known_hosts file.
+# Host keys can be initially queried via 'ssh-keyscan.exe -t ssh-rsa -p $port $host'
+# For some reason the above does not work on linux !?
+#
+# known_hosts_file  = ./known_hosts
+
+
 ## Pool Settings
 # Maximum connections in the pool. Should be >= 2
 pool_size = 4

bot/TS3Auth.py

@@ -11,19 +11,8 @@
 h_auth = '[AuthCheck]'
 h_char_chk = '[CharacterCheck]'
 
-#############################
-# Functions
-
-"""
-def log(msg,silent=False):
-    if not silent:
-        print (msg)
-        sys.stdout.flush()
-    with open(log_file,"a") as logger:
-        new_log = "%s %s\n" %(str(datetime.now()),msg)
-        logger.write(new_log)
-"""
 
+#############################
 
 # Class for an authentication request from user
 
@@ -39,10 +28,14 @@ def __init__(self, api_key, required_servers, required_level,
         self.account_details = {}
         self.world = None
         self.users_server = None
+        self.name = None
         self.id = -1
         self.guilds_error = False
         self.guilds = []
 
+        self.guild_tags = []
+        self.guild_names = []
+
         self.pushCharacterAuth()
         self.pushAccountAuth()
 
@@ -88,8 +81,7 @@ def getAccountDetails(self):
         self.guilds_error = False
 
         # Players Guild Tags (Seems to order it by oldest guild first)
-        self.guild_tags = []
-        self.guild_names = []
+
         for guild_id in self.guilds:
             try:
                 ginfo = gw2api.guild_get(guild_id)

bot/__init__.py

@@ -1,4 +1,8 @@
 from .TS3Bot import Bot
+from .audit_service import AuditService
+from .commander_service import CommanderService
 from .guild_service import GuildService
+from .reset_roster_service import ResetRosterService
+from .user_service import UserService
 
-__all__ = ['GuildService', 'Bot']
+__all__ = ['Bot', 'UserService', 'CommanderService', 'ResetRosterService', 'AuditService', 'GuildService']

bot/__main__.py

@@ -1,14 +1,20 @@
 import logging
 
-from bot.main import main
+from bot.main import main, parse_args
 from bot.util import initialize_logging
 
 LOG = logging.getLogger(__name__)
 
 
 def startup():
-    initialize_logging()
-    main()
+    config, parser = parse_args()
+
+    initialize_logging(config.logging_file, config.logging_level)
+
+    if LOG.isEnabledFor(logging.DEBUG):
+        LOG.debug("Config Sources:\n%s", parser.format_values())
+
+    main(config)
 
 
 if __name__ == '__main__':

bot/audit_service.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import datetime
+import logging
+import threading
+from dataclasses import dataclass, field
+from datetime import date
+from queue import PriorityQueue
+
+from bot.TS3Auth import AuthRequest
+from bot.config import Config
+from bot.connection_pool import ConnectionPool
+from bot.db import ThreadSafeDBConnection
+from bot.ts import TS3Facade, User
+from .user_service import UserService
+
+LOG = logging.getLogger(__name__)
+
+# Queue Priorities, lower entries will be handles before, larger entries.
+
+QUEUE_PRIORITY_AUDIT = 100
+QUEUE_PRIORITY_JOIN = 20
+
+
+@dataclass(order=True)
+class AuditQueueEntry:
+    priority: int
+    account_name: str = field(compare=False)
+    api_key: str = field(compare=False)
+    client_unique_id: str = field(compare=False)
+
+
+class AuditService:
+    def __init__(self, database_connection_pool: ThreadSafeDBConnection, ts_connection_pool: ConnectionPool[TS3Facade], config: Config, user_service: UserService):
+        self._user_service = user_service
+        self._database_connection = database_connection_pool
+        self._ts_connection_pool = ts_connection_pool
+        self._config = config
+
+        self._audit_queue: PriorityQueue[AuditQueueEntry] = PriorityQueue()  # pylint: disable=unsubscriptable-object
+        self._start_audit_queue_worker()
+
+    def queue_user_audit(self, priority: int, account_name: str, api_key: str, client_unique_id: str):
+        queue_entry = AuditQueueEntry(priority, account_name=account_name, api_key=api_key, client_unique_id=client_unique_id)
+        LOG.debug("Adding entry to audit queue for : %s", account_name)
+        self._audit_queue.put(queue_entry)
+
+    def _start_audit_queue_worker(self):
+        def worker():
+            LOG.info("Audit Worker is ready and pulling audit jobs")
+            while True:
+                item = self._audit_queue.get()
+                LOG.debug('Working on %s. %s still in queue.', item.account_name, self._audit_queue.qsize())
+                self.audit_user(item.account_name, item.api_key, item.client_unique_id)
+                LOG.debug('Finished %s', item.account_name)
+                self._audit_queue.task_done()
+
+                # Log queue size if over ten
+                qsize: int = self._audit_queue.qsize()
+                if qsize >= 10:
+                    LOG.info("Queue Size: %s", qsize)
+
+        # start worker - in theory there could be more than one thread, but this will cause stress on the gw2-api, database and teamspeak
+        threading.Thread(name="AuditQueueWorker", target=worker, daemon=True).start()
+
+    def audit_user(self, account_name, api_key, client_unique_id):
+        auth = AuthRequest(api_key, self._config.required_servers, int(self._config.required_level), account_name)
+        if auth.success:
+            LOG.info("User %s is still on %s. Successful audit!", account_name, auth.world.get("name"))
+            with self._ts_connection_pool.item() as ts_facade:
+                self._user_service.update_guild_tags(ts_facade, User(ts_facade, unique_id=client_unique_id), auth)
+            with self._database_connection.lock:
+                self._database_connection.cursor.execute("UPDATE users SET last_audit_date = ? WHERE ts_db_id= ?", ((date.today()), client_unique_id,))
+                self._database_connection.conn.commit()
+        else:
+            LOG.info("User %s is no longer on our server. Removing access....", account_name)
+            self._user_service.remove_permissions(client_unique_id)
+            self._user_service.remove_user_from_db(client_unique_id)
+
+    def trigger_user_audit(self):
+        LOG.info("Auditing users")
+        threading.Thread(name="FullAudit", target=self._audit_users, daemon=True).start()
+
+    def _audit_users(self):
+        current_audit_date = datetime.date.today()  # Update current date everytime run
+
+        with self._database_connection.lock:
+            db_audit_list = self._database_connection.cursor.execute('SELECT * FROM users').fetchall()
+        for audit_user in db_audit_list:
+
+            # Convert to single variables
+            audit_ts_id = audit_user[0]
+            audit_account_name = audit_user[1]
+            audit_api_key = audit_user[2]
+            # audit_created_date = audit_user[3]
+            audit_last_audit_date = audit_user[4]
+
+            LOG.debug("Audit: User %s", audit_account_name)
+            LOG.debug("TODAY |%s|  NEXT AUDIT |%s|", current_audit_date, audit_last_audit_date + datetime.timedelta(days=self._config.audit_period))
+
+            if current_audit_date >= audit_last_audit_date + datetime.timedelta(days=self._config.audit_period):  # compare audit date
+                with self._ts_connection_pool.item() as ts_connection:
+                    ts_uuid = ts_connection.client_db_id_from_uid(audit_ts_id)
+                if ts_uuid is None:
+                    LOG.info("User %s is not found in TS DB and could be deleted.", audit_account_name)
+                    self._database_connection.cursor.execute("UPDATE users SET last_audit_date = ? WHERE ts_db_id= ?", ((datetime.date.today()), audit_ts_id,))
+                    # self.removeUserFromDB(audit_ts_id)
+                else:
+                    LOG.info("User %s is due for auditing! Queueing", audit_account_name)
+                    self.queue_user_audit(QUEUE_PRIORITY_AUDIT, audit_account_name, audit_api_key, audit_ts_id)
+
+        with self._database_connection.lock:
+            self._database_connection.cursor.execute('INSERT INTO bot_info (last_succesful_audit) VALUES (?)', (current_audit_date,))
+            self._database_connection.conn.commit()
+
+    def audit_user_on_join(self, client_unique_id):
+        db_entry = self._user_service.get_user_database_entry(client_unique_id)
+        if db_entry is not None:
+            account_name = db_entry["account_name"]
+            api_key = db_entry["api_key"]
+            self.queue_user_audit(QUEUE_PRIORITY_JOIN, account_name=account_name, api_key=api_key, client_unique_id=client_unique_id)

bot/command_checker.py → bot/commander_service.py

@@ -1,35 +1,40 @@
 import logging
 
+from bot.config import Config
 from bot.ts.user import User
+from .connection_pool import ConnectionPool
+from .ts import TS3Facade
+from .user_service import UserService
 
 LOG = logging.getLogger(__name__)
 
 
-class CommanderChecker:
-    def __init__(self, ts3bot, commander_group_names):
-        self._commander_group_names = commander_group_names
-        self._ts3bot = ts3bot
+class CommanderService:
+    def __init__(self, ts_connection_pool: ConnectionPool[TS3Facade], user_service: UserService, config: Config):
+        self._commander_group_names = config.poll_group_names
+        self._ts_connection_pool = ts_connection_pool
+        self._user_service = user_service
 
-        with self._ts3bot.ts_connection_pool.item() as facade:
+        with self._ts_connection_pool.item() as facade:
             channel_list, ex = facade.channelgroup_list()
-            cgroups = list(filter(lambda g: g.get("name") in commander_group_names, channel_list))
+            cgroups = list(filter(lambda g: g.get("name") in self._commander_group_names, channel_list))
         if len(cgroups) < 1:
-            LOG.info("Could not find any group of %s to determine commanders by. Disabling this feature.", str(commander_group_names))
+            LOG.info("Could not find any group of %s to determine commanders by. Disabling this feature.", str(self._commander_group_names))
             self._commander_groups = []
             return
 
         self._commander_groups = [c.get("cgid") for c in cgroups]
 
-    def execute(self):
+    def get_active_commanders(self):
         if not self._commander_groups:
             return  # disabled if no groups were found
 
         active_commanders = []
 
-        with self._ts3bot.ts_connection_pool.item() as ts_facade:
+        with self._ts_connection_pool.item() as ts_facade:
             acs = ts_facade.channelgroup_client_list(self._commander_groups)
             LOG.info(acs)
-            active_commanders_entries = [(c, self._ts3bot.getUserDBEntry(ts_facade.client_get_name_from_dbid(client_dbid=c.get("cldbid")).get("cluid"))) for c in acs]
+            active_commanders_entries = [(c, self._user_service.get_user_database_entry(ts_facade.client_get_name_from_dbid(client_dbid=c.get("cldbid")).get("cluid"))) for c in acs]
             for ts_entry, db_entry in active_commanders_entries:
                 if db_entry is not None:  # or else the user with the commander group was not registered and therefore not in the DB
                     user = User(ts_facade, ts_db_id=ts_entry.get("cldbid"))

bot/config/config.py

@@ -15,10 +15,12 @@ def __init__(self, config_file_path: str) -> None:
         configs.read(config_file_path, "utf-8")
 
         # Teamspeak Connection Settings
+        self.protocol = configs.get("teamspeak connection settings", "protocol", fallback="telnet")  # telnet or ssh
         self.host = configs.get("teamspeak connection settings", "host")
         self.port = configs.get("teamspeak connection settings", "port")
         self.user = configs.get("teamspeak connection settings", "user")
         self.passwd = configs.get("teamspeak connection settings", "passwd")
+        self.known_hosts_file = configs.get("teamspeak connection settings", "known_hosts_file", fallback=None)
 
         self.pool_size = self._try_get(configs, "teamspeak connection settings", "pool_size", 4)
         self.pool_ttl = self._try_get(configs, "teamspeak connection settings", "pool_ttl", 600)
@@ -42,7 +44,6 @@ def __init__(self, config_file_path: str) -> None:
         self.audit_period = int(configs.get("bot settings", "audit_period"))  # How long a single user can go without being audited
         self.audit_interval = int(configs.get("bot settings", "audit_interval"))  # how often the BOT audits all users
         self.client_restriction_limit = int(configs.get("bot settings", "client_restriction_limit"))
-        self.timer_msg_broadcast = int(configs.get("bot settings", "broadcast_message_timer"))
 
         # tryGet(config, section, key, default = None, lit_eval = False):
         self.purge_completely = self._try_get(configs, "bot settings", "purge_completely", False, True)