From 44b5dd991bc556b7a6ff6abb502952fb2e963510 Mon Sep 17 00:00:00 2001 From: Paramjeet Date: Thu, 22 Jul 2021 09:36:38 +0530 Subject: [PATCH 01/11] Created tables necessary for storing clist users --- tle/util/db/user_db_conn.py | 97 ++++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/tle/util/db/user_db_conn.py b/tle/util/db/user_db_conn.py index 1af18faf..c6cb3575 100644 --- a/tle/util/db/user_db_conn.py +++ b/tle/util/db/user_db_conn.py @@ -95,6 +95,25 @@ def create_tables(self): 'title_photo TEXT' ')' ) + self.conn.execute( + 'CREATE TABLE IF NOT EXISTS clist_account_ids (' + 'guild_id TEXT,' + 'user_id TEXT,' + 'resource TEXT,' + 'account_id INTEGER,' + 'PRIMARY KEY (user_id, guild_id, resource)' + ')' + ) + self.conn.execute( + 'CREATE TABLE IF NOT EXISTS clist_user_cache (' + 'account_id INTEGER PRIMARY KEY,' + 'resource TEXT,' + 'handle TEXT,' + 'name TEXT,' + 'rating INTEGER,' + 'contests INTEGER' + ')' + ) # TODO: Make duel tables guild-aware. self.conn.execute(''' CREATE TABLE IF NOT EXISTS duelist( @@ -396,6 +415,77 @@ def get_handle(self, user_id, guild_id): res = self.conn.execute(query, (user_id, guild_id)).fetchone() return res[0] if res else None + def set_account_id(self, user_id, guild_id, account_id, resource): + query = ('SELECT user_id ' + 'FROM clist_account_ids ' + 'WHERE guild_id = ? AND account_id = ?') + existing = self.conn.execute(query, (guild_id, account_id)).fetchone() + if existing and int(existing[0]) != user_id: + raise UniqueConstraintFailed + + query = ('INSERT OR REPLACE INTO clist_account_ids ' + '(guild_id, account_id, user_id, resource) ' + 'VALUES (?, ?, ?, ?)') + res = None + with self.conn: + res = self.conn.execute(query, (guild_id, account_id, user_id, resource)).rowcount + return res + + def get_account_id(self, user_id, guild_id, resource): + query = ('SELECT account_id ' + 'FROM clist_account_ids ' + 'WHERE user_id = ? AND guild_id = ? AND resource = ?') + res = self.conn.execute(query, (user_id, guild_id, resource)).fetchone() + return res[0] if res else None + + def get_account_ids_for_user(self, user_id, guild_id): + query = ('SELECT account_id, resource ' + 'FROM clist_account_ids ' + 'WHERE user_id = ? AND guild_id = ?') + res = self.conn.execute(query, (user_id, guild_id)).fetchall() + ans = {} + for account_id, resource in res: + ans[resource] = account_id + return ans + + def get_account_ids_for_guild(self, guild_id, resource): + query = ('SELECT user_id, account_id ' + 'FROM clist_account_ids ' + 'WHERE guild_id = ? AND resource = ?') + res = self.conn.execute(query, (guild_id, resource)).fetchall() + return [(int(user_id), int(account_id)) for user_id, account_id in res] + + def get_clist_users_for_guild(self, guild_id, resource): + query = ('SELECT u.user_id, c.account_id , c.resource, c.handle, c.name, c.rating, c.contests ' + 'FROM clist_account_ids AS u ' + 'LEFT JOIN clist_user_cache AS c ' + 'ON u.account_id = c.account_id ' + 'WHERE u.guild_id = ? AND u.resource = ? ') + res = self.conn.execute(query, (guild_id,resource,)).fetchall() + + return [{'user_id':user_id, 'id':account_id, 'resource':resource, + 'handle':handle, 'name':name, 'rating':rating, 'n_contests':contests} + for user_id, account_id, resource, handle, name, rating, contests in res] + + def cache_clist_user(self, user): + query = ('INSERT OR REPLACE INTO clist_user_cache ' + '(account_id, resource, handle, name, rating, contests) ' + 'VALUES (?, ?, ?, ?, ?, ?)') + with self.conn: + return self.conn.execute(query, (user['id'], user['resource'], user['handle'], + user['name'], user['rating'], user['n_contests'])).rowcount + + def fetch_clist_user(self, account_id): + query = ('SELECT account_id, resource, handle, name, rating, contests ' + 'FROM clist_user_cache ' + 'WHERE account_id = ?') + user = self.conn.execute(query, (account_id,)).fetchone() + if not user: + return None + account_id, resource, handle, name, rating, contests = user + return {'id':account_id, 'resource':resource, 'handle':handle, + 'name':name, 'rating':rating, 'n_contests':contests} + def get_user_id(self, handle, guild_id): query = ('SELECT user_id ' 'FROM user_handle ' @@ -404,10 +494,13 @@ def get_user_id(self, handle, guild_id): return int(res[0]) if res else None def remove_handle(self, user_id, guild_id): - query = ('DELETE FROM user_handle ' + query1 = ('DELETE FROM user_handle ' + 'WHERE user_id = ? AND guild_id = ?') + query2 = ('DELETE FROM clist_account_ids ' 'WHERE user_id = ? AND guild_id = ?') with self.conn: - return self.conn.execute(query, (user_id, guild_id)).rowcount + self.conn.execute(query1, (user_id, guild_id)).rowcount + self.conn.execute(query2, (user_id, guild_id)).rowcount def get_handles_for_guild(self, guild_id): query = ('SELECT user_id, handle ' From 8d725ad960425da816ed07452cb29f985c9b1947 Mon Sep 17 00:00:00 2001 From: Paramjeet Date: Thu, 22 Jul 2021 09:58:20 +0530 Subject: [PATCH 02/11] Implemented clist api call and scraper for handle linking --- poetry.lock | 34 +++++++++++++- pyproject.toml | 1 + tle/util/clist_api.py | 105 ++++++++++++++++++++++++++++++++++++++++++ tle/util/scaper.py | 25 ++++++++++ 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 tle/util/clist_api.py create mode 100644 tle/util/scaper.py diff --git a/poetry.lock b/poetry.lock index e9789c1f..7e572499 100644 --- a/poetry.lock +++ b/poetry.lock @@ -60,6 +60,21 @@ docs = ["furo", "sphinx", "zope.interface"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] +[[package]] +name = "beautifulsoup4" +version = "4.9.3" +description = "Screen-scraping library" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +soupsieve = {version = ">1.2", markers = "python_version >= \"3.0\""} + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "chardet" version = "3.0.4" @@ -328,6 +343,14 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "soupsieve" +version = "2.2.1" +description = "A modern CSS selector implementation for Beautiful Soup." +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "typing-extensions" version = "3.7.4.3" @@ -364,7 +387,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "27752f5523c1868b5b7788430a0636433dd2ca5bc1f5884faf6ac780ac4dba50" +content-hash = "c1e423d090ffc4df8054161514ace92d6353adf750911797fea07f84f689176d" [metadata.files] aiocache = [ @@ -398,6 +421,11 @@ attrs = [ {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, ] +beautifulsoup4 = [ + {file = "beautifulsoup4-4.9.3-py2-none-any.whl", hash = "sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35"}, + {file = "beautifulsoup4-4.9.3-py3-none-any.whl", hash = "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"}, + {file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"}, +] chardet = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, @@ -724,6 +752,10 @@ six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] +soupsieve = [ + {file = "soupsieve-2.2.1-py3-none-any.whl", hash = "sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b"}, + {file = "soupsieve-2.2.1.tar.gz", hash = "sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc"}, +] typing-extensions = [ {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, diff --git a/pyproject.toml b/pyproject.toml index 1fe13d9e..985a3ee5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ pillow = "^5.4" pycairo = "^1.19.1" PyGObject = "^3.34.0" aiocache = "^0.11.1" +beautifulsoup4 = "^4.9.3" [tool.poetry.dev-dependencies] pytest = "^3.0" diff --git a/tle/util/clist_api.py b/tle/util/clist_api.py new file mode 100644 index 00000000..32dca604 --- /dev/null +++ b/tle/util/clist_api.py @@ -0,0 +1,105 @@ +import logging +import os +import requests + +from discord.ext import commands + +import functools +import asyncio +from urllib.parse import urlencode + +logger = logging.getLogger(__name__) +URL_BASE = 'https://clist.by/api/v2/' +_SUPPORTED_CLIST_RESOURCES = ('codechef.com', 'atcoder.jp','codingcompetitions.withgoogle.com') +_CLIST_RESOURCE_SHORT_FORMS = {'cc':'codechef.com','codechef':'codechef.com', 'cf':'codeforces.com', + 'codeforces':'codeforces.com','ac':'atcoder.jp', 'atcoder':'atcoder.jp', + 'google':'codingcompetitions.withgoogle.com'} + +class ClistNotConfiguredError(commands.CommandError): + """An error caused when clist credentials are not set in environment variables""" + def __init__(self, message=None): + super().__init__(message or 'Clist API not configured') + +class ClistApiError(commands.CommandError): + """Base class for all API related errors.""" + + def __init__(self, message=None): + super().__init__(message or 'Clist API error') + + +class ClientError(ClistApiError): + """An error caused by a request to the API failing.""" + + def __init__(self): + super().__init__('Error connecting to Clist API') + +class TrueApiError(ClistApiError): + """An error originating from a valid response of the API.""" + def __init__(self, comment=None, message=None): + super().__init__(message) + self.comment = comment + +class HandleNotFoundError(TrueApiError): + def __init__(self, handle, resource=None): + super().__init__(message=f'Handle `{handle}` not found{" on `"+str(resource)+"`" if resource!=None else "."}') + self.handle = handle + +class CallLimitExceededError(TrueApiError): + def __init__(self, comment=None): + super().__init__(message='Clist API call limit exceeded') + self.comment = comment + +def ratelimit(f): + tries = 3 + @functools.wraps(f) + async def wrapped(*args, **kwargs): + for i in range(tries): + delay = 10 + await asyncio.sleep(delay*i) + try: + return await f(*args, **kwargs) + except (ClientError, CallLimitExceededError, ClistApiError) as e: + logger.info(f'Try {i+1}/{tries} at query failed.') + if i < tries - 1: + logger.info(f'Retrying...') + else: + logger.info(f'Aborting.') + raise e + return wrapped + + +@ratelimit +async def _query_clist_api(path, data): + url = URL_BASE + path + clist_token = os.getenv('CLIST_API_TOKEN') + if not clist_token: + raise ClistNotConfiguredError + if data is None: + url += '?'+clist_token + else: + url += '?'+ str(urlencode(data)) + url+='&'+clist_token + try: + resp = requests.get(url) + if resp.status_code != 200: + if resp.status_code == 429: + raise CallLimitExceededError + else: + raise ClistApiError + return resp.json() + except Exception as e: + logger.error(f'Request to Clist API encountered error: {e!r}') + raise ClientError from e + +async def account(handle, resource): + params = {'total_count': True, 'handle':handle} + if resource!=None: + params['resource'] = resource + resp = await _query_clist_api('account', params) + if resp==None or 'objects' not in resp: + raise ClientError + else: + resp = resp['objects'] + if len(resp)==0: + raise HandleNotFoundError(handle=handle, resource=resource) + return resp \ No newline at end of file diff --git a/tle/util/scaper.py b/tle/util/scaper.py new file mode 100644 index 00000000..65dccf90 --- /dev/null +++ b/tle/util/scaper.py @@ -0,0 +1,25 @@ +from bs4 import BeautifulSoup +import requests + +def assert_display_name(username, token, resource, mention): + if resource=='codechef.com': + response = requests.get("https://codechef.com/users/"+str(username)) + if response.status_code != 200: + return False + else: + soup = BeautifulSoup(response.content, 'html.parser') + elements = soup.find_all(class_='h2-style') + for element in elements: + if token in element.text: + return True + elif resource=='atcoder.jp': + response = requests.get("https://atcoder.jp/users/"+str(username)) + if response.status_code != 200: + return False + else: + soup = BeautifulSoup(response.content, 'html.parser') + elements = soup.find_all(class_='break-all') + for element in elements: + if token in element.text: + return True + return False \ No newline at end of file From 4347ac19d6d997ce31b67be10d463eec7f650ad4 Mon Sep 17 00:00:00 2001 From: Paramjeet Date: Thu, 22 Jul 2021 10:01:11 +0530 Subject: [PATCH 03/11] Finished ;handle identify/set for CodeChef/AtCoder/Google(Kickstart) --- tle/cogs/handles.py | 98 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 3 deletions(-) diff --git a/tle/cogs/handles.py b/tle/cogs/handles.py index e97671f8..b6f38b41 100644 --- a/tle/cogs/handles.py +++ b/tle/cogs/handles.py @@ -13,13 +13,16 @@ from gi.repository import Pango, PangoCairo import discord -import random +import random, string from discord.ext import commands from tle import constants from tle.util import cache_system2 from tle.util import codeforces_api as cf from tle.util import codeforces_common as cf_common +from tle.util import clist_api as clist +from tle.util.clist_api import _CLIST_RESOURCE_SHORT_FORMS, _SUPPORTED_CLIST_RESOURCES +from tle.util import scaper from tle.util import discord_common from tle.util import events from tle.util import paginator @@ -79,6 +82,10 @@ def rating_to_color(rating): 'Noto Sans CJK KR', ] +def randomword(length): + letters = string.ascii_lowercase + return ''.join(random.choice(letters) for i in range(length)) + def get_gudgitters_image(rankings): """return PIL image for rankings""" SMOKE_WHITE = (250, 250, 250) @@ -341,12 +348,49 @@ async def update_member_rank_role(member, role_to_assign, *, reason): @handle.command(brief='Set Codeforces handle of a user') @commands.has_any_role(constants.TLE_ADMIN, constants.TLE_MODERATOR) async def set(self, ctx, member: discord.Member, handle: str): - """Set Codeforces handle of a user.""" + """Set codeforces/codechef/atcoder/google handle of a user. + + Some examples are given below + ;handle set @Benjamin Benq + ;handle set @Kamil cf:Errichto + ;handle set @Gennady codechef:gennady.korotkevich + ;handle set @Paramjeet cc:thesupremeone + ;handle set @Jatin atcoder:nagpaljatin1411 + ;handle set @Alex ac:Um_nik + ;handle set @Priyansh google:Priyansh31dec + """ + resource = 'codeforces.com' + if ':' in handle: + resource = handle[0: handle.index(':')] + handle = handle[handle.index(':')+1:] + if resource in _CLIST_RESOURCE_SHORT_FORMS: + resource = _CLIST_RESOURCE_SHORT_FORMS[resource] + if resource!='codeforces.com': + if resource=='all': + resource = None + if resource!=None and resource not in _SUPPORTED_CLIST_RESOURCES: + raise HandleCogError(f'The resource `{resource}` is not supported.') + users = await clist.account(handle=handle, resource=resource) + message = f'Following handles for `{member.mention}` have been linked\n' + for user in users: + if user['resource'] not in _SUPPORTED_CLIST_RESOURCES: + continue + message += user['resource']+' : '+user['handle']+'\n' + await self._set_account_id(member.id, ctx.guild, user) + return await ctx.send(message) # CF API returns correct handle ignoring case, update to it user, = await cf.user.info(handles=[handle]) await self._set(ctx, member, user) embed = _make_profile_embed(member, user, mode='set') await ctx.send(embed=embed) + + async def _set_account_id(self, member_id, guild, user): + try: + guild_id = guild.id + cf_common.user_db.set_account_id(member_id, guild_id, user['id'], user['resource']) + except db.UniqueConstraintFailed: + raise HandleCogError(f'The handle `{user["handle"]}` is already associated with another user.') + cf_common.user_db.cache_clist_user(user) async def _set(self, ctx, member, user): handle = user.handle @@ -370,7 +414,55 @@ async def _set(self, ctx, member, user): @cf_common.user_guard(group='handle', get_exception=lambda: HandleCogError('Identification is already running for you')) async def identify(self, ctx, handle: str): - """Link a codeforces account to discord account by submitting a compile error to a random problem""" + """Link a codeforces/codechef/atcoder account to discord account + + Some examples are given below + ;handle identify Benq + ;handle identify cf:Errichto + ;handle identify codechef:gennady.korotkevich + ;handle identify cc:thesupremeone + ;handle identify atcoder:nagpaljatin1411 + ;handle identify ac:Um_nik + + For linking google handles, please contact a moderator + """ + + invoker = str(ctx.author) + resource = 'codeforces.com' + + if ':' in handle: + resource = handle[0: handle.index(':')] + handle = handle[handle.index(':')+1:] + if resource in _CLIST_RESOURCE_SHORT_FORMS: + resource = _CLIST_RESOURCE_SHORT_FORMS[resource] + + if resource!='codeforces.com': + if resource=='all': + return await ctx.send(f'Sorry `{invoker}`, all keyword can only be used with set command') + if resource=='codingcompetitions.withgoogle.com': + return await ctx.send(f'Sorry `{invoker}`, you can\'t identify handles of codingcompetitions.withgoogle.com, please ask a moderator to link your account.') e + if resource not in _SUPPORTED_CLIST_RESOURCES: + raise HandleCogError(f'The resource `{resource}` is not supported.') + wait_msg = await ctx.channel.send('Fetching account details, please wait...') + users = await clist.account(handle, resource) + user = users[0] + token = randomword(8) + await wait_msg.delete() + field = "name" + if resource=='atcoder.jp': field = 'affiliation' + wait_msg = await ctx.send(f'`{invoker}`, change your {field} to `{token}` on {resource} within 60 seconds') + await asyncio.sleep(60) + await wait_msg.delete() + wait_msg = await ctx.channel.send(f'Verifying {field} change...') + if scaper.assert_display_name(handle, token, resource, ctx.author.mention): + member = ctx.author + await self._set_account_id(member.id, ctx.guild, user) + await wait_msg.delete() + await ctx.send(f'Your handle is now linked, `{invoker}`') + else: + await wait_msg.delete() + await ctx.send(f'Sorry `{invoker}`, can you try again?') + return if cf_common.user_db.get_handle(ctx.author.id, ctx.guild.id): raise HandleCogError(f'{ctx.author.mention}, you cannot identify when your handle is ' 'already set. Ask an Admin or Moderator if you wish to change it') From 161911338b454217468f47dbc64039d9eacccb58 Mon Sep 17 00:00:00 2001 From: Paramjeet Date: Thu, 22 Jul 2021 10:21:37 +0530 Subject: [PATCH 04/11] Implemented ranklist for CodeChef/AtCoder/Google(Kickstart) --- tle/cogs/contests.py | 209 +++++++++++++++++++++++++++++++++- tle/util/clist_api.py | 70 ++++++++++++ tle/util/codeforces_common.py | 53 +++++++-- tle/util/db/user_db_conn.py | 7 ++ 4 files changed, 327 insertions(+), 12 deletions(-) diff --git a/tle/cogs/contests.py b/tle/cogs/contests.py index 024f704b..e2a64564 100644 --- a/tle/cogs/contests.py +++ b/tle/cogs/contests.py @@ -14,6 +14,7 @@ from tle.util import codeforces_common as cf_common from tle.util import cache_system2 from tle.util import codeforces_api as cf +from tle.util import clist_api as clist from tle.util import db from tle.util import discord_common from tle.util import events @@ -32,6 +33,32 @@ _RATED_VC_EXTRA_TIME = 10 * 60 # seconds _MIN_RATED_CONTESTANTS_FOR_RATED_VC = 50 +_PATTERNS = { + 'abc': 'atcoder.jp', + 'arc': 'atcoder.jp', + 'agc': 'atcoder.jp', + 'kickstart': 'codingcompetitions.withgoogle.com', + 'codejam': 'codingcompetitions.withgoogle.com', + 'lunchtime': 'codechef.com', + 'long': 'codechef.com', + 'cookoff': 'codechef.com', + 'starters': 'codechef.com' +} + +def parse_date(arg): + try: + if len(arg) == 8: + fmt = '%d%m%Y' + elif len(arg) == 6: + fmt = '%m%Y' + elif len(arg) == 4: + fmt = '%Y' + else: + raise ValueError + return dt.datetime.strptime(arg, fmt) + except ValueError: + raise ContestCogError(f'{arg} is an invalid date argument') + class ContestCogError(commands.CommandError): pass @@ -426,6 +453,57 @@ def _make_contest_embed_for_ranklist(ranklist): msg = f'{start}{en}|{en}{duration}{en}|{en}Ended {since} ago' embed.add_field(name='When', value=msg, inline=False) return embed + + def _make_clist_standings_pages(self, standings): + if standings is None or len(standings)==0: + return "```No handles found inside ranklist```" + show_rating_changes = standings[0]['rating_change']!=None + + pages = [] + standings_chunks = paginator.chunkify(standings, _STANDINGS_PER_PAGE) + num_chunks = len(standings_chunks) + + if not show_rating_changes: + header_style = '{:>} {:<} {:^} ' + body_style = '{:>} {:<} {:<} ' + header = ['#', 'Name', 'Score'] + else: + header_style = '{:>} {:<} {:^} {:<} {:<} ' + body_style = '{:>} {:<} {:<} {:<} {:<} ' + header = ['#', 'Name', 'Score', 'Delta', 'New Rating'] + + num_pages = 1 + for standings_chunk in standings_chunks: + body = [] + for standing in standings_chunk: + score = int(standing['score']) if standing['score'] else 0 + if show_rating_changes: + delta = int(standing['rating_change']) if standing['rating_change'] else ' ' + if delta!=' ': + delta = '+'+str(delta) if delta>0 else str(delta) + tokens = [int(standing['place']), standing['handle'], score, delta, standing['new_rating']] + else: + tokens = [int(standing['place']), standing['handle'], score] + body.append(tokens) + t = table.Table(table.Style(header=header_style, body=body_style)) + t += table.Header(*header) + t += table.Line('\N{EM DASH}') + for row in body: + t += table.Data(*row) + t += table.Line('\N{EM DASH}') + page_num_footer = f' # Page: {num_pages} / {num_chunks}' if num_chunks > 1 else '' + + # We use yaml to get nice colors in the ranklist. + content = f'```yaml\n{t}\n{page_num_footer}```' + pages.append((content, None)) + num_pages += 1 + return pages + + @staticmethod + def _make_contest_embed_for_cranklist(contest): + embed = discord_common.cf_color_embed(title=contest['event'], url=contest['href']) + embed.add_field(name='Website', value=contest['resource']) + return embed @staticmethod def _make_contest_embed_for_vc_ranklist(ranklist, vc_start_time=None, vc_end_time=None): @@ -440,15 +518,144 @@ def _make_contest_embed_for_vc_ranklist(ranklist, vc_start_time=None, vc_end_tim msg = f'{elapsed} elapsed{en}|{en}{remaining} remaining' embed.add_field(name='Tick tock', value=msg, inline=False) return embed + + async def resolve_contest(self, contest_id, resource): + contest = None + if resource=='clist.by': + contest = await clist.contest(contest_id) + elif resource=='atcoder.jp': + prefix = contest_id[:3] + if prefix=='abc': + prefix = 'AtCoder Beginner Contest ' + if prefix=='arc': + prefix = 'AtCoder Regular Contest ' + if prefix=='agc': + prefix = 'AtCoder Grand Contest ' + suffix = contest_id[3:] + try: + suffix = int(suffix) + except: + raise ContestCogError('Invalid contest_id provided.') + contest_name = prefix+str(suffix) + contests = await clist.search_contest(regex=contest_name, resource=resource) + if contests==None or len(contests)==0: + raise ContestCogError('Contest not found.') + contest = contests[0] + elif resource=='codechef.com': + contest_name = None + if 'lunchtime' in contest_id: + date = parse_date(contest_id[9:]) + contest_name = str(date.strftime('%B'))+' Lunchtime '+str(date.strftime('%Y')) + elif 'cookoff' in contest_id: + date = parse_date(contest_id[7:]) + contest_name = str(date.strftime('%B'))+' Cook-Off '+str(date.strftime('%Y')) + elif 'long' in contest_id: + date = parse_date(contest_id[4:]) + contest_name = str(date.strftime('%B'))+' Challenge '+str(date.strftime('%Y')) + elif 'starters' in contest_id: + date = parse_date(contest_id[8:]) + contest_name = str(date.strftime('%B'))+' CodeChef Starters '+str(date.strftime('%Y')) + contests = await clist.search_contest(regex=contest_name, resource=resource) + if contests==None or len(contests)==0: + raise ContestCogError('Contest not found.') + contest = contests[0] + elif resource=='codingcompetitions.withgoogle.com': + year,round = None,None + contest_name = None + if 'kickstart' in contest_id: + year = contest_id[9:11] + round = contest_id[11:] + contest_name = 'Kick Start.*Round '+round + elif 'codejam' in contest_id: + year = contest_id[7:9] + round = contest_id[9:] + if round=='WF': + round = 'Finals' + contest_name = 'Code Jam.*Finals' + elif round=='QR': + round = 'Qualification Round' + contest_name = 'Code Jam.*Qualification Round' + else: + contest_name = 'Code Jam.*Round '+round + if not round: + raise ContestCogError('Invalid contest_id provided.') + try: + year = int(year) + except: + raise ContestCogError('Invalid contest_id provided.') + start = dt.datetime(int('20'+str(year)), 1, 1) + end = dt.datetime(int('20'+str(year+1)), 1, 1) + date_limit = (start.strftime('%Y-%m-%dT%H:%M:%S'), end.strftime('%Y-%m-%dT%H:%M:%S')) + contests = await clist.search_contest(regex=contest_name, resource=resource, date_limits=date_limit) + if contests==None or len(contests)==0: + raise ContestCogError('Contest not found.') + contest = contests[0] + return contest @commands.command(brief='Show ranklist for given handles and/or server members') async def ranklist(self, ctx, contest_id: int, *handles: str): """Shows ranklist for the contest with given contest id. If handles contains '+server', all server members are included. No handles defaults to '+server'. + + You can frame contest_id as follow + + # For codeforces ranklist + Enter codeforces contest id + + # For codechef ranklist + long + lunchtime + cookoff + starters + + # For atcoder ranklist + abc + arc + agc + + # For google ranklist + kickstart + codejam + + Use QR for Qualification Round and WF for World Finals. + + # If nothing works + Use clist contest_id. You have to prefix - sign to clist contest-id otherwise it will be considered a codeforces contest id. + To know clist contest_id visit https://clist.by. """ + wait_msg = await ctx.channel.send('Generating ranklist, please wait...') + resource = 'codeforces.com' + for pattern in _PATTERNS: + if pattern in contest_id: + resource = _PATTERNS[pattern] + break + if resource=='codeforces.com': + try: + contest_id = int(contest_id) + if contest_id<0: + contest_id = -1*contest_id + resource = 'clist.by' + except: + raise ContestCogError('Invalid contest_id provided.') + if resource!='codeforces.com': + contest = await self.resolve_contest(contest_id=contest_id, resource=resource) + if contest is None: + raise ContestCogError('Contest not found.') + contest_id = contest['id'] + account_ids= await cf_common.resolve_handles(ctx, self.member_converter, handles, maxcnt=None, default_to_all_server=True, resource=contest['resource']) + standings_to_show = [] + standings = await clist.statistics(contest_id=contest_id, account_ids=account_ids) + for standing in standings: + if not standing['place'] or not standing['handle']: + continue + standings_to_show.append(standing) + standings_to_show.sort(key=lambda standing: int(standing['place'])) + pages = self._make_clist_standings_pages(standings_to_show) + await wait_msg.delete() + await ctx.channel.send(embed=self._make_contest_embed_for_cranklist(contest)) + return paginator.paginate(self.bot, ctx.channel, pages, wait_time=_STANDINGS_PAGINATE_WAIT_TIME) handles = await cf_common.resolve_handles(ctx, self.member_converter, handles, maxcnt=None, default_to_all_server=True) contest = cf_common.cache2.contest_cache.get_contest(contest_id) - wait_msg = await ctx.channel.send('Generating ranklist, please wait...') ranklist = None try: ranklist = cf_common.cache2.ranklist_cache.get_ranklist(contest) diff --git a/tle/util/clist_api.py b/tle/util/clist_api.py index 32dca604..a010d4b6 100644 --- a/tle/util/clist_api.py +++ b/tle/util/clist_api.py @@ -102,4 +102,74 @@ async def account(handle, resource): resp = resp['objects'] if len(resp)==0: raise HandleNotFoundError(handle=handle, resource=resource) + return resp + +async def fetch_user_info(resource, account_ids=None, handles=None): + params = {'resource':resource, 'limit':1000} + if account_ids!=None: + ids = "" + for i in range(len(account_ids)): + ids += str(account_ids[i]) + if i!=(len(account_ids)-1): + ids += ',' + params['id__in']=ids + if handles!=None: + regex = '$|^'.join(handles) + params['handle__regex'] = '^'+regex+'$' + resp = await _query_clist_api('account', params) + if resp==None or 'objects' not in resp: + raise ClientError + else: + resp = resp['objects'] + return resp + +async def statistics(account_id=None, contest_id=None, order_by=None, account_ids=None, resource=None): + params = {'limit':1000} + if account_id!=None: params['account_id'] = account_id + if contest_id!=None: params['contest_id'] = contest_id + if order_by!=None: params['order_by'] = order_by + if account_ids!=None: + ids = "" + for i in range(len(account_ids)): + ids += str(account_ids[i]) + if i!=(len(account_ids)-1): + ids += ',' + params['account_id__in']=ids + if resource!=None: params['resource'] = resource + results = [] + offset = 0 + while True: + params['offset'] = offset + resp = await _query_clist_api('statistics', params) + if resp==None or 'objects' not in resp: + if offset==0: + raise ClientError + else: + break + else: + objects = resp['objects'] + results += objects + if(len(objects)<1000): + break + offset+=1000 + return results + +async def contest(contest_id): + resp = await _query_clist_api('contest/'+str(contest_id), None) + return resp + +async def search_contest(regex=None, date_limits=None, resource=None): + params = {'limit':1000} + if resource!=None: + params['resource'] = resource + if regex!=None: + params['event__regex'] = regex + if date_limits!=None: + params['start__gte'] = date_limits[0] + params['start__lt'] = date_limits[1] + resp = await _query_clist_api('contest', data=params) + if resp==None or 'objects' not in resp: + raise ClientError + else: + resp = resp['objects'] return resp \ No newline at end of file diff --git a/tle/util/codeforces_common.py b/tle/util/codeforces_common.py index 0aaf6620..c6cd91d4 100644 --- a/tle/util/codeforces_common.py +++ b/tle/util/codeforces_common.py @@ -12,6 +12,7 @@ from tle import constants from tle.util import cache_system2 from tle.util import codeforces_api as cf +from tle.util import clist_api as clist from tle.util import db from tle.util import events @@ -213,20 +214,26 @@ def days_ago(t): return 'yesterday' return f'{math.floor(days)} days ago' -async def resolve_handles(ctx, converter, handles, *, mincnt=1, maxcnt=5, default_to_all_server=False): +async def resolve_handles(ctx, converter, handles, *, mincnt=1, maxcnt=5, default_to_all_server=False, resource='codeforces.com'): """Convert an iterable of strings to CF handles. A string beginning with ! indicates Discord username, otherwise it is a raw CF handle to be left unchanged.""" handles = set(handles) if default_to_all_server and not handles: handles.add('+server') + account_ids = set() if '+server' in handles: handles.remove('+server') - guild_handles = {handle for discord_id, handle - in user_db.get_handles_for_guild(ctx.guild.id)} - handles.update(guild_handles) - if len(handles) < mincnt or (maxcnt and maxcnt < len(handles)): + if resource=='codeforces.com': + guild_handles = {handle for discord_id, handle + in user_db.get_handles_for_guild(ctx.guild.id)} + handles.update(guild_handles) + else: + guild_account_ids = {account_id for discord_id, account_id in + user_db.get_account_ids_for_guild(ctx.guild.id, resource=resource)} + account_ids.update(guild_account_ids) + if len(account_ids)==0 and len(handles) < mincnt or (maxcnt and maxcnt < len(handles)): raise HandleCountOutOfBoundsError(mincnt, maxcnt) - resolved_handles = [] + resolved_handles = set() for handle in handles: if handle.startswith('!'): # ! denotes Discord user @@ -235,13 +242,37 @@ async def resolve_handles(ctx, converter, handles, *, mincnt=1, maxcnt=5, defaul member = await converter.convert(ctx, member_identifier) except commands.errors.CommandError: raise FindMemberFailedError(member_identifier) - handle = user_db.get_handle(member.id, ctx.guild.id) - if handle is None: - raise HandleNotRegisteredError(member) + if resource=='codeforces.com': + handle = user_db.get_handle(member.id, ctx.guild.id) + if handle is None: + raise HandleNotRegisteredError(member) + resolved_handles.add(handle) + else: + account_id = user_db.get_account_id(member.id, ctx.guild.id, resource=resource) + if account_id is None: + raise HandleNotRegisteredError(member, resource=resource) + else: + account_ids.add(account_id) + else: + if resource=='codeforces.com': + resolved_handles.add(handle) + else: + account_id = user_db.get_account_id_from_handle(handle=handle, resource=resource) + if account_id: + account_ids.add(account_id) + else: + resolved_handles.add(handle) if handle in HandleIsVjudgeError.HANDLES: raise HandleIsVjudgeError(handle) - resolved_handles.append(handle) - return resolved_handles + if resource=='codeforces.com': + return list(resolved_handles) + else: + if len(resolved_handles)!=0: + clist_users = await clist.fetch_user_info(resource=resource, handles=list(resolved_handles)) + if clist_users!=None: + for user in clist_users: + account_ids.add(int(user['id'])) + return list(account_ids) def members_to_handles(members: [discord.Member], guild_id): handles = [] diff --git a/tle/util/db/user_db_conn.py b/tle/util/db/user_db_conn.py index c6cb3575..4aa11337 100644 --- a/tle/util/db/user_db_conn.py +++ b/tle/util/db/user_db_conn.py @@ -485,6 +485,13 @@ def fetch_clist_user(self, account_id): account_id, resource, handle, name, rating, contests = user return {'id':account_id, 'resource':resource, 'handle':handle, 'name':name, 'rating':rating, 'n_contests':contests} + + def get_account_id_from_handle(self, handle, resource): + query = ('SELECT account_id ' + 'FROM clist_user_cache ' + 'WHERE handle = ? AND resource = ?') + res = self.conn.execute(query, (handle, resource,)).fetchone() + return res[0] if res else None def get_user_id(self, handle, guild_id): query = ('SELECT user_id ' From 8d3b159ff7def811a7fd19cb2db83c1d37f56706 Mon Sep 17 00:00:00 2001 From: Paramjeet Date: Thu, 22 Jul 2021 10:42:07 +0530 Subject: [PATCH 05/11] Minor fixes --- poetry.lock | 68 +++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + tle/cogs/contests.py | 2 +- tle/cogs/handles.py | 5 ++-- 4 files changed, 72 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7e572499..23a582cf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -75,6 +75,14 @@ soupsieve = {version = ">1.2", markers = "python_version >= \"3.0\""} html5lib = ["html5lib"] lxml = ["lxml"] +[[package]] +name = "certifi" +version = "2021.5.30" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "chardet" version = "3.0.4" @@ -83,6 +91,17 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "charset-normalizer" +version = "2.0.3" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + [[package]] name = "colorama" version = "0.4.4" @@ -310,6 +329,24 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "requests" +version = "2.26.0" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] + [[package]] name = "scipy" version = "1.5.4" @@ -359,6 +396,19 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "urllib3" +version = "1.26.6" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.extras] +brotli = ["brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "yarl" version = "1.5.1" @@ -387,7 +437,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "c1e423d090ffc4df8054161514ace92d6353adf750911797fea07f84f689176d" +content-hash = "b189f22c23d63bef13c59c7046ea1be17d78bf6c6caca4c80000391965a8bfcb" [metadata.files] aiocache = [ @@ -426,10 +476,18 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.9.3-py3-none-any.whl", hash = "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"}, {file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"}, ] +certifi = [ + {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, + {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, +] chardet = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, ] +charset-normalizer = [ + {file = "charset-normalizer-2.0.3.tar.gz", hash = "sha256:c46c3ace2d744cfbdebceaa3c19ae691f53ae621b39fd7570f59d14fb7f2fd12"}, + {file = "charset_normalizer-2.0.3-py3-none-any.whl", hash = "sha256:88fce3fa5b1a84fdcb3f603d889f723d1dd89b26059d0123ca435570e848d5e1"}, +] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, @@ -717,6 +775,10 @@ pytz = [ {file = "pytz-2020.4-py2.py3-none-any.whl", hash = "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"}, {file = "pytz-2020.4.tar.gz", hash = "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268"}, ] +requests = [ + {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, + {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, +] scipy = [ {file = "scipy-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4f12d13ffbc16e988fa40809cbbd7a8b45bc05ff6ea0ba8e3e41f6f4db3a9e47"}, {file = "scipy-1.5.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a254b98dbcc744c723a838c03b74a8a34c0558c9ac5c86d5561703362231107d"}, @@ -761,6 +823,10 @@ typing-extensions = [ {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, ] +urllib3 = [ + {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, + {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, +] yarl = [ {file = "yarl-1.5.1-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:db6db0f45d2c63ddb1a9d18d1b9b22f308e52c83638c26b422d520a815c4b3fb"}, {file = "yarl-1.5.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:17668ec6722b1b7a3a05cc0167659f6c95b436d25a36c2d52db0eca7d3f72593"}, diff --git a/pyproject.toml b/pyproject.toml index 985a3ee5..de51e059 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ pycairo = "^1.19.1" PyGObject = "^3.34.0" aiocache = "^0.11.1" beautifulsoup4 = "^4.9.3" +requests = "^2.26.0" [tool.poetry.dev-dependencies] pytest = "^3.0" diff --git a/tle/cogs/contests.py b/tle/cogs/contests.py index e2a64564..7d81a05f 100644 --- a/tle/cogs/contests.py +++ b/tle/cogs/contests.py @@ -593,7 +593,7 @@ async def resolve_contest(self, contest_id, resource): return contest @commands.command(brief='Show ranklist for given handles and/or server members') - async def ranklist(self, ctx, contest_id: int, *handles: str): + async def ranklist(self, ctx, contest_id: str, *handles: str): """Shows ranklist for the contest with given contest id. If handles contains '+server', all server members are included. No handles defaults to '+server'. diff --git a/tle/cogs/handles.py b/tle/cogs/handles.py index b6f38b41..ec8eb308 100644 --- a/tle/cogs/handles.py +++ b/tle/cogs/handles.py @@ -370,8 +370,9 @@ async def set(self, ctx, member: discord.Member, handle: str): resource = None if resource!=None and resource not in _SUPPORTED_CLIST_RESOURCES: raise HandleCogError(f'The resource `{resource}` is not supported.') + member_username = str(member) users = await clist.account(handle=handle, resource=resource) - message = f'Following handles for `{member.mention}` have been linked\n' + message = f'Following handles for `{member_username}` have been linked\n' for user in users: if user['resource'] not in _SUPPORTED_CLIST_RESOURCES: continue @@ -440,7 +441,7 @@ async def identify(self, ctx, handle: str): if resource=='all': return await ctx.send(f'Sorry `{invoker}`, all keyword can only be used with set command') if resource=='codingcompetitions.withgoogle.com': - return await ctx.send(f'Sorry `{invoker}`, you can\'t identify handles of codingcompetitions.withgoogle.com, please ask a moderator to link your account.') e + return await ctx.send(f'Sorry `{invoker}`, you can\'t identify handles of codingcompetitions.withgoogle.com, please ask a moderator to link your account.') if resource not in _SUPPORTED_CLIST_RESOURCES: raise HandleCogError(f'The resource `{resource}` is not supported.') wait_msg = await ctx.channel.send('Fetching account details, please wait...') From 8d786011132c19d980cac264777adf870c633e4a Mon Sep 17 00:00:00 2001 From: Paramjeet Date: Thu, 22 Jul 2021 11:15:28 +0530 Subject: [PATCH 06/11] Implemented handle list for clist resources --- tle/cogs/handles.py | 48 +++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/tle/cogs/handles.py b/tle/cogs/handles.py index ec8eb308..9bed18b1 100644 --- a/tle/cogs/handles.py +++ b/tle/cogs/handles.py @@ -233,23 +233,25 @@ def _make_profile_embed(member, user, *, mode): return embed -def _make_pages(users, title): +def _make_pages(users, title, resource='codeforces.com'): chunks = paginator.chunkify(users, _HANDLES_PER_PAGE) pages = [] done = 0 - + no_rating = resource=='codingcompetitions.withgoogle.com' + no_rating_suffix = resource!='codeforces.com' style = table.Style('{:>} {:<} {:<} {:<}') for chunk in chunks: t = table.Table(style) - t += table.Header('#', 'Name', 'Handle', 'Rating') + t += table.Header('#', 'Name', 'Handle', 'Contests' if no_rating else 'Rating') t += table.Line() - for i, (member, handle, rating) in enumerate(chunk): - name = member.display_name + for i, (member, handle, rating, n_contests) in enumerate(chunk): + name = member.display_name if member else "" if len(name) > _NAME_MAX_LEN: name = name[:_NAME_MAX_LEN - 1] + '…' rank = cf.rating2rank(rating) rating_str = 'N/A' if rating is None else str(rating) - t += table.Data(i + done, name, handle, f'{rating_str} ({rank.title_abbr})') + fourth = n_contests if no_rating else ((f'{rating_str}')+((f'({rank.title_abbr})') if not no_rating_suffix else '')) + t += table.Data(i + done, name, handle, fourth) table_str = '```\n'+str(t)+'\n```' embed = discord_common.cf_color_embed(description=table_str) pages.append((title, embed)) @@ -613,19 +615,39 @@ async def list(self, ctx, *countries): if you wish to display only members from those countries. Country data is sourced from codeforces profiles. e.g. ;handle list Croatia Slovenia """ - countries = [country.title() for country in countries] - res = cf_common.user_db.get_cf_users_for_guild(ctx.guild.id) - users = [(ctx.guild.get_member(user_id), cf_user.handle, cf_user.rating) - for user_id, cf_user in res if not countries or cf_user.country in countries] - users = [(member, handle, rating) for member, handle, rating in users if member is not None] + resource = 'codeforces.com' + if len(countries)==1: + country = countries[0] + if country in _CLIST_RESOURCE_SHORT_FORMS: + resource = _CLIST_RESOURCE_SHORT_FORMS[country] + countries = [] + elif country in _SUPPORTED_CLIST_RESOURCES: + resource = country + countries = [] + if resource!='codeforces.com': + clist_users = cf_common.user_db.get_clist_users_for_guild(ctx.guild.id, resource=resource) + users = [] + for user in clist_users: + handle = user['handle'] + rating = int(user['rating']) if user['rating']!=None else None + member = ctx.guild.get_member(int(user['user_id'])) + n_contests = user['n_contests'] + users.append((member, handle, rating, n_contests)) + else: + countries = [country.title() for country in countries] + res = cf_common.user_db.get_cf_users_for_guild(ctx.guild.id) + users = [(ctx.guild.get_member(user_id), cf_user.handle, cf_user.rating) + for user_id, cf_user in res if not countries or cf_user.country in countries] + users = [(member, handle, rating, 0) for member, handle, rating in users if member is not None] if not users: raise HandleCogError('No members with registered handles.') users.sort(key=lambda x: (1 if x[2] is None else -x[2], x[1])) # Sorting by (-rating, handle) - title = 'Handles of server members' + title = 'Handles of server members ('+str(resource)+')' + if countries: title += ' from ' + ', '.join(f'`{country}`' for country in countries) - pages = _make_pages(users, title) + pages = _make_pages(users, title, resource) paginator.paginate(self.bot, ctx.channel, pages, wait_time=_PAGINATE_WAIT_TIME, set_pagenum_footers=True) From 29e1c28ef28a37fc1dfe031e91526fd1769ae46b Mon Sep 17 00:00:00 2001 From: Paramjeet Date: Thu, 22 Jul 2021 11:26:20 +0530 Subject: [PATCH 07/11] Updated readme (Provided information about clist environment variable) --- README.md | 2 ++ environment.template | 1 + 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 414428ac..6f685ca1 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ The features of the bot are split into a number of cogs, each handling their own - **CacheControl** Commands related to data caching. ## Installation + > If you want to run the bot inside a docker container follow these [instructions](/Docker.md) Clone the repository @@ -84,6 +85,7 @@ Fill in appropriate variables in new "environment" file. - **ALLOW_DUEL_SELF_REGISTER**: boolean value indicating if self registration for duels is enabled. - **TLE_ADMIN**: the name of the role that can run admin commands of the bot. If this is not set, the role name will default to "Admin". - **TLE_MODERATOR**: the name of the role that can run moderator commands of the bot. If this is not set, the role name will default to "Moderator". +- **CLIST_API_TOKEN**: Credential for accessing clist api, You can find your api key [here][https://clist.by/api/v2/doc/] after creating an account on clist.by. If this is not set, codechef/atcoder/google(kickstart) related commands won't work. To start TLE just run: diff --git a/environment.template b/environment.template index 86f46cf1..09ccb85d 100644 --- a/environment.template +++ b/environment.template @@ -1,3 +1,4 @@ export BOT_TOKEN="XXXXXXXXXXXXXXXXXXXXXXXX.XXXXXX.XXXXXXXXXXXXXXXXXXXXXXXXXXX" export LOGGING_COG_CHANNEL_ID="XXXXXXXXXXXXXXXXXX" export ALLOW_DUEL_SELF_REGISTER="false" +export CLIST_API_TOKEN="username=xxxxxxxx&api_key=xxxxxxxxxxxxxxxxxxxxxxxxxx" From efdccb75bbbb6302bd4b0ee0daca84860f7f99cd Mon Sep 17 00:00:00 2001 From: Paramjeet Date: Thu, 22 Jul 2021 11:46:58 +0530 Subject: [PATCH 08/11] Created task for refreshing clist user cache every 2 hour --- tle/cogs/handles.py | 11 ++++++++++- tle/util/db/user_db_conn.py | 6 ++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tle/cogs/handles.py b/tle/cogs/handles.py index 9bed18b1..51a00ef3 100644 --- a/tle/cogs/handles.py +++ b/tle/cogs/handles.py @@ -40,7 +40,7 @@ _TOP_DELTAS_COUNT = 10 _MAX_RATING_CHANGES_PER_EMBED = 15 _UPDATE_HANDLE_STATUS_INTERVAL = 6 * 60 * 60 # 6 hours - +_UPDATE_CLIST_CACHE_INTERVAL = 2 * 60 * 60 # 2 hours class HandleCogError(commands.CommandError): pass @@ -325,6 +325,15 @@ async def update_for_guild(guild): return_exceptions=True) self.logger.info(f'All guilds updated for contest {contest.id}.') + @tasks.task_spec(name='RefreshClistUserCache', + waiter=tasks.Waiter.fixed_delay(_UPDATE_CLIST_CACHE_INTERVAL)) + async def _update_clist_users_cache(self, _): + account_ids = cf_common.user_db.get_all_account_ids() + clist_users = clist.fetch_user_info(resource=None, account_ids=account_ids) + if clist_users: + for user in clist_users: + cf_common.user_db.cache_clist_user(user) + @commands.group(brief='Commands that have to do with handles', invoke_without_command=True) async def handle(self, ctx): """Change or collect information about specific handles on Codeforces""" diff --git a/tle/util/db/user_db_conn.py b/tle/util/db/user_db_conn.py index 4aa11337..02c22687 100644 --- a/tle/util/db/user_db_conn.py +++ b/tle/util/db/user_db_conn.py @@ -493,6 +493,12 @@ def get_account_id_from_handle(self, handle, resource): res = self.conn.execute(query, (handle, resource,)).fetchone() return res[0] if res else None + def get_all_account_ids(self): + query = ('SELECT account_id ' + 'FROM clist_user_cache ') + res = self.conn.execute(query,()).fetchall() + return [int(account_id) for account_id, in res] + def get_user_id(self, handle, guild_id): query = ('SELECT user_id ' 'FROM user_handle ' From 4cb734cdcb1d9a58de6d197463b70675e7ad8bf4 Mon Sep 17 00:00:00 2001 From: Paramjeet Date: Thu, 22 Jul 2021 12:02:17 +0530 Subject: [PATCH 09/11] Added clist error messages to discord_common --- tle/cogs/handles.py | 3 ++- tle/util/discord_common.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tle/cogs/handles.py b/tle/cogs/handles.py index 51a00ef3..3c95e8cf 100644 --- a/tle/cogs/handles.py +++ b/tle/cogs/handles.py @@ -270,6 +270,7 @@ def __init__(self, bot): async def on_ready(self): cf_common.event_sys.add_listener(self._on_rating_changes) self._set_ex_users_inactive_task.start() + self._update_clist_users_cache.start() @commands.Cog.listener() async def on_member_remove(self, member): @@ -329,7 +330,7 @@ async def update_for_guild(guild): waiter=tasks.Waiter.fixed_delay(_UPDATE_CLIST_CACHE_INTERVAL)) async def _update_clist_users_cache(self, _): account_ids = cf_common.user_db.get_all_account_ids() - clist_users = clist.fetch_user_info(resource=None, account_ids=account_ids) + clist_users = await clist.fetch_user_info(resource=None, account_ids=account_ids) if clist_users: for user in clist_users: cf_common.user_db.cache_clist_user(user) diff --git a/tle/util/discord_common.py b/tle/util/discord_common.py index b296b01f..b9d209e8 100644 --- a/tle/util/discord_common.py +++ b/tle/util/discord_common.py @@ -7,6 +7,7 @@ from discord.ext import commands from tle.util import codeforces_api as cf +from tle.util import clist_api as clist from tle.util import db from tle.util import tasks @@ -81,6 +82,8 @@ async def bot_error_handler(ctx, exception): await ctx.send(embed=embed_alert('Sorry, this command is temporarily disabled')) elif isinstance(exception, (cf.CodeforcesApiError, commands.UserInputError)): await ctx.send(embed=embed_alert(exception)) + elif isinstance(exception, (clist.ClistNotConfiguredError, clist.ClistApiError)): + await ctx.send(embed=embed_alert(exception)) else: msg = 'Ignoring exception in command {}:'.format(ctx.command) exc_info = type(exception), exception, exception.__traceback__ From 90ad47dd50734b2f781dccc3b0c04f20b371884e Mon Sep 17 00:00:00 2001 From: Paramjeet Date: Sun, 1 Aug 2021 19:55:30 +0530 Subject: [PATCH 10/11] Restructured clist code to clist_common --- tle/cogs/contests.py | 220 ++------------------------ tle/cogs/handles.py | 78 +++------ tle/util/clist_api.py | 114 ++++++++++++-- tle/util/clist_common.py | 287 ++++++++++++++++++++++++++++++++++ tle/util/codeforces_api.py | 4 + tle/util/codeforces_common.py | 53 ++----- tle/util/db/user_db_conn.py | 12 +- tle/util/ranklist/ranklist.py | 88 ++++++++++- tle/util/scaper.py | 2 +- 9 files changed, 530 insertions(+), 328 deletions(-) create mode 100644 tle/util/clist_common.py diff --git a/tle/cogs/contests.py b/tle/cogs/contests.py index 7d81a05f..7c5a4691 100644 --- a/tle/cogs/contests.py +++ b/tle/cogs/contests.py @@ -14,7 +14,7 @@ from tle.util import codeforces_common as cf_common from tle.util import cache_system2 from tle.util import codeforces_api as cf -from tle.util import clist_api as clist +from tle.util import clist_common as clist_common from tle.util import db from tle.util import discord_common from tle.util import events @@ -33,32 +33,6 @@ _RATED_VC_EXTRA_TIME = 10 * 60 # seconds _MIN_RATED_CONTESTANTS_FOR_RATED_VC = 50 -_PATTERNS = { - 'abc': 'atcoder.jp', - 'arc': 'atcoder.jp', - 'agc': 'atcoder.jp', - 'kickstart': 'codingcompetitions.withgoogle.com', - 'codejam': 'codingcompetitions.withgoogle.com', - 'lunchtime': 'codechef.com', - 'long': 'codechef.com', - 'cookoff': 'codechef.com', - 'starters': 'codechef.com' -} - -def parse_date(arg): - try: - if len(arg) == 8: - fmt = '%d%m%Y' - elif len(arg) == 6: - fmt = '%m%Y' - elif len(arg) == 4: - fmt = '%Y' - else: - raise ValueError - return dt.datetime.strptime(arg, fmt) - except ValueError: - raise ContestCogError(f'{arg} is an invalid date argument') - class ContestCogError(commands.CommandError): pass @@ -401,7 +375,7 @@ def _make_standings_pages(self, contest, problem_indices, handle_standings, delt num_chunks = len(handle_standings_chunks) delta_chunks = paginator.chunkify(deltas, _STANDINGS_PER_PAGE) if deltas else [None] * num_chunks - if contest.type == 'CF': + if contest.type == 'CF' or contest.type == 'CLIST': get_table = functools.partial(self._get_cf_or_ioi_standings_table, mode='cf') elif contest.type == 'ICPC': get_table = self._get_icpc_standings_table @@ -453,57 +427,6 @@ def _make_contest_embed_for_ranklist(ranklist): msg = f'{start}{en}|{en}{duration}{en}|{en}Ended {since} ago' embed.add_field(name='When', value=msg, inline=False) return embed - - def _make_clist_standings_pages(self, standings): - if standings is None or len(standings)==0: - return "```No handles found inside ranklist```" - show_rating_changes = standings[0]['rating_change']!=None - - pages = [] - standings_chunks = paginator.chunkify(standings, _STANDINGS_PER_PAGE) - num_chunks = len(standings_chunks) - - if not show_rating_changes: - header_style = '{:>} {:<} {:^} ' - body_style = '{:>} {:<} {:<} ' - header = ['#', 'Name', 'Score'] - else: - header_style = '{:>} {:<} {:^} {:<} {:<} ' - body_style = '{:>} {:<} {:<} {:<} {:<} ' - header = ['#', 'Name', 'Score', 'Delta', 'New Rating'] - - num_pages = 1 - for standings_chunk in standings_chunks: - body = [] - for standing in standings_chunk: - score = int(standing['score']) if standing['score'] else 0 - if show_rating_changes: - delta = int(standing['rating_change']) if standing['rating_change'] else ' ' - if delta!=' ': - delta = '+'+str(delta) if delta>0 else str(delta) - tokens = [int(standing['place']), standing['handle'], score, delta, standing['new_rating']] - else: - tokens = [int(standing['place']), standing['handle'], score] - body.append(tokens) - t = table.Table(table.Style(header=header_style, body=body_style)) - t += table.Header(*header) - t += table.Line('\N{EM DASH}') - for row in body: - t += table.Data(*row) - t += table.Line('\N{EM DASH}') - page_num_footer = f' # Page: {num_pages} / {num_chunks}' if num_chunks > 1 else '' - - # We use yaml to get nice colors in the ranklist. - content = f'```yaml\n{t}\n{page_num_footer}```' - pages.append((content, None)) - num_pages += 1 - return pages - - @staticmethod - def _make_contest_embed_for_cranklist(contest): - embed = discord_common.cf_color_embed(title=contest['event'], url=contest['href']) - embed.add_field(name='Website', value=contest['resource']) - return embed @staticmethod def _make_contest_embed_for_vc_ranklist(ranklist, vc_start_time=None, vc_end_time=None): @@ -519,79 +442,6 @@ def _make_contest_embed_for_vc_ranklist(ranklist, vc_start_time=None, vc_end_tim embed.add_field(name='Tick tock', value=msg, inline=False) return embed - async def resolve_contest(self, contest_id, resource): - contest = None - if resource=='clist.by': - contest = await clist.contest(contest_id) - elif resource=='atcoder.jp': - prefix = contest_id[:3] - if prefix=='abc': - prefix = 'AtCoder Beginner Contest ' - if prefix=='arc': - prefix = 'AtCoder Regular Contest ' - if prefix=='agc': - prefix = 'AtCoder Grand Contest ' - suffix = contest_id[3:] - try: - suffix = int(suffix) - except: - raise ContestCogError('Invalid contest_id provided.') - contest_name = prefix+str(suffix) - contests = await clist.search_contest(regex=contest_name, resource=resource) - if contests==None or len(contests)==0: - raise ContestCogError('Contest not found.') - contest = contests[0] - elif resource=='codechef.com': - contest_name = None - if 'lunchtime' in contest_id: - date = parse_date(contest_id[9:]) - contest_name = str(date.strftime('%B'))+' Lunchtime '+str(date.strftime('%Y')) - elif 'cookoff' in contest_id: - date = parse_date(contest_id[7:]) - contest_name = str(date.strftime('%B'))+' Cook-Off '+str(date.strftime('%Y')) - elif 'long' in contest_id: - date = parse_date(contest_id[4:]) - contest_name = str(date.strftime('%B'))+' Challenge '+str(date.strftime('%Y')) - elif 'starters' in contest_id: - date = parse_date(contest_id[8:]) - contest_name = str(date.strftime('%B'))+' CodeChef Starters '+str(date.strftime('%Y')) - contests = await clist.search_contest(regex=contest_name, resource=resource) - if contests==None or len(contests)==0: - raise ContestCogError('Contest not found.') - contest = contests[0] - elif resource=='codingcompetitions.withgoogle.com': - year,round = None,None - contest_name = None - if 'kickstart' in contest_id: - year = contest_id[9:11] - round = contest_id[11:] - contest_name = 'Kick Start.*Round '+round - elif 'codejam' in contest_id: - year = contest_id[7:9] - round = contest_id[9:] - if round=='WF': - round = 'Finals' - contest_name = 'Code Jam.*Finals' - elif round=='QR': - round = 'Qualification Round' - contest_name = 'Code Jam.*Qualification Round' - else: - contest_name = 'Code Jam.*Round '+round - if not round: - raise ContestCogError('Invalid contest_id provided.') - try: - year = int(year) - except: - raise ContestCogError('Invalid contest_id provided.') - start = dt.datetime(int('20'+str(year)), 1, 1) - end = dt.datetime(int('20'+str(year+1)), 1, 1) - date_limit = (start.strftime('%Y-%m-%dT%H:%M:%S'), end.strftime('%Y-%m-%dT%H:%M:%S')) - contests = await clist.search_contest(regex=contest_name, resource=resource, date_limits=date_limit) - if contests==None or len(contests)==0: - raise ContestCogError('Contest not found.') - contest = contests[0] - return contest - @commands.command(brief='Show ranklist for given handles and/or server members') async def ranklist(self, ctx, contest_id: str, *handles: str): """Shows ranklist for the contest with given contest id. If handles contains @@ -624,69 +474,19 @@ async def ranklist(self, ctx, contest_id: str, *handles: str): To know clist contest_id visit https://clist.by. """ wait_msg = await ctx.channel.send('Generating ranklist, please wait...') - resource = 'codeforces.com' - for pattern in _PATTERNS: - if pattern in contest_id: - resource = _PATTERNS[pattern] - break - if resource=='codeforces.com': - try: - contest_id = int(contest_id) - if contest_id<0: - contest_id = -1*contest_id - resource = 'clist.by' - except: - raise ContestCogError('Invalid contest_id provided.') - if resource!='codeforces.com': - contest = await self.resolve_contest(contest_id=contest_id, resource=resource) - if contest is None: - raise ContestCogError('Contest not found.') - contest_id = contest['id'] - account_ids= await cf_common.resolve_handles(ctx, self.member_converter, handles, maxcnt=None, default_to_all_server=True, resource=contest['resource']) - standings_to_show = [] - standings = await clist.statistics(contest_id=contest_id, account_ids=account_ids) - for standing in standings: - if not standing['place'] or not standing['handle']: - continue - standings_to_show.append(standing) - standings_to_show.sort(key=lambda standing: int(standing['place'])) - pages = self._make_clist_standings_pages(standings_to_show) - await wait_msg.delete() - await ctx.channel.send(embed=self._make_contest_embed_for_cranklist(contest)) - return paginator.paginate(self.bot, ctx.channel, pages, wait_time=_STANDINGS_PAGINATE_WAIT_TIME) - handles = await cf_common.resolve_handles(ctx, self.member_converter, handles, maxcnt=None, default_to_all_server=True) - contest = cf_common.cache2.contest_cache.get_contest(contest_id) - ranklist = None - try: - ranklist = cf_common.cache2.ranklist_cache.get_ranklist(contest) - except cache_system2.RanklistNotMonitored: - if contest.phase == 'BEFORE': - raise ContestCogError(f'Contest `{contest.id} | {contest.name}` has not started') - ranklist = await cf_common.cache2.ranklist_cache.generate_ranklist(contest.id, - fetch_changes=True) + contest = await clist_common.get_contest(contest_id) + handles = await clist_common.resolve_handles(ctx, self.member_converter, handles, maxcnt=None, default_to_all_server=True, resource=contest.resource) + ranklist = await clist_common.get_ranklist(contest, handles) await wait_msg.delete() await ctx.channel.send(embed=self._make_contest_embed_for_ranklist(ranklist)) - await self._show_ranklist(channel=ctx.channel, contest_id=contest_id, handles=handles, ranklist=ranklist) + await self._show_ranklist(channel=ctx.channel, contest_id=contest_id, handles=handles, ranklist=ranklist, contest=contest) - async def _show_ranklist(self, channel, contest_id: int, handles: [str], ranklist, vc: bool = False, delete_after: float = None): - contest = cf_common.cache2.contest_cache.get_contest(contest_id) + async def _show_ranklist(self, channel, contest_id: int, handles, ranklist, vc: bool = False, delete_after: float = None, contest=None): + contest = contest or cf_common.cache2.contest_cache.get_contest(contest_id) if ranklist is None: raise ContestCogError('No ranklist to show') - handle_standings = [] - for handle in handles: - try: - standing = ranklist.get_standing_row(handle) - except rl.HandleNotPresentError: - continue - - # Database has correct handle ignoring case, update to it - # TODO: It will throw an exception if this row corresponds to a team. At present ranklist doesnt show teams. - # It should be fixed in https://github.com/cheran-senthil/TLE/issues/72 - handle = standing.party.members[0].handle - if vc and standing.party.participantType != 'VIRTUAL': - continue - handle_standings.append((handle, standing)) + handle_standings = ranklist.get_handle_standings(handles, vc=vc) if not handle_standings: error = f'None of the handles are present in the ranklist of `{contest.name}`' @@ -700,7 +500,7 @@ async def _show_ranklist(self, channel, contest_id: int, handles: [str], ranklis if ranklist.is_rated: deltas = [ranklist.get_delta(handle) for handle, standing in handle_standings] - problem_indices = [problem.index for problem in ranklist.problems] + problem_indices = ranklist.get_problem_indexes() pages = self._make_standings_pages(contest, problem_indices, handle_standings, deltas) paginator.paginate(self.bot, channel, pages, wait_time=_STANDINGS_PAGINATE_WAIT_TIME, delete_after=delete_after) diff --git a/tle/cogs/handles.py b/tle/cogs/handles.py index 3c95e8cf..f7feb350 100644 --- a/tle/cogs/handles.py +++ b/tle/cogs/handles.py @@ -21,7 +21,8 @@ from tle.util import codeforces_api as cf from tle.util import codeforces_common as cf_common from tle.util import clist_api as clist -from tle.util.clist_api import _CLIST_RESOURCE_SHORT_FORMS, _SUPPORTED_CLIST_RESOURCES +from tle.util import clist_common as clist_common +from tle.util.clist_common import _SUPPORTED_RESOURCES, Resources from tle.util import scaper from tle.util import discord_common from tle.util import events @@ -233,12 +234,12 @@ def _make_profile_embed(member, user, *, mode): return embed -def _make_pages(users, title, resource='codeforces.com'): +def _make_pages(users, title, resource=Resources.CODEFORCES): chunks = paginator.chunkify(users, _HANDLES_PER_PAGE) pages = [] done = 0 - no_rating = resource=='codingcompetitions.withgoogle.com' - no_rating_suffix = resource!='codeforces.com' + no_rating = resource==Resources.GOOGLE + no_rating_suffix = resource!=Resources.CODEFORCES style = table.Style('{:>} {:<} {:<} {:<}') for chunk in chunks: t = table.Table(style) @@ -371,26 +372,17 @@ async def set(self, ctx, member: discord.Member, handle: str): ;handle set @Alex ac:Um_nik ;handle set @Priyansh google:Priyansh31dec """ - resource = 'codeforces.com' - if ':' in handle: - resource = handle[0: handle.index(':')] - handle = handle[handle.index(':')+1:] - if resource in _CLIST_RESOURCE_SHORT_FORMS: - resource = _CLIST_RESOURCE_SHORT_FORMS[resource] - if resource!='codeforces.com': - if resource=='all': - resource = None - if resource!=None and resource not in _SUPPORTED_CLIST_RESOURCES: - raise HandleCogError(f'The resource `{resource}` is not supported.') + resource, handle = clist_common.resource_from_handle_notation(handle) + if resource!=Resources.CODEFORCES: member_username = str(member) users = await clist.account(handle=handle, resource=resource) - message = f'Following handles for `{member_username}` have been linked\n' + embed = discord.Embed(description=f'Following handles for `{member_username}` have been linked') for user in users: - if user['resource'] not in _SUPPORTED_CLIST_RESOURCES: + if user.resource not in _SUPPORTED_RESOURCES: continue - message += user['resource']+' : '+user['handle']+'\n' + embed.add_field(name=user.resource, value=user.handle, inline=True) await self._set_account_id(member.id, ctx.guild, user) - return await ctx.send(message) + return await ctx.send(embed=embed) # CF API returns correct handle ignoring case, update to it user, = await cf.user.info(handles=[handle]) await self._set(ctx, member, user) @@ -400,7 +392,7 @@ async def set(self, ctx, member: discord.Member, handle: str): async def _set_account_id(self, member_id, guild, user): try: guild_id = guild.id - cf_common.user_db.set_account_id(member_id, guild_id, user['id'], user['resource']) + cf_common.user_db.set_account_id(member_id, guild_id, user.id, user.resource) except db.UniqueConstraintFailed: raise HandleCogError(f'The handle `{user["handle"]}` is already associated with another user.') cf_common.user_db.cache_clist_user(user) @@ -441,33 +433,24 @@ async def identify(self, ctx, handle: str): """ invoker = str(ctx.author) - resource = 'codeforces.com' + resource, handle = clist_common.resource_from_handle_notation(handle) + if resource!=Resources.CODEFORCES: + if resource==None: + return await ctx.send(f'Sorry `{invoker}`, all keyword can only be used with set command.') + if resource==Resources.GOOGLE: + return await ctx.send(f'Sorry `{invoker}`, you can\'t identify handles of `{Resources.GOOGLE}`, please ask a moderator to link your account.') - if ':' in handle: - resource = handle[0: handle.index(':')] - handle = handle[handle.index(':')+1:] - if resource in _CLIST_RESOURCE_SHORT_FORMS: - resource = _CLIST_RESOURCE_SHORT_FORMS[resource] - - if resource!='codeforces.com': - if resource=='all': - return await ctx.send(f'Sorry `{invoker}`, all keyword can only be used with set command') - if resource=='codingcompetitions.withgoogle.com': - return await ctx.send(f'Sorry `{invoker}`, you can\'t identify handles of codingcompetitions.withgoogle.com, please ask a moderator to link your account.') - if resource not in _SUPPORTED_CLIST_RESOURCES: - raise HandleCogError(f'The resource `{resource}` is not supported.') wait_msg = await ctx.channel.send('Fetching account details, please wait...') users = await clist.account(handle, resource) user = users[0] token = randomword(8) await wait_msg.delete() - field = "name" - if resource=='atcoder.jp': field = 'affiliation' + field = 'affiliation' if resource==Resources.ATCODER else 'name' wait_msg = await ctx.send(f'`{invoker}`, change your {field} to `{token}` on {resource} within 60 seconds') await asyncio.sleep(60) await wait_msg.delete() wait_msg = await ctx.channel.send(f'Verifying {field} change...') - if scaper.assert_display_name(handle, token, resource, ctx.author.mention): + if scaper.assert_field(handle, token, resource, ctx.author.mention): member = ctx.author await self._set_account_id(member.id, ctx.guild, user) await wait_msg.delete() @@ -625,24 +608,13 @@ async def list(self, ctx, *countries): if you wish to display only members from those countries. Country data is sourced from codeforces profiles. e.g. ;handle list Croatia Slovenia """ - resource = 'codeforces.com' - if len(countries)==1: - country = countries[0] - if country in _CLIST_RESOURCE_SHORT_FORMS: - resource = _CLIST_RESOURCE_SHORT_FORMS[country] - countries = [] - elif country in _SUPPORTED_CLIST_RESOURCES: - resource = country - countries = [] - if resource!='codeforces.com': + resource = clist_common.detect_loose_resource(countries) + if resource!=Resources.CODEFORCES: clist_users = cf_common.user_db.get_clist_users_for_guild(ctx.guild.id, resource=resource) users = [] - for user in clist_users: - handle = user['handle'] - rating = int(user['rating']) if user['rating']!=None else None - member = ctx.guild.get_member(int(user['user_id'])) - n_contests = user['n_contests'] - users.append((member, handle, rating, n_contests)) + for user_id, user in clist_users: + member = ctx.guild.get_member(user_id) + users.append((member, user.handle, user.rating, user.n_contests)) else: countries = [country.title() for country in countries] res = cf_common.user_db.get_cf_users_for_guild(ctx.guild.id) diff --git a/tle/util/clist_api.py b/tle/util/clist_api.py index a010d4b6..da3f5aa0 100644 --- a/tle/util/clist_api.py +++ b/tle/util/clist_api.py @@ -1,5 +1,9 @@ import logging import os +import datetime as dt +import time +from tle.util import codeforces_api as cf +from tle.util.codeforces_api import Contest as CfContest, make_from_dict import requests from discord.ext import commands @@ -10,10 +14,6 @@ logger = logging.getLogger(__name__) URL_BASE = 'https://clist.by/api/v2/' -_SUPPORTED_CLIST_RESOURCES = ('codechef.com', 'atcoder.jp','codingcompetitions.withgoogle.com') -_CLIST_RESOURCE_SHORT_FORMS = {'cc':'codechef.com','codechef':'codechef.com', 'cf':'codeforces.com', - 'codeforces':'codeforces.com','ac':'atcoder.jp', 'atcoder':'atcoder.jp', - 'google':'codingcompetitions.withgoogle.com'} class ClistNotConfiguredError(commands.CommandError): """An error caused when clist credentials are not set in environment variables""" @@ -71,6 +71,7 @@ async def wrapped(*args, **kwargs): @ratelimit async def _query_clist_api(path, data): url = URL_BASE + path + logger.info(f'Querying Clist API at {url} with {data}') clist_token = os.getenv('CLIST_API_TOKEN') if not clist_token: raise ClistNotConfiguredError @@ -90,6 +91,8 @@ async def _query_clist_api(path, data): except Exception as e: logger.error(f'Request to Clist API encountered error: {e!r}') raise ClientError from e +class User(cf.namedtuple('User', 'id handle resource name rating n_contests')): + __slots__ = () async def account(handle, resource): params = {'total_count': True, 'handle':handle} @@ -102,7 +105,7 @@ async def account(handle, resource): resp = resp['objects'] if len(resp)==0: raise HandleNotFoundError(handle=handle, resource=resource) - return resp + return [cf.make_from_dict(User, user_dict) for user_dict in resp] async def fetch_user_info(resource, account_ids=None, handles=None): params = {'resource':resource, 'limit':1000} @@ -121,13 +124,56 @@ async def fetch_user_info(resource, account_ids=None, handles=None): raise ClientError else: resp = resp['objects'] - return resp + return [cf.make_from_dict(User, user_dict) for user_dict in resp] -async def statistics(account_id=None, contest_id=None, order_by=None, account_ids=None, resource=None): +def format_standings(standings, index_map, indexes): + results = [] + for standing in standings: + member_dict = { + 'handle': standing['handle'] + } + party_dict = { + 'contestId': standing['contest_id'], + 'members': [cf.make_from_dict(cf.Member, member_dict)], + 'participantType': 'CONTESTANT' , + 'teamId': None , + 'teamName': None , + 'ghost': None , + 'room': None , + 'startTimeSeconds': None + } + problems = [] + for index in indexes: + id = index_map[index] + points = ' ' + if 'problems' in standing and id in standing['problems']: + problem = standing['problems'][id] + points = problem['result'] + problem_dict = { + 'points': points, + 'penalty': None, + 'rejectedAttemptCount': None, + 'type': None, + 'bestSubmissionTimeSeconds': None + } + problems.append(cf.make_from_dict(cf.ProblemResult, problem_dict)) + standing_dict = { + 'party': cf.make_from_dict(cf.Party, party_dict) , + 'rank': standing['place'] , + 'points': standing['score'] , + 'penalty': None , + 'problemResults': problems + } + results.append(cf.make_from_dict(cf.RanklistRow, standing_dict)) + return results + +async def statistics(account_id=None, contest_id=None, order_by=None, account_ids=None, resource=None, with_problems=False, with_extra_fields=False): params = {'limit':1000} if account_id!=None: params['account_id'] = account_id if contest_id!=None: params['contest_id'] = contest_id if order_by!=None: params['order_by'] = order_by + if with_problems: params['with_problems'] = True + if with_extra_fields: params['with_more_fields'] = True if account_ids!=None: ids = "" for i in range(len(account_ids)): @@ -154,9 +200,59 @@ async def statistics(account_id=None, contest_id=None, order_by=None, account_id offset+=1000 return results +class Contest(CfContest): + @property + def resource(self): + return self._resource + + @resource.setter + def resource(self, value): + self._resource = value + + @property + def url(self): + return self._url + + @url.setter + def url(self, value): + self._url = value + + @property + def register_url(self): + return self._url + +def time_in_seconds(time_str): + time = dt.datetime.strptime(time_str,'%Y-%m-%dT%H:%M:%S') + return int((time-dt.datetime(1970,1,1)).total_seconds()) + +def format_contest(contest): + start = time_in_seconds(contest['start']) + now = int(time.time()) + duration = contest['duration'] + phase = '' + if now Date: Wed, 4 Aug 2021 15:17:26 +0530 Subject: [PATCH 11/11] minor bug fix --- tle/cogs/contests.py | 5 ++++- tle/util/clist_common.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tle/cogs/contests.py b/tle/cogs/contests.py index 7c5a4691..606abe44 100644 --- a/tle/cogs/contests.py +++ b/tle/cogs/contests.py @@ -313,7 +313,10 @@ def _get_cf_or_ioi_standings_table(problem_indices, handle_standings, deltas=Non assert mode in ('cf', 'ioi') def maybe_int(value): - return int(value) if mode == 'cf' else value + try: + return int(value) + except: + return value header_style = '{:>} {:<} {:^} ' + ' '.join(['{:^}'] * len(problem_indices)) body_style = '{:>} {:<} {:>} ' + ' '.join(['{:>}'] * len(problem_indices)) diff --git a/tle/util/clist_common.py b/tle/util/clist_common.py index 6364aa68..d29fff4d 100644 --- a/tle/util/clist_common.py +++ b/tle/util/clist_common.py @@ -112,7 +112,7 @@ async def resolve_handles(ctx, converter, handles, *, mincnt=1, maxcnt=5, defaul clist_users = await clist.fetch_user_info(resource=resource, handles=list(unresolved_handles)) if clist_users!=None: for user in clist_users: - account_ids.add(int(user['id'])) + account_ids.add(int(user.id)) return list(account_ids) async def resolve_contest(contest_id, resource):