From 88cec3c6a33d6ab969fe5e920db97191847b1843 Mon Sep 17 00:00:00 2001 From: Ritsuki Otake Date: Tue, 19 Sep 2023 13:12:52 +0900 Subject: [PATCH 1/2] update README --- README.md | 146 +----------------------------------------------------- 1 file changed, 1 insertion(+), 145 deletions(-) diff --git a/README.md b/README.md index 50b4c40..009348e 100644 --- a/README.md +++ b/README.md @@ -1,145 +1 @@ -# letus-scraper - -```bash -/home/otkrickey/dev/letus-scraper/.venv/bin/python /home/otkrickey/dev/letus-scraper/main.py -l info -e 1 -/home/otkrickey/dev/letus-scraper/.venv/bin/python /home/otkrickey/dev/letus-scraper/main.py -s checkAccount -d 601235188571176961 -l info -e 1098772108579962950:1099020068198809610 -``` - -## TODO - -### Task on MongoDB - -- default - - - Discord: (user_id, guild_id, category_id, channel_id, message_id) - -- Account - - - Register Account - - TUS: (student_id, encrypted_password) - - Update Account Info - - TUS: (student_id, encrypted_password) - - Unregister Account - - TUS: (student_id, encrypted_password) - - Check Account Info - - TUS: (student_id) - -- Page - - Register Page - - LETUS: (page_id) - - TUS: (student_id) # Optional: If not provided, use the student_id from the account - - Fetch Page - - LETUS: (page_id) - - TUS: (student_id) # Optional: If not provided, use the student_id from the account - - Unregister Page - - LETUS: (page_id) - - TUS: (student_id) # Optional: If not provided, use the student_id from the account - -### Flow - -```mermaid -graph TD - -subgraph ServiceManager - SM{ServiceManager} - - S1[CheckAccount] - S2[RegisterAccount] - S3[CheckContent] - - SM --> S1 - SM --> S3 - SM --> S2 -end - -subgraph Middleware - M1[fetchAccount] - M2[loginAccount] - M3[registerAccount] - M4[uploadAccount] - M5[fetchPage] - M6[fetchContent] -end - -subgraph Manager - subgraph AccountManager - - AM1[check_account_info] - AM2[fetch_account_info] - AM3[register_account] - AM4[update_account] - - AM1 --> AM2 - AM3 --> AM4 - end -end - -subgraph Model - subgraph LetusAccount - LA1[discord_id] - subgraph LetusPage - LP1[discord_id] - subgraph LetusContent - LC1[discord_id] - LC2[DB_args] - end - end - end - - subgraph LetusSession - LS1[login] - LS2[register] - LS3[__login_letus] - LS4[__login_microsoft] - LS5[load_cookie] - - LS1 --> LS2 --> LS3 --> LS4 --> LS5 - end -end - -S1 --> M1 -S2 --> M3 -S3 --> M5 - -M1 --> AM1 -M2 --> LS1 -M3 --> LS2 -M4 --> AM3 -M5 --> M6 - -LS5 --> M4 - -AM2 --> M2 -LetusPage --> LetusContent -``` - -```json -[ - { - "task": "account:status", - "discord_id": "601235188571176961" - }, - { - "task": "account:login", - "multi_id": "601235188571176961", - }, - { - "task": "account:register", - "student_id": "7322023", - "discord_id": "601235188571176961", - "Letus": { - "encrypted_password": "password", - "email": "email" - }, - "Discord": { - "username": "otkrickey", - "discriminator": "otkrickey" - } - } -] - - - - - -``` +# letusc: `v7.0.0` From 5895b03e708e817ebabff2417ac4276e9bde245e Mon Sep 17 00:00:00 2001 From: Ritsuki Otake Date: Tue, 19 Sep 2023 13:16:16 +0900 Subject: [PATCH 2/2] =?UTF-8?q?=E8=A9=A6=E9=A8=93=E7=9A=84=E3=81=AA?= =?UTF-8?q?=E6=A9=9F=E8=83=BD=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- letusc/models/account.py | 3 +-- letusc/models/page.py | 12 +----------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/letusc/models/account.py b/letusc/models/account.py index fb4cc6b..d075bb3 100644 --- a/letusc/models/account.py +++ b/letusc/models/account.py @@ -1,8 +1,7 @@ from dataclasses import dataclass, field -from letusc.logger import L - from ..db import DBManager +from ..logger import L from .base import BaseDatabase, BaseModel, attrs, from_api_attrs, to_api_attrs, types from .cookie import Cookie from .discord import DiscordUser, DiscordUserAny, DiscordUserBase diff --git a/letusc/models/page.py b/letusc/models/page.py index 0d5ff7d..d1288fd 100644 --- a/letusc/models/page.py +++ b/letusc/models/page.py @@ -8,7 +8,6 @@ from ..db import DBManager from ..logger import L from ..session import SessionManager -from ..util import env_bool from .base import ( BaseDatabase, BaseModel, @@ -150,13 +149,6 @@ async def get(self, cookie: Cookie) -> bs4.BeautifulSoup: html = await response.text() return BeautifulSoup(html, "html.parser") - async def get_local(self) -> bs4.BeautifulSoup: - _l = self._l.gm("get_local") - _l.info(f"Requesting page: {self.url}") - with open(f"{self.page_id}.example.html", "r") as f: - html = f.read() - return BeautifulSoup(html, "html.parser") - @dataclass class Page(PageBase): @@ -233,9 +225,7 @@ async def parse(cls, code: PageCode, cookie: Cookie) -> "NewPage": page = NewCoursePage(code.code) case _: raise ValueError(_l.c("UnknownPageType")) - soup = ( - await page.get_local() if env_bool("USE_LOCAL") else await page.get(cookie) - ) + soup = await page.get(cookie) await page._parse(soup) return page