From 58a7e93940b474c9df6efb33492f68204c75f24c Mon Sep 17 00:00:00 2001 From: adbenitez Date: Tue, 5 Mar 2024 00:52:05 -0500 Subject: [PATCH] port to deltabot-cli --- .github/workflows/python-ci.yml | 14 +- .isort.cfg | 2 - .mypy.ini | 2 - CHANGELOG.rst | 7 - MANIFEST.in | 2 - README.md | 42 ++++ README.rst | 51 ----- feedsbot/__init__.py | 11 ++ feedsbot/hooks.py | 271 ++++++++++++++++++++++++++ feedsbot/orm.py | 56 ++++++ {simplebot_feeds => feedsbot}/util.py | 165 ++++++++-------- pylama.ini | 4 +- pyproject.toml | 49 +++++ requirements/requirements-dev.txt | 9 - requirements/requirements-test.txt | 2 - requirements/requirements.txt | 5 - setup.py | 66 ------- simplebot_feeds/__init__.py | 142 -------------- simplebot_feeds/db.py | 117 ----------- simplebot_feeds/subcommands.py | 53 ----- tests/test_plugin.py | 36 ---- 21 files changed, 512 insertions(+), 594 deletions(-) delete mode 100644 .isort.cfg delete mode 100644 .mypy.ini delete mode 100644 CHANGELOG.rst delete mode 100644 MANIFEST.in create mode 100644 README.md delete mode 100644 README.rst create mode 100644 feedsbot/__init__.py create mode 100644 feedsbot/hooks.py create mode 100644 feedsbot/orm.py rename {simplebot_feeds => feedsbot}/util.py (59%) create mode 100644 pyproject.toml delete mode 100644 requirements/requirements-dev.txt delete mode 100644 requirements/requirements-test.txt delete mode 100644 requirements/requirements.txt delete mode 100644 setup.py delete mode 100644 simplebot_feeds/__init__.py delete mode 100644 simplebot_feeds/db.py delete mode 100644 simplebot_feeds/subcommands.py delete mode 100644 tests/test_plugin.py diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 53227f1..9b2d73c 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.9] + python-version: ['3.8', '3.11'] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -24,9 +24,6 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install '.[dev]' - - name: Check code with isort - run: | - isort --check . - name: Check code with black run: | black --check . @@ -35,7 +32,7 @@ jobs: pylama - name: Test with pytest run: | - pytest + #pytest deploy: needs: test @@ -57,10 +54,3 @@ jobs: build: true # only upload if a tag is pushed (otherwise just build & check) upload: ${{ github.event_name == 'push' && steps.check-tag.outputs.match == 'true' }} - - name: Create GitHub release - if: ${{ github.event_name == 'push' && steps.check-tag.outputs.match == 'true' }} - uses: Roang-zero1/github-create-release-action@master - with: - version_regex: ^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+ - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index 9caba16..0000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[isort] -profile=black diff --git a/.mypy.ini b/.mypy.ini deleted file mode 100644 index 976ba02..0000000 --- a/.mypy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -ignore_missing_imports = True diff --git a/CHANGELOG.rst b/CHANGELOG.rst deleted file mode 100644 index 5a18fcc..0000000 --- a/CHANGELOG.rst +++ /dev/null @@ -1,7 +0,0 @@ -Changelog -********* - -1.0.0 ------ - -- initial release diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 7d61281..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include LICENSE -include *.rst diff --git a/README.md b/README.md new file mode 100644 index 0000000..ed7d48c --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +# Feeds + +[![Latest Release](https://img.shields.io/pypi/v/feedsbot.svg)](https://pypi.org/project/feedsbot) +[![CI](https://github.com/deltachat-bot/feedsbot/actions/workflows/python-ci.yml/badge.svg)](https://github.com/deltachat-bot/feedsbot/actions/workflows/python-ci.yml) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + +Delta Chat bot that allows to subscribe to RSS/Atom feeds. + +## Install + +```sh +pip install feedsbot +``` + +Configure the bot: + +```sh +feedsbot init bot@example.com PASSWORD +``` + +Start the bot: + +```sh +feedsbot serve +``` + +Run `feedsbot --help` to see all available options. + +## User Guide + +To subscribe an existing group to some feed: + +1. Add the bot to the group. +2. Send `/sub https://delta.chat/feed.xml` (replace the URL with the desired feed) + +To subscribe to a feed and let the bot create a dedicated group for you with the feed image as group avatar, etc., just send the command `/sub https://delta.chat/feed.xml` (replacing the URL for the desired feed) to the bot in private/direct (1:1) chat. + +To unsubscribe the group from all feeds, just remove the bot from the group, or to unsubscribe from a particular feed (replace feed URL as appropriate): + +`/unsub https://delta.chat/feed.xml` + +To see all feeds a group is subscribed to, just send `/list` inside the desired group. diff --git a/README.rst b/README.rst deleted file mode 100644 index 7051e4b..0000000 --- a/README.rst +++ /dev/null @@ -1,51 +0,0 @@ -Feeds -===== - -.. image:: https://github.com/simplebot-org/simplebot_feeds/actions/workflows/python-ci.yml/badge.svg - :target: https://github.com/simplebot-org/simplebot_feeds/actions/workflows/python-ci.yml - -.. image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black - -A `SimpleBot`_ plugin that allows to subscribe to RSS/Atom feeds. - -Install -------- - -To install run:: - - pip install git+https://github.com/simplebot-org/simplebot_feeds - -Configuration -------------- - -By default the time (in seconds) between checks for feeds updates is 300 seconds (5 minutes), to change it:: - - simplebot -a bot@example.com feeds --interval 600 - -To limit the total number of feeds subscriptions the bot will allow (by default it is unlimited):: - - simplebot -a bot@example.com feeds --max 1000 - -If this plugin has collisions with commands from other plugins in your bot, you can set a command prefix like ``/feed_`` for all commands:: - - simplebot -a bot@example.com feeds --prefix feed_ - -User Guide ----------- - -To subscribe an existing group to some feed: - -1. Add the bot to the group. -2. Send `/sub https://delta.chat/feed.xml` (replace the URL with the desired feed) - -To subscribe to a feed and let the bot create a dedicated group for you with the feed image as group avatar, etc., just send the command `/sub https://delta.chat/feed.xml` (replacing the URL for the desired feed) to the bot in private/direct (1:1) chat. - -To unsubscribe the group from all feeds, just remove the bot from the group, or to unsubscribe from a particular feed (replace feed URL as appropriate): - -`/unsub https://delta.chat/feed.xml` - -To see all feeds a group is subscribed to, just send `/list` inside the desired group. - - -.. _SimpleBot: https://github.com/simplebot-org/simplebot diff --git a/feedsbot/__init__.py b/feedsbot/__init__.py new file mode 100644 index 0000000..e15f0ea --- /dev/null +++ b/feedsbot/__init__.py @@ -0,0 +1,11 @@ +"""Feeds bot.""" + +from .hooks import cli + + +def main() -> None: + """Run the application.""" + try: + cli.start() + except KeyboardInterrupt: + pass diff --git a/feedsbot/hooks.py b/feedsbot/hooks.py new file mode 100644 index 0000000..55b0ec4 --- /dev/null +++ b/feedsbot/hooks.py @@ -0,0 +1,271 @@ +"""Event Hooks""" + +import os +from argparse import Namespace +from threading import Thread + +from deltabot_cli import ( + AttrDict, + Bot, + BotCli, + ChatType, + EventType, + SpecialContactId, + events, + is_not_known_command, +) +from feedparser import FeedParserDict +from sqlalchemy import delete, func, select + +from .orm import Fchat, Feed, init, session_scope +from .util import ( + check_feeds, + format_entries, + get_latest_date, + get_old_entries, + normalize_url, + parse_feed, + set_group_image, +) + +cli = BotCli("feedsbot") +cli.add_generic_option( + "--interval", + type=int, + default=60 * 5, + help="how many seconds to sleep before checking the feeds again (default: %(default)s", +) +cli.add_generic_option( + "--max", + type=int, + default=-1, + help="the maximum number of feeds the bot will subscribe to before rejecting new unknown feeds, by default: -1 (infinite)", +) + + +@cli.on_init +def on_init(bot: Bot, _args: Namespace) -> None: + for accid in bot.rpc.get_all_account_ids(): + if not bot.rpc.get_config(accid, "displayname"): + bot.rpc.set_config(accid, "displayname", "FeedsBot") + status = "I am a Delta Chat bot, send me /help for more info" + bot.rpc.set_config(accid, "selfstatus", status) + bot.rpc.set_config(accid, "delete_server_after", "1") + bot.rpc.set_config(accid, "delete_device_after", "3600") + + +@cli.on_start +def on_start(bot: Bot, args: Namespace) -> None: + bot.add_hook( + (lambda b, a, e: _sub(args.max, b, a, e)), events.NewMessage(command="/sub") + ) + path = os.path.join(args.config_dir, "sqlite.db") + init(f"sqlite:///{path}") + Thread(target=check_feeds, args=(bot, args.interval), daemon=True).start() + + +@cli.on(events.RawEvent) +def log_event(bot: Bot, accid: int, event: AttrDict) -> None: + if event.kind == EventType.INFO: + bot.logger.info(event.msg) + elif event.kind == EventType.WARNING: + bot.logger.warning(event.msg) + elif event.kind == EventType.ERROR: + bot.logger.error(event.msg) + elif event.kind == EventType.SECUREJOIN_INVITER_PROGRESS: + if event.progress == 1000: + bot.logger.debug("QR scanned by contact id=%s", event.contact_id) + chatid = bot.rpc.create_chat_by_contact_id(accid, event.contact_id) + send_help(bot, accid, chatid) + + +@cli.on(events.MemberListChanged) +def on_memberlist_change(bot: Bot, accid: int, event: AttrDict) -> None: + bot.logger.info(f"memberlist changed!!!! {event}") + if event.member_added: + return + chat_id = event.msg.chat_id + chat = bot.rpc.get_full_chat_by_id(accid, chat_id) + if SpecialContactId.SELF not in chat.contact_ids or len(chat.contact_ids) <= 1: + with session_scope() as session: + stmt = delete(Fchat).where(Fchat.accid == accid, Fchat.gid == chat_id) + session.execute(stmt) + + +@cli.on(events.NewMessage(is_info=False)) +def markseen_commands(bot: Bot, accid: int, event: AttrDict) -> None: + if not is_not_known_command(bot, event): + bot.rpc.markseen_msgs(accid, [event.msg.id]) + + +@cli.on(events.NewMessage(is_info=False, func=is_not_known_command)) +def on_unknown_cmd(bot: Bot, accid: int, event: AttrDict) -> None: + msg = event.msg + chat = bot.rpc.get_basic_chat_info(accid, msg.chat_id) + if chat.chat_type == ChatType.SINGLE: + bot.rpc.markseen_msgs(accid, [msg.id]) + send_help(bot, accid, event.msg.chat_id) + + +@cli.on(events.NewMessage(command="/help")) +def _help(bot: Bot, accid: int, event: AttrDict) -> None: + send_help(bot, accid, event.msg.chat_id) + + +def send_help(bot: Bot, accid: int, chat_id: int) -> None: + text = """Hello, I'm a bot 🤖, with me you can subscribe group chats to RSS/Atom feeds. + +**Available commands** + +/sub URL - Subscribe current chat to the given feed. + Examples: + /sub https://delta.chat/feed.xml + /sub https://delta.chat/feed.xml keyword + +/unsub URL - Unsubscribe current chat from the given feed. + Example: + /unsub https://delta.chat/feed.xml + +/list - List feed subscriptions in the group the command is sent. + + +**How to use me?** + +Add me to a group then you can use the /sub command there to subscribe the group to RSS/Atom feeds. + """ + bot.rpc.send_msg(accid, chat_id, {"text": text}) + + +def _sub(max_feed_count: int, bot: Bot, accid: int, event: AttrDict) -> None: + chat = bot.rpc.get_basic_chat_info(accid, event.msg.chat_id) + args = event.payload.split(maxsplit=1) + url = normalize_url(args[0]) if args else "" + filter_ = args[1] if len(args) == 2 else "" + + with session_scope() as session: + feed = session.execute(select(Feed).where(Feed.url == url)).scalar() + if feed: + try: + d = parse_feed(feed.url) + except Exception as ex: + reply = { + "text": "❌ Invalid feed url.", + "quotedMessageId": event.msg.id, + } + bot.rpc.send_msg(accid, event.msg.chat_id, reply) + bot.logger.exception("Invalid feed %s: %s", url, ex) + return + else: + stmt = select(func.count()).select_from( # noqa: func.count is callable + Feed + ) + if 0 <= max_feed_count <= session.execute(stmt).scalar_one(): + reply = {"text": "❌ Sorry, maximum number of feeds reached"} + bot.rpc.send_msg(accid, event.msg.chat_id, reply) + return + try: + d = parse_feed(url) + feed = Feed( + url=url, + etag=d.get("etag"), + modified=d.get("modified") or d.get("updated"), + latest=get_latest_date(d.entries), + ) + session.add(feed) + except Exception as ex: + reply = { + "text": "❌ Invalid feed url.", + "quotedMessageId": event.msg.id, + } + bot.rpc.send_msg(accid, event.msg.chat_id, reply) + bot.logger.exception("Invalid feed %s: %s", url, ex) + return + + if chat.chat_type == ChatType.SINGLE: + chat_id = bot.rpc.create_group_chat( + accid, d.feed.get("title") or url, False + ) + bot.rpc.add_contact_to_chat(accid, chat_id, event.msg.from_id) + url = d.feed.get("image", {}).get("href") or d.feed.get("logo") + if url: + set_group_image(bot, url, accid, chat_id) + else: + chat_id = event.msg.chat_id + stmt = select(Fchat).where( + Fchat.accid == accid, Fchat.gid == chat.id, Fchat.feed_url == feed.url + ) + if session.execute(stmt).scalar(): + reply = { + "text": "❌ Chat already subscribed to that feed.", + "quotedMessageId": event.msg.id, + } + bot.rpc.send_msg(accid, chat_id, reply) + return + + session.add(Fchat(accid=accid, gid=chat_id, feed_url=feed.url, filter=filter_)) + + reply = {"text": _format_feed_info(d, feed.url, filter_)} + + if d.entries and feed.latest: + reply["html"] = format_entries( + get_old_entries(d.entries, tuple(map(int, feed.latest.split())))[:15], + filter_, + ) + + bot.rpc.send_msg(accid, chat_id, reply) + + +def _format_feed_info(d: FeedParserDict, url: str, filter_: str) -> str: + url = f"{url} ({filter_})" if filter_ else url + title = d.feed.get("title") or "-" + desc = d.feed.get("description") or "-" + return f"Title: {title}\n\nURL: {url}\n\nDescription: {desc}" + + +@cli.on(events.NewMessage(command="/unsub")) +def _unsub(bot: Bot, accid: int, event: AttrDict) -> None: + if not event.payload: + _list(bot, accid, event) + return + + msg = event.msg + with session_scope() as session: + stmt = select(Feed).where(Feed.url == normalize_url(event.payload)) + feed = session.execute(stmt).scalar() + fchat = None + if feed: + stmt = select(Fchat).where( + Fchat.accid == accid, + Fchat.gid == msg.chat_id, + Fchat.feed_url == feed.url, + ) + fchat = session.execute(stmt).scalar() + if fchat: + session.delete(fchat) + reply = {"text": f"Chat unsubscribed from: {feed.url}"} + bot.rpc.send_msg(accid, msg.chat_id, reply) + else: + reply = { + "text": "❌ This chat is not subscribed to that feed", + "quotedMessageId": msg.id, + } + bot.rpc.send_msg(accid, msg.chat_id, reply) + + +@cli.on(events.NewMessage(command="/list")) +def _list(bot: Bot, accid: int, event: AttrDict) -> None: + msg = event.msg + chat = bot.rpc.get_basic_chat_info(accid, msg.chat_id) + if chat.chat_type == ChatType.SINGLE: + text = ( + "❌ You must send that command in the group where you have the subscriptions.\n" + "You can check the groups you share with me in my profile" + ) + reply = {"text": text, "quotedMessageId": msg.id} + else: + with session_scope() as session: + stmt = select(Fchat).where(Fchat.accid == accid, Fchat.gid == msg.chat_id) + fchats = session.execute(stmt).scalars() + text = "\n\n".join(fchat.feed_url for fchat in fchats) + reply = {"text": text or "❌ No feed subscriptions in this chat"} + bot.rpc.send_msg(accid, msg.chat_id, reply) diff --git a/feedsbot/orm.py b/feedsbot/orm.py new file mode 100644 index 0000000..e338ea8 --- /dev/null +++ b/feedsbot/orm.py @@ -0,0 +1,56 @@ +"""database""" + +from contextlib import contextmanager +from threading import Lock +from typing import Any, Generator + +from sqlalchemy import Column, ForeignKey, Integer, String, create_engine +from sqlalchemy.orm import Session, declarative_base, relationship, sessionmaker + +Base: Any = declarative_base() +_Session = sessionmaker(expire_on_commit=False) +_lock = Lock() + + +class Feed(Base): + __tablename__ = "feeds" + url = Column(String, primary_key=True) + etag = Column(String) + modified = Column(String) + latest = Column(String) + errors = Column(Integer, nullable=False) + fchats = relationship("Fchat", backref="feed", cascade="all, delete, delete-orphan") + + def __init__(self, **kwargs): + kwargs.setdefault("errors", 0) + super().__init__(**kwargs) + + +class Fchat(Base): + __tablename__ = "fchats" + accid = Column(Integer, primary_key=True) + gid = Column(Integer, primary_key=True) + feed_url = Column(String, ForeignKey("feeds.url"), primary_key=True) + filter = Column(String) + + +@contextmanager +def session_scope() -> Generator[Session, None, None]: + """Provide a transactional scope around a series of operations.""" + with _lock: + session: Session = _Session() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + +def init(path: str, debug: bool = False) -> None: + """Initialize engine.""" + engine = create_engine(path, echo=debug) + Base.metadata.create_all(engine) + _Session.configure(bind=engine) diff --git a/simplebot_feeds/util.py b/feedsbot/util.py similarity index 59% rename from simplebot_feeds/util.py rename to feedsbot/util.py index 038818c..c2ec55a 100644 --- a/simplebot_feeds/util.py +++ b/feedsbot/util.py @@ -3,9 +3,7 @@ import datetime import functools import mimetypes -import os import re -import sqlite3 import time from tempfile import NamedTemporaryFile from typing import Optional @@ -13,82 +11,104 @@ import bs4 import feedparser import requests -from deltachat import Chat +from deltabot_cli import Bot, JsonRpcError from feedparser.datetimes import _parse_date from feedparser.exceptions import CharacterEncodingOverride -from simplebot.bot import DeltaBot, Replies +from sqlalchemy import delete, select, update -from simplebot_feeds import db +from .orm import Fchat, Feed, session_scope -session = requests.Session() -session.headers.update( +www = requests.Session() +www.headers.update( { "user-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" } ) -session.request = functools.partial(session.request, timeout=15) # type: ignore -scope = __name__.split(".", maxsplit=1)[0] +www.request = functools.partial(www.request, timeout=15) # type: ignore -def check_feeds(bot: DeltaBot) -> None: +def check_feeds(bot: Bot, interval: int) -> None: while True: bot.logger.debug("Checking feeds") - now = time.time() - for f in db.manager.get_feeds(): - bot.logger.debug("Checking feed: %s", f["url"]) + starting_time = time.time() + with session_scope() as session: + feeds = session.execute(select(Feed)).scalars().all() + for feed in feeds: + bot.logger.debug("Checking feed: %s", feed.url) try: - _check_feed(bot, f) - if f["errors"] != 0: - db.manager.set_feed_errors(f["url"], 0) + _check_feed(bot, feed) + if feed.errors != 0: + with session_scope() as session: + stmt = update(Feed).where(Feed.url == feed.url).values(errors=0) + session.execute(stmt) except Exception as err: bot.logger.exception(err) - if f["errors"] < 50: - db.manager.set_feed_errors(f["url"], f["errors"] + 1) + if feed.errors < 50: + with session_scope() as session: + stmt = update(Feed).where(Feed.url == feed.url) + session.execute(stmt.values(errors=feed.errors + 1)) continue - for gid in db.manager.get_fchat_ids(f["url"]): + + with session_scope() as session: + stmt = select(Fchat.accid, Fchat.gid).where( + Fchat.feed_url == feed.url + ) + fchats = session.execute(stmt).scalars().all() + session.execute(delete(Feed).where(Feed.url == feed.url)) + reply = { + "text": f"❌ Due to errors, this chat was unsubscribed from feed: {feed.url}" + } + for accid, gid in fchats: try: - replies = Replies(bot, logger=bot.logger) - replies.add( - text=f"❌ Due to errors, this chat was unsubscribed from feed: {f['url']}", - chat=bot.get_chat(gid), - ) - replies.send_reply_messages() - except (ValueError, AttributeError): + bot.rpc.send_msg(accid, gid, reply) + except JsonRpcError: pass - db.manager.remove_feed(f["url"]) bot.logger.debug("Done checking feeds") - delay = int(get_default(bot, "delay")) - time.time() + now + delay = interval - (time.time() - starting_time) if delay > 0: time.sleep(delay) -def _check_feed(bot: DeltaBot, f: sqlite3.Row) -> None: - d = parse_feed(f["url"], etag=f["etag"], modified=f["modified"]) +def _check_feed(bot: Bot, feed: Feed) -> None: + d = parse_feed(feed.url, etag=feed.etag, modified=feed.modified) - if d.entries and f["latest"]: - d.entries = get_new_entries(d.entries, tuple(map(int, f["latest"].split()))) + if d.entries and feed.latest: + d.entries = get_new_entries(d.entries, tuple(map(int, feed.latest.split()))) if not d.entries: return full_html = format_entries(d.entries[:100], "") - for gid, filter_ in db.manager.get_fchats(f["url"]): - html = full_html if not filter_ else format_entries(d.entries[:100], filter_) - if not html: - continue - replies = Replies(bot, logger=bot.logger) + with session_scope() as session: + stmt = select(Fchat).where(Fchat.feed_url == feed.url) + fchats = session.execute(stmt).scalars().all() + if not fchats: + session.delete(feed) + return + + for fchat in fchats: + if fchat.filter: + html = format_entries(d.entries[:100], fchat.filter) + if not html: + continue + else: + html = full_html + reply = {"html": html, "OverrideSenderName": d.feed.get("title") or feed.url} try: - replies.add( - html=html, - chat=bot.get_chat(gid), - sender=d.feed.get("title") or f["url"], - ) - replies.send_reply_messages() - except (ValueError, AttributeError): - db.manager.remove_fchat(gid) - - latest = get_latest_date(d.entries) or f["latest"] + bot.rpc.send_msg(fchat.accid, fchat.gid, reply) + except JsonRpcError: + with session_scope() as session: + stmt = delete(Fchat).where( + Fchat.accid == fchat.accid, Fchat.gid == fchat.gid + ) + session.execute(stmt) + + latest = get_latest_date(d.entries) or feed.latest modified = d.get("modified") or d.get("updated") - db.manager.update_feed(f["url"], d.get("etag"), modified, latest) + with session_scope() as session: + stmt = update(Feed).where(Feed.url == feed.url) + session.execute( + stmt.values(etag=d.get("etag"), modified=modified, latest=latest) + ) def format_entries(entries: list, filter_: str) -> str: @@ -163,29 +183,8 @@ def get_latest_date(entries: list) -> Optional[str]: return " ".join(map(str, max(dates))) if dates else None -def get_default(bot: DeltaBot, key: str, value=None) -> str: - val = bot.get(key, scope=scope) - if val is None and value is not None: - bot.set(key, value, scope=scope) - val = value - return val - - -def set_config(bot: DeltaBot, key: str, value) -> None: - bot.set(key, value, scope=scope) - - -def init_db(bot: DeltaBot) -> None: - path = os.path.join( - os.path.dirname(bot.account.db_path), __name__.split(".", maxsplit=1)[0] - ) - if not os.path.exists(path): - os.makedirs(path) - db.manager = db.DBManager(os.path.join(path, "sqlite.db")) - - def parse_feed( - url: str, etag: str = None, modified: tuple = None + url: str, etag: Optional[str] = None, modified: Optional[tuple] = None ) -> feedparser.FeedParserDict: headers = {"A-IM": "feed", "Accept-encoding": "gzip, deflate"} if etag: @@ -219,7 +218,7 @@ def parse_feed( modified[4], modified[5], ) - with session.get(url, headers=headers) as resp: + with www.get(url, headers=headers) as resp: resp.raise_for_status() dict_ = feedparser.parse(resp.text) bozo_exception = dict_.get("bozo_exception", ValueError("Invalid feed")) @@ -255,19 +254,13 @@ def normalize_url(url: str) -> str: return url.rstrip("/") -def set_group_image(bot: DeltaBot, url: str, group: Chat) -> None: - with session.get(url) as resp: +def set_group_image(bot: Bot, url: str, accid: int, chatid: int) -> None: + with www.get(url) as resp: if resp.status_code < 400 or resp.status_code >= 600: - with NamedTemporaryFile( - dir=bot.account.get_blobdir(), - prefix="group-image-", - suffix=get_img_ext(resp), - delete=False, - ) as temp_file: - path = temp_file.name - with open(path, "wb") as file: - file.write(resp.content) - try: - group.set_profile_image(path) - except ValueError as ex: - bot.logger.exception(ex) + with NamedTemporaryFile(suffix=get_img_ext(resp)) as temp_file: + with open(temp_file.name, "wb") as file: + file.write(resp.content) + try: + bot.rpc.set_chat_profile_image(accid, chatid, temp_file.name) + except JsonRpcError as ex: + bot.logger.exception(ex) diff --git a/pylama.ini b/pylama.ini index fb29e1a..886918a 100644 --- a/pylama.ini +++ b/pylama.ini @@ -1,4 +1,4 @@ [pylama] linters=mccabe,pyflakes,pylint,isort,mypy -ignore=C0116,C0103,C0301,W0703,C901 -skip=.*,tests/*,build/*,simplebot_*/orm.py +ignore=C0116,C0115,R0903,C0103,C0301,W0703,C901 +skip=.*,build/*,tests/*,*/flycheck_* diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8867db2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,49 @@ +[build-system] +requires = ["setuptools>=64", "setuptools_scm>=8"] +build-backend = "setuptools.build_meta" + +[project] +name = "feedsbot" +description = "Subscribe to RSS/Atom feeds in Delta Chat" +dynamic = ["version"] +readme = "README.md" +requires-python = ">=3.8" +license = {file = "LICENSE"} +keywords = ["deltachat", "bot", "feeds", "rss", "atom"] +authors = [ + {name = "adbenitez", email = "adb@merlinux.eu"}, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3", +] +dependencies = [ + "deltabot-cli>=3.0.0,<4.0", + "feedparser==6.0.10", + "requests>=2.28.1", + "beautifulsoup4>=4.11.1", + "html5lib>=1.1", +] + +[project.optional-dependencies] +dev = [ + "black", + "mypy", + "isort", + "pylint", + "pylama", + "pytest", + "types-requests", +] + +[project.scripts] +feedsbot = "feedsbot:main" + +[tool.setuptools_scm] +# can be empty if no extra settings are needed, presence enables setuptools_scm + +[tool.isort] +profile = "black" + +[tool.mypy] +ignore_missing_imports = "True" diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt deleted file mode 100644 index f2560a4..0000000 --- a/requirements/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ --r requirements.txt --r requirements-test.txt -black==22.6.0 -mypy==0.971 -isort==5.10.1 -pylint==2.14.5 -pylama==8.4.1 -types-pkg-resources==0.1.3 -types-requests==2.28.9 diff --git a/requirements/requirements-test.txt b/requirements/requirements-test.txt deleted file mode 100644 index 9b2511d..0000000 --- a/requirements/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest==7.1.2 - diff --git a/requirements/requirements.txt b/requirements/requirements.txt deleted file mode 100644 index 55118b6..0000000 --- a/requirements/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -simplebot==3.2.0 -feedparser==6.0.10 -requests==2.28.1 -beautifulsoup4==4.11.1 -html5lib==1.1 diff --git a/setup.py b/setup.py deleted file mode 100644 index 6c703fc..0000000 --- a/setup.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Setup module installation.""" - -import os - -from setuptools import find_packages, setup - - -def load_requirements(path: str) -> list: - """Load requirements from the given relative path.""" - with open(path, encoding="utf-8") as file: - requirements = [] - for line in file.read().split("\n"): - if line.startswith("-r"): - dirname = os.path.dirname(path) - filename = line.split(maxsplit=1)[1] - requirements.extend(load_requirements(os.path.join(dirname, filename))) - elif line and not line.startswith("#"): - requirements.append(line.replace("==", ">=")) - return requirements - - -if __name__ == "__main__": - MODULE_NAME = "simplebot_feeds" - DESC = "Subscribe to RSS/Atom feeds in Delta Chat (SimpleBot plugin)" - KEYWORDS = "simplebot plugin deltachat feeds rss atom" - - with open("README.rst", encoding="utf-8") as fh: - long_description = fh.read() - - setup( - name=MODULE_NAME, - setup_requires=["setuptools_scm"], - use_scm_version={ - "root": ".", - "relative_to": __file__, - "tag_regex": r"^(?Pv)?(?P[^\+]+)(?P.*)?$", - "git_describe_command": "git describe --dirty --tags --long --match v*.*.*", - }, - description=DESC, - long_description=long_description, - long_description_content_type="text/x-rst", - author="The SimpleBot Contributors", - author_email="adbenitez@nauta.cu", - url=f"https://github.com/simplebot-org/{MODULE_NAME}", - keywords=KEYWORDS, - license="MPL", - classifiers=[ - "Development Status :: 3 - Alpha", - "Environment :: Plugins", - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", - "Operating System :: OS Independent", - "Topic :: Utilities", - ], - zip_safe=False, - include_package_data=True, - packages=find_packages(), - install_requires=load_requirements("requirements/requirements.txt"), - extras_require={ - "test": load_requirements("requirements/requirements-test.txt"), - "dev": load_requirements("requirements/requirements-dev.txt"), - }, - entry_points={ - "simplebot.plugins": f"{MODULE_NAME} = {MODULE_NAME}", - }, - ) diff --git a/simplebot_feeds/__init__.py b/simplebot_feeds/__init__.py deleted file mode 100644 index e3106f3..0000000 --- a/simplebot_feeds/__init__.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Hooks and commands.""" - -from threading import Thread - -import simplebot -from deltachat import Chat, Contact, Message -from pkg_resources import DistributionNotFound, get_distribution -from simplebot.bot import DeltaBot, Replies - -from .subcommands import def_interval, feeds -from .util import ( - check_feeds, - db, - format_entries, - get_default, - get_latest_date, - get_old_entries, - init_db, - normalize_url, - parse_feed, - set_group_image, -) - -try: - __version__ = get_distribution(__name__).version -except DistributionNotFound: - # package is not installed - __version__ = "0.0.0.dev0-unknown" - - -@simplebot.hookimpl -def deltabot_init_parser(parser) -> None: - parser.add_subcommand(feeds) - - -@simplebot.hookimpl -def deltabot_init(bot: DeltaBot) -> None: - init_db(bot) - - get_default(bot, "delay", def_interval) - get_default(bot, "max_feed_count", -1) - prefix = get_default(bot, "cmd_prefix", "") - - desc = f"Subscribe current chat to the given feed.\n\nExample:\n/{prefix}sub https://delta.chat/feed.xml\n/{prefix}sub https://delta.chat/feed.xml keyword" - bot.commands.register(func=sub_cmd, name=f"/{prefix}sub", help=desc) - desc = f"Unsubscribe current chat from the given feed.\n\nExample:\n/{prefix}unsub https://delta.chat/feed.xml" - bot.commands.register(func=unsub_cmd, name=f"/{prefix}unsub", help=desc) - bot.commands.register(func=list_cmd, name=f"/{prefix}list") - - -@simplebot.hookimpl -def deltabot_start(bot: DeltaBot) -> None: - Thread(target=check_feeds, args=(bot,), daemon=True).start() - - -@simplebot.hookimpl -def deltabot_member_removed(bot: DeltaBot, chat: Chat, contact: Contact) -> None: - me = bot.self_contact - if me == contact or len(chat.get_contacts()) <= 1: - db.manager.remove_fchat(chat.id) - - -def sub_cmd(bot: DeltaBot, payload: str, message: Message, replies: Replies) -> None: - args = payload.split(maxsplit=1) - url = normalize_url(args[0]) if args else "" - filter_ = args[1] if len(args) == 2 else "" - feed = dict(db.manager.get_feed(url) or {}) - - if feed: - try: - d = parse_feed(feed["url"]) - except Exception as ex: - replies.add(text="❌ Invalid feed url.", quote=message) - bot.logger.exception("Invalid feed %s: %s", url, ex) - return - else: - max_fc = int(get_default(bot, "max_feed_count")) - if 0 <= max_fc <= db.manager.get_feeds_count(): - replies.add(text="❌ Sorry, maximum number of feeds reached") - return - try: - d = parse_feed(url) - except Exception as ex: - replies.add(text="❌ Invalid feed url.", quote=message) - bot.logger.exception("Invalid feed %s: %s", url, ex) - return - feed = dict( - url=url, - etag=d.get("etag"), - modified=d.get("modified") or d.get("updated"), - latest=get_latest_date(d.entries), - ) - db.manager.add_feed(url, feed["etag"], feed["modified"], feed["latest"]) - - if message.chat.is_multiuser(): - chat = message.chat - if chat.id in db.manager.get_fchat_ids(feed["url"]): - replies.add( - text="❌ Chat already subscribed to that feed.", chat=chat, quote=message - ) - return - else: - chat = bot.create_group( - d.feed.get("title") or url, [message.get_sender_contact()] - ) - url = d.feed.get("image", {}).get("href") or d.feed.get("logo") - if url: - set_group_image(bot, url, chat) - - db.manager.add_fchat(chat.id, feed["url"], filter_) - url = f"{feed['url']} ({filter_})" if filter_ else feed["url"] - text = f"Title: {d.feed.get('title') or '-'}\n\nURL: {url}\n\nDescription: {d.feed.get('description') or '-'}" - - if d.entries and feed["latest"]: - latest = tuple(map(int, feed["latest"].split())) - html = format_entries(get_old_entries(d.entries, latest)[:15], filter_) - replies.add(text=text, html=html, chat=chat) - else: - replies.add(text=text, chat=chat) - - -def unsub_cmd(payload: str, message: Message, replies: Replies) -> None: - feed = db.manager.get_feed(normalize_url(payload)) - - if not feed or message.chat.id not in db.manager.get_fchat_ids(feed["url"]): - replies.add(text="❌ This chat is not subscribed to that feed", quote=message) - return - - db.manager.remove_fchat(message.chat.id, feed["url"]) - replies.add(text=f"Chat unsubscribed from: {feed['url']}") - - -def list_cmd(message: Message, replies: Replies) -> None: - """List feed subscriptions in the group the command is sent.""" - if message.chat.is_multiuser(): - text = "\n\n".join(f["url"] for f in db.manager.get_feeds(message.chat.id)) - replies.add(text=text or "❌ No feed subscriptions in this chat") - else: - replies.add( - text="❌ You must send that command in the group where you have the subscriptions", - quote=message, - ) diff --git a/simplebot_feeds/db.py b/simplebot_feeds/db.py deleted file mode 100644 index ade9520..0000000 --- a/simplebot_feeds/db.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Database utilities""" - -import sqlite3 -from typing import Iterator, Optional - - -class DBManager: - """Database manager""" - - def __init__(self, db_path: str) -> None: - self.db = sqlite3.connect(db_path, check_same_thread=False) - self.db.row_factory = sqlite3.Row - with self.db: - self.db.execute("PRAGMA foreign_keys = ON;") - self.db.execute( - """CREATE TABLE IF NOT EXISTS feeds - (url TEXT PRIMARY KEY, - etag TEXT, - modified TEXT, - latest TEXT, - errors INTEGER DEFAULT 0)""" - ) - self.db.execute( - """CREATE TABLE IF NOT EXISTS fchats - (gid INTEGER, - feed TEXT REFERENCES feeds(url) ON DELETE CASCADE, - filter TEXT, - PRIMARY KEY(gid, feed))""" - ) - - def close(self) -> None: - self.db.close() - - # ==== feeds ===== - - def add_feed(self, url: str, etag: str, modified: str, latest: str) -> None: - with self.db: - self.db.execute( - "INSERT INTO feeds VALUES (?,?,?,?,?)", (url, etag, modified, latest, 0) - ) - - def remove_feed(self, url: str) -> None: - with self.db: - self.db.execute("DELETE FROM feeds WHERE url=?", (url,)) - - def update_feed( - self, - url: str, - etag: Optional[str], - modified: Optional[str], - latest: Optional[str], - ) -> None: - with self.db: - self.db.execute( - "UPDATE feeds SET etag=?, modified=?, latest=? WHERE url=?", - (etag, modified, latest, url), - ) - - def set_feed_errors(self, url: str, errors: int) -> None: - with self.db: - self.db.execute("UPDATE feeds SET errors=? WHERE url=?", (errors, url)) - - def get_feed(self, url: str) -> Optional[sqlite3.Row]: - return self.db.execute("SELECT * FROM feeds WHERE url=?", (url,)).fetchone() - - def get_feeds_count(self) -> int: - return self.db.execute("SELECT count(*) FROM feeds").fetchone()[0] - - def get_feeds(self, gid: int = None) -> Iterator[sqlite3.Row]: - if gid is None: - for row in self.db.execute("SELECT * FROM feeds"): - yield row - return - rows = self.db.execute("SELECT feed FROM fchats WHERE gid=?", (gid,)).fetchall() - if not rows: - return - rows = [r[0] for r in rows] - q = "SELECT * FROM feeds WHERE " - q += " or ".join("url=?" for r in rows) - for row in self.db.execute(q, rows): - yield row - - def add_fchat(self, gid: int, url: str, filter_: str) -> None: - with self.db: - self.db.execute("INSERT INTO fchats VALUES (?,?,?)", (gid, url, filter_)) - - def remove_fchat(self, gid: int, url: str = None) -> None: - if url: - rows = self.db.execute( - "SELECT feed FROM fchats WHERE gid=? AND feed=?", (gid, url) - ) - else: - rows = self.db.execute("SELECT feed FROM fchats WHERE gid=?", (gid,)) - for row in rows: - fchats_count = self.db.execute( - "SELECT count(*) FROM fchats WHERE feed=?", (row[0],) - ).fetchone()[0] - if fchats_count <= 1: - self.remove_feed(row[0]) - with self.db: - if url: - self.db.execute("DELETE FROM fchats WHERE gid=? AND feed=?", (gid, url)) - else: - self.db.execute("DELETE FROM fchats WHERE gid=?", (gid,)) - - def get_fchats(self, url: str) -> Iterator[tuple]: - for row in self.db.execute( - "SELECT gid, filter FROM fchats WHERE feed=?", (url,) - ): - yield row[0], row[1] - - def get_fchat_ids(self, url: str) -> Iterator[int]: - for gid, _ in self.get_fchats(url): - yield gid - - -manager: DBManager = None # type: ignore diff --git a/simplebot_feeds/subcommands.py b/simplebot_feeds/subcommands.py deleted file mode 100644 index eddaeb4..0000000 --- a/simplebot_feeds/subcommands.py +++ /dev/null @@ -1,53 +0,0 @@ -"""extra command line subcommands for simplebot's CLI""" - -from simplebot import DeltaBot - -from .util import get_default, set_config - -def_interval = 60 * 5 - - -# pylama:ignore=C0103 -class feeds: - """Customize simplebot_feeds plugin's settings. - - Run without any arguments to see existing values. - """ - - def add_arguments(self, parser) -> None: - parser.add_argument( - "--interval", - type=int, - default=None, - help=f"set how many seconds to sleep between checking the feeds again (default: ${def_interval})", - ) - parser.add_argument( - "--max", - type=int, - default=None, - help="set the maximum amount of feeds the bot will subscribe to before rejecting new unknown feeds, by default: -1 (infinite)", - ) - parser.add_argument( - "--prefix", - default=None, - help='set a prefix to append to every of this plugin\'s commands by default: "" (no prefix)', - ) - - def run(self, bot: DeltaBot, args, out) -> None: - if args.interval is not None: - set_config(bot, "delay", args.interval) - out.line(f"interval: {args.interval}") - if args.max is not None: - set_config(bot, "max_feed_count", args.max) - out.line(f"max feeds count: {args.max}") - if args.prefix is not None: - set_config(bot, "cmd_prefix", args.prefix) - out.line(f"command prefix: {args.prefix}") - - if (args.interval, args.max, args.prefix) == (None, None, None): - interval = get_default(bot, "delay", def_interval) - max_feeds = get_default(bot, "max_feed_count", -1) - prefix = get_default(bot, "cmd_prefix", "") - out.line(f"interval: {interval}") - out.line(f"max feeds count: {max_feeds}") - out.line(f"command prefix: {prefix}") diff --git a/tests/test_plugin.py b/tests/test_plugin.py deleted file mode 100644 index 86349be..0000000 --- a/tests/test_plugin.py +++ /dev/null @@ -1,36 +0,0 @@ -class TestPlugin: - """Online tests""" - - def test_sub(self, mocker) -> None: - msg = mocker.get_one_reply("/sub") - assert "❌" in msg.text - - msg = mocker.get_one_reply("/sub https://delta.chat/feed.xml") - assert "❌" not in msg.text - chat = msg.chat - - msg = mocker.get_one_reply("/sub https://delta.chat/feed.xml", group=chat) - assert "❌" in msg.text - - def test_unsub(self, mocker) -> None: - msg = mocker.get_one_reply("/unsub https://delta.chat/feed.xml") - assert "❌" in msg.text - - msg = mocker.get_one_reply("/sub https://delta.chat/feed.xml") - chat = msg.chat - - msg = mocker.get_one_reply("/unsub https://delta.chat/feed.xml", group=chat) - assert "❌" not in msg.text - - def test_list(self, mocker) -> None: - msg = mocker.get_one_reply("/list") - assert "❌" in msg.text - - msg = mocker.get_one_reply("/list", group="group1") - assert "❌" in msg.text - - msg = mocker.get_one_reply("/sub https://delta.chat/feed.xml") - chat = msg.chat - - msg = mocker.get_one_reply("/list", group=chat) - assert "❌" not in msg.text