diff --git a/CHANGELOG.md b/CHANGELOG.md index 26c279ec8..4f116a061 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## v3.2.0 - 2024-11-21 +### What's Changed +**Full Changelog**: https://github.com/obervinov/pyinstabot-downloader/compare/v3.1.3...v3.2.0 by @obervinov in https://github.com/obervinov/pyinstabot-downloader/pull/116 +#### 🐛 Bug Fixes +* fix conflict between device metadata and user-agent in the `Downloader()` class +* fix the login strategy for exceptions +#### 💥 Breaking Changes +* `Downloader()` class configuration has been changed. **Please, check the new parameters in [README.md](README.md#bot-configuration-source-and-supported-parameters) and update Vault configuration** +* remove `Post` and `Posts List` buttons from the bot commands, because it is have been replaced by the `Posts` button (buttons have the same functionality). **Required to update the users roles in the vault and recreate start up message** +#### 🚀 Features +* add validators for session settings in the `Downloader()` class + + ## v3.1.3 - 2024-11-01 ### What's Changed **Full Changelog**: https://github.com/obervinov/pyinstabot-downloader/compare/v3.1.2...v3.1.3 by @obervinov in https://github.com/obervinov/pyinstabot-downloader/pull/115 diff --git a/README.md b/README.md index c8c3d7cfc..34fbb7a06 100644 --- a/README.md +++ b/README.md @@ -115,15 +115,17 @@ This project is a Telegram bot that allows you to upload posts from your Instagr "2fa-enabled": "False", "2fa-seed": "my_2fa_secret", "country-code": "1", + "country": "US", "delay-requests": "1", "locale": "en_US", "username": "my_username", "password": "my_password", "session-file": "session.json", "timezone-offset": "10800", - "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) ...", - "request-timeout": "10" + "request-timeout": "10", + "device-settings": {"app_version": "269.0.0.18.75", "version_code": "314665256", "manufacturer": "OnePlus", "model": "6T Dev", "device": "devitron", "cpu": "qcom", "dpi": "480dpi", "resolution": "1080x1920", "android_release": "8.0.0", "android_version": "26"} } + ``` Description of parameters @@ -131,14 +133,25 @@ This project is a Telegram bot that allows you to upload posts from your Instagr - `locale`: the locale of the instagram account - `session-file`: the path to the file where the session data will be stored - `timezone-offset`: the offset of the timezone in seconds - - `user-agent`: the user-agent of the instagram account - `2fa-enabled`: two-factor authentication status - `2fa-seed`: two-factor authentication secret - `country-code`: the country code of the instagram account + - `country`: the country of the instagram account - `enabled`: the status of the downloader module - `username`: the username of the instagram account - `password`: the password of the instagram account - `request-timeout`: the timeout for requests to the instagram api + - `device-settings`: the device settings of the instagram account + - `app_version`: the version of the instagram app + - `version_code`: the version code of the instagram app + - `manufacturer`: the manufacturer of the device + - `model`: the model of the device + - `device`: the device name + - `cpu`: the cpu of the device + - `dpi`: the dpi of the device + - `resolution`: the resolution of the device + - `android_release`: the android release version of the device + - `android_version`: the android api version of the device
- `configuration/uploader-api`: uploader module configuration (for upload content to the target storage) @@ -167,15 +180,15 @@ This project is a Telegram bot that allows you to upload posts from your Instagr ```json { - "requests": "{\"requests_per_day\": 10, \"requests_per_hour\": 1, \"random_shift_minutes\": 60}", - "roles": "[\"post\", \"posts_list\"]", + "requests": {"requests_per_day": 10, "requests_per_hour": 1, "random_shift_minutes": 60}, + "roles": ["posts", "reschedule_queue"], "status": "allowed" } ``` Description of parameters - `requests`: the number of requests that the user can make per day and per hour, as well as the random shift in minutes (scheduling of message processing from the queue works on the basis of this parameter) - - `roles`: list of roles that allow to use the corresponding functionality ([available roles](src/configs/constants.py#L11-L15)). + - `roles`: list of roles that allow to use the corresponding functionality ([available roles](src/configs/constants.py#L11-L14)). - `status`: allowed or denied user access to the bot diff --git a/docker-compose.yml b/docker-compose.yml index d5e8d665e..224567497 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,7 +46,7 @@ services: args: PROJECT_NAME: pyinstabot-downloader PROJECT_DESCRIPTION: "This project is a Telegram bot that allows you to upload posts from your Instagram profile to clouds like any WebDav compatible cloud storage." - PROJECT_VERSION: 3.1.3 + PROJECT_VERSION: 3.2.0 container_name: pyinstabot-downloader restart: always environment: diff --git a/pyproject.toml b/pyproject.toml index 44c51bd4b..1e9efe0c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pyinstabot-downloader" -version = "3.1.3" +version = "3.2.0" description = "This project is a Telegram bot that allows you to upload posts from your Instagram profile to clouds like any WebDav compatible cloud storage." authors = ["Bervinov Oleg "] maintainers = ["Bervinov Oleg "] diff --git a/src/bot.py b/src/bot.py index 41b508bab..ebb7d3db9 100644 --- a/src/bot.py +++ b/src/bot.py @@ -120,19 +120,12 @@ def bot_callback_query_handler(call: telegram.callback_query = None) -> None: 'chat_id': call.message.chat.id, 'message_id': call.message.message_id } if users.user_access_check(**requestor).get('permissions', None) == users.user_status_allow: - if call.data == "Post": - help_message = telegram.send_styled_message( - chat_id=call.message.chat.id, - messages_template={'alias': 'help_for_post'} - ) - bot.register_next_step_handler(call.message, process_one_post, help_message) - - elif call.data == "Posts List": + if call.data == "Posts": help_message = telegram.send_styled_message( chat_id=call.message.chat.id, messages_template={'alias': 'help_for_posts_list'} ) - bot.register_next_step_handler(call.message, process_list_posts, help_message) + bot.register_next_step_handler(call.message, process_posts, help_message) elif call.data == "Reschedule Queue": help_message = telegram.send_styled_message( @@ -367,6 +360,9 @@ def process_one_post( """ Processes an Instagram post link sent by a user and adds it to the queue for download. + Notice: This method will merge with the `process_posts` method in v3.3.0. + After combining the two buttons into a `Posts` button in version 3.2.0, it makes no sense to split one functionality into two methods. + Args: message (telegram.telegram_types.Message): The Telegram message object containing the post link. help_message (telegram.telegram_types.Message): The help message to be deleted. @@ -376,7 +372,7 @@ def process_one_post( None """ requestor = { - 'user_id': message.chat.id, 'role_id': ROLES_MAP['Post'], + 'user_id': message.chat.id, 'role_id': ROLES_MAP['Posts'], 'chat_id': message.chat.id, 'message_id': message.message_id } user = users_rl.user_access_check(**requestor) @@ -407,12 +403,12 @@ def process_one_post( telegram.delete_message(message.chat.id, help_message.id) -def process_list_posts( +def process_posts( message: telegram.telegram_types.Message = None, help_message: telegram.telegram_types.Message = None ) -> None: """ - Process a list of Instagram post links. + Process a single or multiple posts from the user's message. Args: message (telegram.telegram_types.Message, optional): The message containing the list of post links. Defaults to None. @@ -422,7 +418,7 @@ def process_list_posts( None """ requestor = { - 'user_id': message.chat.id, 'role_id': ROLES_MAP['Posts List'], + 'user_id': message.chat.id, 'role_id': ROLES_MAP['Posts'], 'chat_id': message.chat.id, 'message_id': message.message_id } user = users.user_access_check(**requestor) @@ -611,12 +607,6 @@ def queue_handler_thread() -> None: def main(): """ The main entry point of the project. - - Args: - None - - Returns: - None """ # Thread for processing queue thread_queue_handler = threading.Thread(target=queue_handler_thread, args=(), name="QueueHandlerThread") diff --git a/src/configs/constants.py b/src/configs/constants.py index 374922a3e..2d9dd4e0d 100644 --- a/src/configs/constants.py +++ b/src/configs/constants.py @@ -9,8 +9,7 @@ # permissions roles and buttons mapping # 'button_title': 'role' ROLES_MAP = { - 'Post': 'post', - 'Posts List': 'posts_list', + 'Posts': 'posts', 'Reschedule Queue': 'reschedule_queue', } diff --git a/src/configs/messages.json b/src/configs/messages.json index 825d55c3e..d0ba95925 100644 --- a/src/configs/messages.json +++ b/src/configs/messages.json @@ -16,10 +16,6 @@ "text": "{0} Sorry, you have not permissions for use this feature. Please, check you permission roles.\nMetadata:\nuserid: {1}\nusername: {2}", "args": [":no_entry:", "userid", "username"] }, - "help_for_post": { - "text": "{0} For a download post, just send a link to this post.\nfor example:\nhttps://www.instagram.com/p/QwErtY_1234", - "args": [":link:"] - }, "help_for_posts_list": { "text": "{0} To get a backup copy of the list of posts, send links to posts in a list (each new link with a new message line). This message will be automatically split into a number of messages equal to the number of links and processed in the order of the queue.\n {1} Example:\nhttps://www.instagram.com/p/QwEr_tY1234\nhttps://www.instagram.com/p/QwEr_tY1235\nhttps://www.instagram.com/p/QwEr_tY1236", "args": [":information:", ":link:"] diff --git a/src/modules/downloader.py b/src/modules/downloader.py index 8cb2516dc..f26f4c0e1 100644 --- a/src/modules/downloader.py +++ b/src/modules/downloader.py @@ -6,6 +6,8 @@ """ import os import time +import json +import random from pathlib import Path from urllib3.exceptions import ReadTimeoutError from requests.exceptions import ConnectionError as RequestsConnectionError @@ -19,6 +21,48 @@ class Downloader: """ An Instagram API instance is created by this class and contains a set of all the necessary methods to upload content from Instagram to a temporary directory. + + Attributes: + :attribute configuration (dict): dictionary with configuration parameters for instagram api communication. + :attribute client (object): instance of the instagram api client. + :attribute download_methods (dict): dictionary with download methods for instagram api client. + :attribute general_settings_list (list): list of general session settings for the instagram api. + :attribute device_settings_list (list): list of device settings for the instagram api. + + Methods: + :method _get_login_args: get login arguments for the instagram api. + :method _create_new_session: create a new session file for the instagram api. + :method _handle_relogin: handle re-authentication in the instagram api. + :method _load_session: load or create a session. + :method _set_session_settings: setting general session settings for the instagram api. + :method _validate_session_settings: checking the correctness between the session settings and the configuration settings. + :method exceptions_handler: decorator for handling exceptions in the Downloader class. + :method login: authentication in instagram api. + :method get_post_content: getting the content of a post. + + Examples: + >>> configuration = { + ... 'username': 'my_username', + ... 'password': 'my_password', + ... 'session-file': 'data/session.json', + ... 'delay-requests': 1, + ... '2fa-enabled': False, + ... '2fa-seed': 'my_seed_secret', + ... 'locale': 'en_US', + ... 'country-code': '1', + ... 'country': 'US', + ... 'timezone-offset': 10800, + ... 'proxy-dsn': 'http://localhost:8080' + ... 'request-timeout': 10, + ... 'device-settings': { + ... 'app_version': '269.0.0.18.75', 'version_code': '314665256', + ... 'manufacturer': 'OnePlus', 'model': '6T Dev', 'device': 'devitron', 'cpu': 'qcom', 'dpi': '480dpi', 'resolution': '1080x1920', + ... 'android_release': '8.0.0', 'android_version': '26' + ... } + ... } + >>> vault = Vault() + >>> downloader = Downloader(configuration, vault) + >>> status = downloader.get_post_content('shortcode') """ def __init__( self, @@ -38,36 +82,22 @@ def __init__( :param 2fa-seed (str): seed for two-factor authentication (secret key). :param locale (str): locale for requests. :param country-code (str): country code for requests. + :param country (str): country for requests. :param timezone-offset (int): timezone offset for requests. - :param user-agent (str): user agent for requests. :param proxy-dsn (str): proxy dsn for requests. :param request-timeout (int): request timeout for requests. + :device-settings (dict): dictionary with device settings for requests. + :param app_version (str): application version. + :param version_code (str): version code. + :param manufacturer (str): manufacturer of the device. + :param model (str): model of the device. + :param device (str): device name. + :param cpu (str): cpu name. + :param dpi (str): dpi resolution. + :param resolution (str): screen resolution. + :param android_release (str): android release version. + :param android_version (str): android version. :param vault (object): instance of vault for reading configuration downloader-api. - - Returns: - None - - Attributes: - :attribute configuration (dict): dictionary with configuration parameters for instagram api communication. - :attribute client (object): instance of the instagram api client. - - Examples: - >>> configuration = { - ... 'username': 'my_username', - ... 'password': 'my_password', - ... 'session-file': 'data/session.json', - ... 'delay-requests': 1, - ... '2fa-enabled': False, - ... '2fa-seed': 'my_seed_secret', - ... 'locale': 'en_US', - ... 'country-code': '1', - ... 'timezone-offset': 10800, - ... 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', - ... 'proxy-dsn': 'http://localhost:8080' - ... 'request-timeout': 10 - ... } - >>> vault = Vault() - >>> downloader = Downloader(configuration, vault) """ if not vault: raise WrongVaultInstance("Wrong vault instance, you must pass the vault instance to the class argument.") @@ -85,15 +115,23 @@ def __init__( log.info('[Downloader]: Creating a new instance...') self.client = Client() - log.info('[Downloader]: Configuring client settings...') + log.info('[Downloader]: Setting up the client configuration...') self.client.delay_range = [1, int(self.configuration['delay-requests'])] self.client.request_timeout = int(self.configuration['request-timeout']) - self.client.set_locale(locale=self.configuration['locale']) - self.client.set_country_code(country_code=int(self.configuration['country-code'])) - self.client.set_timezone_offset(seconds=int(self.configuration['timezone-offset'])) - self.client.set_user_agent(user_agent=self.configuration['user-agent']) self.client.set_proxy(dsn=self.configuration.get('proxy-dsn', None)) - log.info('[Downloader]: Client settings: %s', self.client.get_settings()) + self.general_settings_list = [ + 'locale', 'country_code', 'country', 'timezone_offset' + ] + self.device_settings_list = [ + 'app_version', 'version_code', 'manufacturer', 'model', 'device', 'cpu', 'dpi', 'resolution', 'android_release', 'android_version' + ] + self.download_methods = { + (1, 'any'): self.client.photo_download, + (2, 'feed'): self.client.video_download, + (2, 'clips'): self.client.clip_download, + (2, 'igtv'): self.client.igtv_download, + (8, 'any'): self.client.album_download + } auth_status = self.login() if auth_status == 'logged_in': @@ -101,14 +139,107 @@ def __init__( else: raise FailedAuthInstagram("Failed to authenticate the Instaloader instance.") - self.download_methods = { - (1, 'any'): self.client.photo_download, - (2, 'feed'): self.client.video_download, - (2, 'clips'): self.client.clip_download, - (2, 'igtv'): self.client.igtv_download, - (8, 'any'): self.client.album_download + def _get_login_args(self) -> dict: + """Get login arguments for the Instagram API""" + if self.configuration['2fa-enabled']: + totp_code = self.client.totp_generate_code(seed=self.configuration['2fa-seed']) + log.info('[Downloader]: Two-factor authentication is enabled. TOTP code: %s', totp_code) + return { + 'username': self.configuration['username'], + 'password': self.configuration['password'], + 'verification_code': totp_code + } + return { + 'username': self.configuration['username'], + 'password': self.configuration['password'] } + def _create_new_session(self, login_args: dict) -> None: + """Create a new session file for the Instagram API""" + self._set_session_settings() + self.client.login(**login_args) + self.client.dump_settings(self.configuration['session-file']) + log.info('[Downloader]: The new session file was created successfully: %s', self.configuration['session-file']) + + def _handle_relogin(self, login_args: dict) -> None: + """Handle re-authentication in the Instagram API""" + log.info('[Downloader]: Authentication with the clearing of the session...') + old_uuids = self.client.get_settings().get("uuids", {}) + self.client.set_settings({}) + self.client.set_uuids(old_uuids) + self._create_new_session(login_args) + + def _load_session(self, login_args: dict) -> None: + """Load or create a session.""" + log.info('[Downloader]: Authentication with the existing session...') + session_file = self.configuration['session-file'] + if os.path.exists(session_file): + self.client.load_settings(session_file) + # Temporarily fix for country, because it is not working in set_settings + self.client.set_country(country=self.configuration['country']) + if not self._validate_session_settings(): + self._create_new_session(login_args) + else: + self._create_new_session(login_args) + + def _set_session_settings(self) -> None: + """ + The method for setting general session settings for the Instagram API, such as + - locale + - country code + - country + - timezone offset + - device settings + - user agent + """ + log.info('[Downloader]: Extracting device settings...') + device_settings = json.loads(self.configuration['device-settings']) + if not all(item in device_settings.keys() for item in self.device_settings_list): + raise ValueError("Incorrect device settings in the configuration. Please check the configuration in the Vault.") + + # Extract other settings except device settings + log.info('[Downloader]: Extracting other settings...') + other_settings = {item: None for item in self.general_settings_list} + for item in other_settings.keys(): + other_settings[item] = self.configuration[item.replace('_', '-')] + + log.debug('[Downloader]: Retrieved settings: %s', {**other_settings, 'device_settings': device_settings}) + + # Apply all session settings + self.client.set_settings(settings={**other_settings, 'device_settings': device_settings}) + # Temporarily fix for country + # Country in set_settings is not working + self.client.set_country(country=other_settings['country']) + self.client.set_user_agent() + log.info('[Downloader]: General session settings have been successfully set: %s', self.client.get_settings()) + + def _validate_session_settings(self) -> bool: + """ + The method for checking the correctness between the session settings and the configuration settings. + + Returns: + (bool) True if the session settings are equal to the configuration settings, otherwise False. + """ + log.info('[Downloader]: Checking the difference between the session settings and the configuration settings...') + session_settings = self.client.get_settings() + for item in self.general_settings_list: + if str(session_settings[item]) != str(self.configuration[item.replace('_', '-')]): + log.info( + '[Downloader]: The session key value are not equal to the expected value: %s != %s. Session will be reset', + session_settings[item], self.configuration[item.replace('_', '-')] + ) + return False + device_settings = self.client.get_settings()['device_settings'] + for item in self.device_settings_list: + if str(device_settings[item]) != str(json.loads(self.configuration['device-settings'])[item]): + log.info( + '[Downloader]: The session key value are not equal to the expected value: %s != %s. Session will be reset', + device_settings[item], json.loads(self.configuration['device-settings'])[item] + ) + return False + log.info('[Downloader]: The session settings are equal to the expected settings.') + return True + @staticmethod def exceptions_handler(method) -> None: """ @@ -118,18 +249,23 @@ def exceptions_handler(method) -> None: :param method (function): method to be wrapped. """ def wrapper(self, *args, **kwargs): + random_shift = random.randint(600, 7200) try: return method(self, *args, **kwargs) except LoginRequired: - log.error('[Downloader]: Instagram API login required. Re-authentication...') + log.error('[Downloader]: Instagram API login required. Re-authenticate after %s minutes', round(random_shift/60)) + time.sleep(random_shift) + log.info('[Downloader]: Re-authenticate after timeout due to login required') self.login(method='relogin') except ChallengeRequired: - log.error('[Downloader]: Instagram API requires challenge. Need manually pass in browser. Retry after 1 hour') - time.sleep(3600) + log.error('[Downloader]: Instagram API requires challenge in browser. Retry after %s minutes', round(random_shift/60)) + time.sleep(random_shift) + log.info('[Downloader]: Re-authenticate after timeout due to challenge required') self.login() except PleaseWaitFewMinutes: - log.error('[Downloader]: Device or IP address has been restricted. Just wait a one hour and try again') - time.sleep(3600) + log.error('[Downloader]: Device or IP address has been restricted. Just wait a %s minutes and retry', round(random_shift/60)) + time.sleep(random_shift) + log.info('[Downloader]: Retry after timeout due to restriction') self.login(method='relogin') except (ReadTimeoutError, RequestsConnectionError, ClientRequestTimeout): log.error('[Downloader]: Timeout error downloading post content. Retry after 1 minute') @@ -145,8 +281,8 @@ def login(self, method: str = 'session') -> str | None: Args: :param method (str): the type of authentication in the Instagram API. Default: 'session'. possible values: - 'session' - authentication by existing session file. Or create session file for existing device. - 'relogin' - authentication as an existing device. This will create a new session file and clear the old attributes. + 'session' - authentication by existing session file or create session file for existing device. + 'relogin' - authentication as an existing device. Creates a new session file and clears the old attributes. Returns: (str) logged_in @@ -155,37 +291,14 @@ def login(self, method: str = 'session') -> str | None: """ log.info('[Downloader]: Authentication in the Instagram API with type: %s', method) - # 2FA authentication settings - if self.configuration['2fa-enabled']: - totp_code = self.client.totp_generate_code(seed=self.configuration['2fa-seed']) - log.info('[Downloader]: Two-factor authentication is enabled. TOTP code: %s', totp_code) - login_args = { - 'username': self.configuration['username'], - 'password': self.configuration['password'], - 'verification_code': totp_code - } - else: - login_args = { - 'username': self.configuration['username'], - 'password': self.configuration['password'] - } + # Generate login arguments + login_args = self._get_login_args() - # Login to the Instagram API - if method == 'session' and os.path.exists(self.configuration['session-file']): - log.info('[Downloader]: Loading session file with creation date %s', time.ctime(os.path.getctime(self.configuration['session-file']))) - self.client.load_settings(self.configuration['session-file']) - self.client.login(**login_args) - elif method == 'relogin': - log.info('[Downloader]: Relogin to the Instagram API...') - old_session = self.client.get_settings() - self.client.set_settings({}) - self.client.set_uuids(old_session["uuids"]) - self.client.login(**login_args) - self.client.dump_settings(self.configuration['session-file']) + # Handle authentication method + if method == 'relogin': + self._handle_relogin(login_args) else: - log.info('[Downloader]: Creating a new session file...') - self.client.login(**login_args) - self.client.dump_settings(self.configuration['session-file']) + self._load_session(login_args) # Check the status of the authentication log.info('[Downloader]: Checking the status of the authentication...') @@ -252,8 +365,8 @@ def get_post_content(self, shortcode: str = None, error_count: int = 0) -> dict 'status': status if status else 'failed' } - except (MediaUnavailable, MediaNotFound) as error: - log.warning('[Downloader]: Post %s not found, perhaps it was deleted. Message will be marked as processed: %s', shortcode, error) + except (MediaUnavailable, MediaNotFound): + log.warning('[Downloader]: Post %s not found, perhaps it was deleted. Message will be marked as processed', shortcode) response = { 'post': shortcode, 'owner': 'undefined',