diff --git a/.flake8 b/.flake8 index 8b6524be5..1ee19e57c 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] -max-line-length=100 +max-line-length=150 exclude = .git,__pycache__ extend-ignore = \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index dd84ea782..62c651738 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,38 +1,25 @@ --- name: Bug report -about: Create a report to help us improve -title: '' -labels: '' -assignees: '' +about: Create a bug report to help us improve +title: 'Bug: ' +labels: 'bug' +assignees: 'obervinov' --- - **Describe the bug** -A clear and concise description of what the bug is. +Not working functionality +Error message: -**To Reproduce** -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error +**Additional context** +Information about where the bug is happening: +1. Functionality 1 in module 1 not working as expected +2. Functionality 2 in module 2 not working as expected -**Expected behavior** -A clear and concise description of what you expected to happen. +**Possible correction** +If you have a possible solution to the bug, please describe it. **Screenshots** If applicable, add screenshots to help explain your problem. -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] - -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - -**Additional context** -Add any other context about the problem here. +**Check list after fixing the bug** +- [ ] Fix 1 in module 1 tested and working diff --git a/.github/ISSUE_TEMPLATE/custom.md b/.github/ISSUE_TEMPLATE/custom.md index 48d5f81fa..ceb63b568 100644 --- a/.github/ISSUE_TEMPLATE/custom.md +++ b/.github/ISSUE_TEMPLATE/custom.md @@ -1,9 +1,9 @@ --- name: Custom issue template about: Describe this issue template's purpose here. -title: '' -labels: '' -assignees: '' +title: 'Custom issue: ' +labels: 'help wanted' +assignees: 'obervinov' --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index bbcbbe7d6..1431e6d3f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,9 +1,9 @@ --- name: Feature request about: Suggest an idea for this project -title: '' -labels: '' -assignees: '' +title: 'Feature request: ' +labels: 'enhancement' +assignees: 'obervinov' --- diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0f17753a5..51bd89658 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,7 @@ updates: directory: "/" schedule: interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/.github/workflows" + schedule: + interval: "weekly" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 0d441130c..370adb8cb 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,16 +1,19 @@ -# PR-1: NAME -**Full Changelog**: https://github.com/obervinov/pyinstabot-downloader/compare/1...2 by @obervinov in https://github.com/obervinov/pyinstabot-downloader/pull/1 -## VERSION - YYYY-MM-DD +--- +name: 'Pull request' +about: Create a release to help us improve +title: 'Release : ' +labels: '' +assignees: 'obervinov' + +--- +## - ### What's Changed +**Full Changelog**: https://github.com/obervinov/pyinstabot-downloader/compare/1...2 by @obervinov in https://github.com/obervinov/pyinstabot-downloader/pull/1 #### πŸ› Bug Fixes * -* #### πŸ“š Documentation * -* #### πŸ’₯ Breaking Changes * -* #### πŸš€ Features -* -* +* \ No newline at end of file diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 000000000..a3e33055d --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,30 @@ +--- +name: PR + +on: + push: + branches: + - '*' + - '*/*' + - '**' + - '!main' + +jobs: + changelog: + uses: obervinov/_templates/.github/workflows/changelog.yaml@v1.2.2 + + pylint: + uses: obervinov/_templates/.github/workflows/pylint.yaml@v1.2.2 + + pytest: + uses: obervinov/_templates/.github/workflows/pytest-with-vault.yaml@v1.2.2 + + pyproject: + uses: obervinov/_templates/.github/workflows/pyproject.yaml@v1.2.2 + + pr: + uses: obervinov/_templates/.github/workflows/pr.yaml@v1.2.2 + + build-pr-image: + uses: obervinov/_templates/.github/workflows/docker.yaml@v1.2.2 + needs: [changelog, pylint, pytest, pyproject] diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..4a1af49ba --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,34 @@ +--- +name: Release + +on: + pull_request: + branches: + - main + types: + - closed + +jobs: + changelog: + uses: obervinov/_templates/.github/workflows/changelog.yaml@v1.2.2 + + pylint: + uses: obervinov/_templates/.github/workflows/pylint.yaml@v1.2.2 + + pytest: + uses: obervinov/_templates/.github/workflows/pytest-with-vault.yaml@v1.2.2 + + pyproject: + uses: obervinov/_templates/.github/workflows/pyproject.yaml@v1.2.2 + + build-release-image: + uses: obervinov/_templates/.github/workflows/docker.yaml@v1.2.2 + needs: [changelog, pylint, pytest, pyproject] + + create-release: + uses: obervinov/_templates/.github/workflows/release.yaml@v1.2.2 + needs: [changelog, pylint, pytest, pyproject, build-release-image] + + # milestone: + # uses: obervinov/_templates/.github/workflows/milestone.yaml@v1.2.2 + # needs: [create-release, build-release-image] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index d0d063d4e..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -name: Create GitHub Release - -# yamllint disable-line rule:truthy -on: - push: - branches: [main] - -jobs: - create-release: - uses: obervinov/_templates/.github/workflows/create.release.yml@v1.0.5 diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml deleted file mode 100644 index 9e29b7f33..000000000 --- a/.github/workflows/workflow.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -name: Test and Build release - -# yamllint disable-line rule:truthy -on: - push: - branches: - - '*' - - '*/*' - - '**' - -env: - # yamllint disable-line rule:line-length - PROJECT_DESCRIPTION: "This project is a telegram bot that allows you to create backups of content from your Instagram profile to Dropbox or Mega clouds, as well as in the local file system." - - -jobs: - pylint: - uses: obervinov/_templates/.github/workflows/test.pylint.yml@v1.0.5 - verify-changelog: - uses: obervinov/_templates/.github/workflows/verify.changelog.yml@v1.0.5 - build-release: - uses: obervinov/_templates/.github/workflows/build.docker.yml@v1.0.5 - needs: [pylint, verify-changelog] diff --git a/.gitignore b/.gitignore index 7b7d95216..ead8ba926 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -.vscode/* +.vscode .DS_Store *.tmp vault/data/ diff --git a/.pylintrc b/.pylintrc index d9f4215c4..f233cb7ef 100644 --- a/.pylintrc +++ b/.pylintrc @@ -330,7 +330,7 @@ indent-after-paren=4 indent-string=' ' # Maximum number of characters on a single line. -max-line-length=100 +max-line-length=150 # Maximum number of lines in a module. max-module-lines=1000 diff --git a/CHANGELOG.md b/CHANGELOG.md index 248776992..b39ad96dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,28 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## v2.1.0 - 2024-05-28 +### What's Changed +**Full Changelog**: https://github.com/obervinov/pyinstabot-downloader/compare/v2.0.0...v2.1.0 by @obervinov in https://github.com/obervinov/pyinstabot-downloader/pull/37 +#### πŸ› Bug Fixes +* [GitHub Actions: delete redundant jobs in the main branch b to make workflow more logical](https://github.com/obervinov/pyinstabot-downloader/issues/23) +#### πŸ“š Documentation +* [Update documentation release/v2.1.0](https://github.com/obervinov/pyinstabot-downloader/issues/26) +* Update repository issues template +#### πŸ’₯ Breaking Changes +* Add `PostgreSQL` support to the bot stack (instead of `Vault`). All bot data except `configurations` and `user data` is now stored in the database. +* Remove outdated method for processing full account data per link of `user profile`. +* Move all configuration of components form `environment variables` to the `Vault`. +#### πŸš€ Features +* [Change the structure of the secret configurations in Vault](https://github.com/obervinov/pyinstabot-downloader/issues/54) +* [A new concept for processing input messages](https://github.com/obervinov/pyinstabot-downloader/issues/32) +* [Add PostgreSQL support to the bot stack](https://github.com/obervinov/pyinstabot-downloader/issues/30) +* [Add support env files in docker compose](https://github.com/obervinov/pyinstabot-downloader/issues/28) +* [Extend users attributes and add automatic rate limit control](https://github.com/obervinov/pyinstabot-downloader/issues/14) +* [Add the method for processing multiline messages](https://github.com/obervinov/pyinstabot-downloader/issues/20) +* [Rollback to old environment variable names PB_VAULT_APPROLE_ID PB_VAULT_APPROLE_SECRETID](https://github.com/obervinov/pyinstabot-downloader/issues/27) +* [Move parameter `session` to the vault configuration](https://github.com/obervinov/pyinstabot-downloader/issues/24) + ## v2.0.0 - 2023-09-16 ### What's Changed @@ -12,21 +34,21 @@ In this release, the approach with issue and github project was implemented alre - issue contains a list of mixed issues - these issue are duplicated in the readme sections #### πŸ› Bug Fixes -* (Update dependencies: 2023.06.13)[https://github.com/obervinov/pyinstabot-downloader/issues/6] -* (Update the project code and fix bugs)[https://github.com/obervinov/pyinstabot-downloader/issues/13] -* (Redundant login and password reading from vault)[https://github.com/obervinov/pyinstabot-downloader/issues/16] -* (The status is "None" when an exception occurred when uploading to mega, and the retry method)[https://github.com/obervinov/pyinstabot-downloader/issues/15] +* [Update dependencies: 2023.06.13](https://github.com/obervinov/pyinstabot-downloader/issues/6) +* [Update the project code and fix bugs](https://github.com/obervinov/pyinstabot-downloader/issues/13) +* [Redundant login and password reading from vault](https://github.com/obervinov/pyinstabot-downloader/issues/16) +* [The status is "None" when an exception occurred when uploading to mega, and the retry method](https://github.com/obervinov/pyinstabot-downloader/issues/15) #### πŸ“š Documentation -* (Update project repository: 2023.06.13)[https://github.com/obervinov/pyinstabot-downloader/issues/8] +* [Update project repository: 2023.06.13](https://github.com/obervinov/pyinstabot-downloader/issues/8) #### πŸ’₯ Breaking Changes -* (Update dependencies: 2023.06.13)[https://github.com/obervinov/pyinstabot-downloader/issues/6] -* (Update the project code and fix bugs)[https://github.com/obervinov/pyinstabot-downloader/issues/13] +* [Update dependencies: 2023.06.13](https://github.com/obervinov/pyinstabot-downloader/issues/6) +* [Update the project code and fix bugs](https://github.com/obervinov/pyinstabot-downloader/issues/13) #### πŸš€ Features -* (Update dependencies: 2023.06.13)[https://github.com/obervinov/pyinstabot-downloader/issues/6] -* (Added the support GitHub Actions)[https://github.com/obervinov/pyinstabot-downloader/issues/10] -* (Update project repository: 2023.06.13)[https://github.com/obervinov/pyinstabot-downloader/issues/8] -* (Update the project code and fix bugs)[https://github.com/obervinov/pyinstabot-downloader/issues/13] -* (Check the download history for the specified post)[https://github.com/obervinov/pyinstabot-downloader/issues/17] +* [Update dependencies: 2023.06.13](https://github.com/obervinov/pyinstabot-downloader/issues/6) +* [Added the support GitHub Actions](https://github.com/obervinov/pyinstabot-downloader/issues/10) +* [Update project repository: 2023.06.13](https://github.com/obervinov/pyinstabot-downloader/issues/8) +* [Update the project code and fix bugs](https://github.com/obervinov/pyinstabot-downloader/issues/13) +* [Check the download history for the specified post](https://github.com/obervinov/pyinstabot-downloader/issues/17) ## v1.0.1 - 2022-11-06 diff --git a/Dockerfile b/Dockerfile index c82f05cd6..7a3c52794 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,42 +1,56 @@ -FROM python:3.10.7-alpine3.16 +FROM python:3.9.15-alpine3.16 ### External argumetns ### -ARG PROJECT_NAME ARG PROJECT_DESCRIPTION +ARG PROJECT_NAME ARG PROJECT_VERSION ### Labels ### LABEL org.opencontainers.image.source https://github.com/obervinov/${PROJECT_NAME} -LABEL org.opencontainers.image.description $PROJECT_DESCRIPTION -LABEL org.opencontainers.image.title "Telegram bot: pyinstabot-downloader" -LABEL org.opencontainers.image.version $PROJECT_VERSION +LABEL org.opencontainers.image.description ${PROJECT_DESCRIPTION} +LABEL org.opencontainers.image.version ${PROJECT_VERSION} LABEL org.opencontainers.image.authors github.obervinov@proton.me -LABEL org.opencontainers.image.licenses https://github.com/obervinov/pyinstabot-downloader/blob/$PROJECT_VERSION/LICENSE -LABEL org.opencontainers.image.documentation https://github.com/obervinov/pyinstabot-downloader/blob/$PROJECT_VERSION/README.md -LABEL org.opencontainers.image.source https://github.com/obervinov/pyinstabot-downloader/blob/$PROJECT_VERSION +LABEL org.opencontainers.image.licenses https://github.com/obervinov/${PROJECT_NAME}/blob/${PROJECT_VERSION}/LICENSE +LABEL org.opencontainers.image.documentation https://github.com/obervinov/${PROJECT_NAME}/blob/${PROJECT_VERSION}/README.md +LABEL org.opencontainers.image.source https://github.com/obervinov/${PROJECT_NAME}/blob/${PROJECT_VERSION} ### Environment variables ### -ENV PATH=/home/${PROJECT_NAME}/.local/bin:$PATH +ENV PIP_NO_CACHE_DIR=off +ENV PIP_DISABLE_PIP_VERSION_CHECK=on +ENV POETRY_VIRTUALENVS_IN_PROJECT=true +ENV POETRY_NO_INTERACTION=1 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV VENV_PATH=/home/${PROJECT_NAME}/app/.venv +ENV PATH=/home/${PROJECT_NAME}/.local/bin:$VENV_PATH/bin:$PATH + -### Preparing user and dirs ### +### Preparing user and directories ### RUN adduser -D -h /home/${PROJECT_NAME} -s /bin/sh ${PROJECT_NAME} && \ mkdir -p /home/${PROJECT_NAME} && \ mkdir -p /home/${PROJECT_NAME}/app && \ - mkdir -p /home/${PROJECT_NAME}/tmp && \ + mkdir -p /home/${PROJECT_NAME}/tmp && \ chown ${PROJECT_NAME}. /home/${PROJECT_NAME} -R -### Prepare git -RUN apk add git +### Prepare tools and fix vulnerabilities ### +RUN apk upgrade --no-cache && apk add --no-cache git curl ### Switching context ### USER ${PROJECT_NAME} WORKDIR /home/${PROJECT_NAME}/app ### Copy source code ### -COPY requirements.txt ./ -COPY src/ ./ - -### Installing a python dependeces - requirements.txt ### -RUN pip3 install -r requirements.txt - -CMD [ "python3", "bot.py" ] +COPY src/ src/ +COPY tests/ tests/ +COPY pyproject.toml . +COPY poetry.lock . +COPY *.md ./ +COPY LICENSE ./ + +### Installing poetry and python dependeces ### +RUN curl -sSL https://install.python-poetry.org | python - +RUN poetry install +ENV PYTHONPATH=/home/${PROJECT_NAME}/app/src:/home/${PROJECT_NAME}/app/.venv/lib/python3.10/site-packages + +### Entrypoint ### +CMD [ "python3", "src/bot.py" ] diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 6841bed4e..000000000 --- a/Pipfile +++ /dev/null @@ -1,2 +0,0 @@ -[requires] -python_version = "3.10.7" diff --git a/README.md b/README.md index 4defd9533..97e1ea020 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Pyinstabot-downloader -[![Release](https://github.com/obervinov/pyinstabot-downloader/actions/workflows/release.yml/badge.svg)](https://github.com/obervinov/pyinstabot-downloader/actions/workflows/release.yml) +[![Release](https://github.com/obervinov/pyinstabot-downloader/actions/workflows/release.yaml/badge.svg)](https://github.com/obervinov/pyinstabot-downloader/actions/workflows/release.yaml) [![CodeQL](https://github.com/obervinov/pyinstabot-downloader/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/obervinov/pyinstabot-downloader/actions/workflows/github-code-scanning/codeql) -[![Test and Build image](https://github.com/obervinov/pyinstabot-downloader/actions/workflows/workflow.yml/badge.svg?branch=main&event=pull_request)](https://github.com/obervinov/pyinstabot-downloader/actions/workflows/workflow.yml) +[![Test and Build image](https://github.com/obervinov/pyinstabot-downloader/actions/workflows/pr.yaml/badge.svg?branch=main&event=pull_request)](https://github.com/obervinov/pyinstabot-downloader/actions/workflows/pr.yaml) ![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/obervinov/pyinstabot-downloader?style=for-the-badge) ![GitHub last commit](https://img.shields.io/github/last-commit/obervinov/pyinstabot-downloader?style=for-the-badge) @@ -11,140 +11,171 @@ [![Python version](https://img.shields.io/badge/python-3.10.7-blue.svg?style=for-the-badge)](https://www.python.org/downloads/release/python-3107/) [![License](https://img.shields.io/badge/license-MIT-green.svg?style=for-the-badge)](https://opensource.org/licenses/MIT) - -## GitHub Actions -| Name | Version | -| ------------------------ | ----------- | -| GitHub Actions Templates | [v1.0.5](https://github.com/obervinov/_templates/tree/v1.0.5) | +## Table of contents +- [About this project](#-about-this-project) +- [Project architecture](#-project-architecture) +- [Requirements](#-requirements) +- [Environment variables](#-environment-variables) +- [Prepare and configure environment](#-prepare-and-configure-environment) + - [Target storage of the content](#target-storage-of-the-content) + - [Bot configuration source and supported parameters](#bot-configuration-source-and-supported-parameters) + - [Bot persistent data storage](#bot-persistent-data-storage) +- [How to run project](#-how-to-run-project) -## About this project +## About this project This project is a telegram bot that allows you to create backups of content from your Instagram profile to Dropbox or Mega clouds, as well as in the local file system. - -Main functions: -- a backup copy of all posts from the profile -- a backup copy of a specific post by link -- the ability to backup to the mega or dropbox cloud - -Review:

- +

+ +**Main functions** +- a backup copy of a `specific post` by link +- a backup copy of `list of posts` by links +- the ability to backup to the `Mega` or `Dropbox` clouds + +**Preview of the bot in action**

- bot-preview-post - bot-preview-account + preview-one-post + preview-list-posts

-## Project architecture -Code logic -![Diagram](doc/diagram-logic.png) +## Project architecture +**Users flow** +![Diagram](doc/diagram-flow.png) -Code dependencies -![Diagram](doc/diagram-code.png) +**Code structure** +![Diagram](doc/diagram-structure.png) +
-## Repository map -```sh -. -β”œβ”€β”€ CHANGELOG.md -β”œβ”€β”€ Dockerfile -β”œβ”€β”€ LICENSE -β”œβ”€β”€ Pipfile -β”œβ”€β”€ README.md -β”œβ”€β”€ SECURITY.md -β”œβ”€β”€ doc -β”‚ β”œβ”€β”€ bot-preview-account.png -β”‚ β”œβ”€β”€ bot-preview-post.png -β”‚ β”œβ”€β”€ bot-preview.gif -β”‚ β”œβ”€β”€ diagram-code.png -β”‚ β”œβ”€β”€ diagram-logic.png -β”‚ └── pyinstabot-downloader.drawio -β”œβ”€β”€ docker-compose.dev.yml -β”œβ”€β”€ docker-compose.prerelease.yml -β”œβ”€β”€ docker-compose.release.yml -β”œβ”€β”€ requirements.txt -β”œβ”€β”€ src -β”‚ β”œβ”€β”€ bot.py -β”‚ β”œβ”€β”€ configs -β”‚ β”‚ └── messages.json -β”‚ β”œβ”€β”€ constants.py -β”‚ └── modules -β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ downloader.py -β”‚ └── uploader.py -└── vault - └── policy.hcl - -6 directories, 23 files -``` +## Requirements +- Vault server - [a storage of secrets for bot with kv v2 engine](https://developer.hashicorp.com/vault/docs/secrets/kv/kv-v2) +- Dropbox [api token](https://dropbox.tech/developers/generate-an-access-token-for-your-own-account) or Mega.nz [account](https://mega.nz) +- Telegram bot api token - [instructions for creating bot and getting a token of api](https://learn.microsoft.com/en-us/azure/bot-service/bot-service-channel-connect-telegram?view=azure-bot-service-4.0) +- Instagram username/password - [login and password from the instagram account, it is advisable to create a new account](https://www.instagram.com/accounts/emailsignup/) +- Postgresql - [a storage of project persistent data](https://www.postgresql.org/download/) +
-## Requirements -- Vault server - [a storage of secrets for bot with kv v2 engine](https://developer.hashicorp.com/vault/docs/secrets/kv/kv-v2) -- Dropbox [api token](https://dropbox.tech/developers/generate-an-access-token-for-your-own-account) or Mega.nz [account](https://mega.nz) -- Telegram bot api token - [instructions for creating bot and getting a token of api](https://learn.microsoft.com/en-us/azure/bot-service/bot-service-channel-connect-telegram?view=azure-bot-service-4.0) -- Instagram username/password - [login and password from the instagram account, it is advisable to create a new account](https://www.instagram.com/accounts/emailsignup/) - - -## Environment variables +## Environment variables | Variable | Description | Default value | | ------------- | ------------- | ------------- | +| `PROJECT_ENVIRONMENT` | The environment in which the project is running (`dev`, `prod`) | `dev` | | `LOGGER_LEVEL` | [The logging level of the logging module](https://docs.python.org/3/library/logging.html#logging-levels) | `INFO` | | `BOT_NAME` | The name of the bot, used to determine the unique mount point in the vault | `pyinstabot-downloader` | | `MESSAGES_CONFIG` | The path to the message template file | `src/configs/messages.json` | -| `STORAGE_TYPE` | Type of target storage for saving uploaded content from instagram (`dropbox`, `mega` or `local`) | `mega` | -| `STORAGE_EXCLUDE_TYPE`| Types of files that you want to exclude from uploading to the cloud | `.txt` | -| `TEMPORARY_DIR` | Temporary directory for saving uploaded content from instagram | `tmp/` | -| `INSTAGRAM_SESSION` | The path for storing the file with the instagram session | `.session` | -| `INSTAGRAM_USERAGENT` | [User Agent to use for HTTP requests. Per default, Instaloader pretends being Chrome/92 on Linux](https://instaloader.github.io/cli-options.html#cmdoption-user-agent) | `None` | | `VAULT_ADDR` | The address at which the vault server will be available to the bot | `None` | | `VAULT_APPROLE_ID` | [Approle id created during vault setup](https://developer.hashicorp.com/vault/docs/auth/approle) | `None` | | `VAULT_APPROLE_SECRETID` | [Approle secret id created during vault setup](https://developer.hashicorp.com/vault/docs/auth/approle) | `None` | +
+## Prepare and configure environment -## Prepare ### Target storage of the content -#### If dropbox is going to be used as the target storage, you need to: +#### If dropbox is going to be used as the target storage, you need to - [Create a dropbox account](https://www.dropbox.com/register) - Generate an application token according to the instructions [here](https://dropbox.tech/developers/generate-an-access-token-for-your-own-account) and [here](https://developers.dropbox.com/ru-ru/oauth-guide) - [More documentation](https://www.dropbox.com/developers/documentation/python#overview) -#### If mega is going to be used as the target storage, you need to: +#### If mega is going to be used as the target storage, you need to - [Create a mega account](https://mega.nz/register) -- Don't turn on 2fa (because the module mega.py can't work with 2fa https://github.com/odwyersoftware/mega.py/issues/19) - -#### If the local file system will be used as the target storage: -- Set to environment variable `TEMPORARY_DIR` the desired local path for saving content (ex. `/opt/backup/instagram`) - -Such a strange variable name comes from the logic of the bot. The `TEMPORARY_DIR` variable is used as an intermediate buffer between the stage of downloading content from Instagram and then uploading it to the target storage. - -If the target storage is dropbox or mega, then files from the temporary directory are simply deleted after successful upload to the cloud. - -If the target storage is a local file system, then any further steps to process the files will be unnecessary. The process just immediately uploads the content from Instagram to the target directory (temporary directory), after which nothing happens to the files. - -### Storing project configuration and project history - All persistent project data is stored in **Vault**: -- stores project configuration parameters -- keeps the history of already uploaded posts from instagram -- stores information about user authorization events -- stores attributes and user rights - -#### You can use an existing vault-server or launch a new one using docker-compose: +- Don't turn on `2fa`, because the library `mega.py` [can't work with 2fa](https://github.com/odwyersoftware/mega.py/issues/19) (it'll probably be fixed in https://github.com/obervinov/pyinstabot-downloader/issues/36) +
+ +### Bot configuration source and supported parameters + All bot configuration is stored in the `Vault Secrets`
+_except for the part of the configuration that configures the connection to `Vault` and external modules_
+- `configuration/database-`: database connection parameters (depends on the environment variable) + ```json + { + "database": "pyinstabot-downloader", + "host": "postgresql.example.com", + "password": "qwerty123", + "port": "5432", + "user": "user1" + } + ``` +
+- `configuration/downloader-api`: downloader module configuration (for downloading content from instagram) + ```json + { + "enabled": "True", + "fatal-status-codes": "[400, 401, 429, 500]", + "iphone-support": "False", + "login-method": "session", + "password": "qwerty123", + "session-base64": "string==", + "session-file": "/directory/downloader/session", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "username": "username1" + } + ``` + Clarification of non-obvious parameters + - `fatal-status-codes`: a list of status codes that are considered fatal and stop downloader module work + - `iphone-support`: if `True`, the downloader module will use the iphone user-agent + - `login-method`: the method of logging into the instagram account (`session`, `password`, `anonymous`) + - `session-base64`: the session file content in base64 format +
+- `configuration/telegram`: telegram bot configuration + ```json + { + "token": "123456:qwerty" + } + ``` +
+- `configuration/uploader-api`: uploader module configuration (for upload content to the target storage) + ```json + { + "destination-directory": "cloud-directory/", + "enabled": "True", + "exclude-types": "[\".txt\", \".tmp\"]", + "password": "qwerty123", + "source-directory": "data/", + "storage-type": "dropbox", + "username": "username1" + } + ``` + Clarification of non-obvious parameters + - `destination-directory`: the directory in the target storage where the content will be uploaded + - `exclude-types`: a list of file extensions that will be excluded from the upload (for example, `.txt` - text from the post) + - `source-directory`: the directory where the content will be stored before uploading (temporary directory) + - `storage-type`: the type of storage where the content will be uploaded (`dropbox`, `mega`) +
+- `configuration/users/`: user permissions configuration + ```json + { + "requests": "{\"requests_per_day\": 10, \"requests_per_hour\": 1, \"random_shift_minutes\": 60}", + "roles": "[\"post\", \"posts_list\"]", + "status": "allowed" + } + ``` + Clarification of non-obvious parameters + - `requests`: the number of requests that the user can make per day and per hour, as well as the random shift in minutes (scheduling of message processing from the queue works on the basis of this parameter) + - `roles`: list of roles that allow to use the corresponding functionality ([available roles](src/configs/constants.py#L19-L23)). + - `status`: allowed or denied user access to the bot + +#### You can use an existing vault-server or launch a new one using docker-compose - instructions for starting and configuring a new vault-server ```bash -docker-compose -f docker-compose.dev.yml up vault-server -d -pip3 install -r requirements.txt +docker-compose -f docker-compose.yml up vault-server -d +poetry install curl -L https://gist.githubusercontent.com/obervinov/9bd452fee681f0493da7fd0b2bfe1495/raw/bbc4aad0ed7be064e9876dde64ad8b26b185091b/setup_vault_server.py | python3 --url=http://localhost:8200 --name=pyinstabot-downloader --policy=vault/policy.hcl ``` - instructions for configuring an existing vault server ```bash -pip3 install -r requirements.txt +poetry install curl -L https://gist.githubusercontent.com/obervinov/9bd452fee681f0493da7fd0b2bfe1495/raw/bbc4aad0ed7be064e9876dde64ad8b26b185091b/setup_vault_server.py | python3 --url=http://localhost:8200 --name=pyinstabot-downloader --policy=vault/policy.hcl --token=hvs.123456qwerty ``` -`setup_vault_server.py` - This script performs a quick and convenient configuration of the vault-server for this bot project: `initial` initialization of vault-server, `unseal` vault-server, creating an isolated `mount point`, loading `policy.hcl`, creating an `approle`. + `setup_vault_server.py` - This script performs a quick and convenient configuration of the vault-server for this bot project + - `initial` initialization of vault-server (_if it is new vault-server_) + - `unseal` vault-server (_if it is new vault-server_) + - creating an isolated `mount point` + - loading `policy.hcl` + - creating an `approle` -All these actions can also be performed using the vault cli: +All these actions can also be performed using the official `vault` cli ```bash vault operator init vault operator unseal @@ -161,42 +192,36 @@ vault write auth/pyinstabot-downloader/role/pyinstabot-downloader \ mount_point="pyinstabot-downloader" \ secret_id_ttl=0 ``` - - -#### Required bot configuration parameters -```bash -vault kv put pyinstabot-downloader/configuration/dropbox token={dropbox_token} -vault kv put pyinstabot-downloader/configuration/telegram token={telegram_token} -vault kv put pyinstabot-downloader/configuration/permissions {your_telegram_userid}=allow -vault kv put pyinstabot-downloader/configuration/instagram username={username} password={password} -vault kv put pyinstabot-downloader/configuration/mega username={username} password={password} -``` - -## How to run with docker-compose +
+ +### Bot persistent data storage + Persistent data storage is implemented using `Postgresql`
+You can familiarize yourself with the +- data structure, tables and assignment of tables [here](src/configs/databases.json) +- migrations [here](src/migrations/) + +The database structure is created automatically when the bot starts. Bot checks the database structure and creates missing tables if necessary. +After checking the database structure, the bot executes the migrations in the order of their numbering.
+Required only database owner rights in `Vault` for the bot to create tables and execute migrations. + +**What data is stored in tables:** +- user request queue +- history of processed user requests +- information about users activity (requests and contacts) +- completed migrations +- messages sent by the bot (to update them) +
+ +## How to run project ```sh export VAULT_APPROLE_ID={change_me} export VAULT_APPROLE_SECRETID={change_me} export VAULT_ADDR={change_me} - -docker-compose -f docker-compose.dev.yml up -d -# or -docker-compose -f docker-compose.release.yml up -d +docker compose -f docker-compose.yml up -d ``` +
- -## How to run a bot locally without a docker -**You need an already running and configured vault to use the approle and kv v2 engine** -```sh -pip3 install -r requirements.txt - -export VAULT_APPROLE_ID={change_me} -export VAULT_APPROLE_SECRETID={change_me} -export VAULT_ADDR={change_me} -export BOT_NAME=pyinstabot-downloader -export LOGGER_LEVEL=INFO -export STORAGE_TYPE=mega -export INSTAGRAM_SESSION=/home/python/.config/instaloader/.session -export STORAGE_EXCLUDE_TYPE=".txt" - -python3 src/bot.py -``` +## GitHub Actions +| Name | Version | +| ------------------------ | ----------- | +| GitHub Actions Templates | [v1.2.2](https://github.com/obervinov/_templates/tree/v1.2.2) | diff --git a/doc/bot-preview-account.png b/doc/bot-preview-account.png deleted file mode 100644 index 0822b487f..000000000 Binary files a/doc/bot-preview-account.png and /dev/null differ diff --git a/doc/bot-preview-post.png b/doc/bot-preview-post.png deleted file mode 100644 index 9a764cb64..000000000 Binary files a/doc/bot-preview-post.png and /dev/null differ diff --git a/doc/bot-preview.gif b/doc/bot-preview.gif deleted file mode 100644 index 0217c9449..000000000 Binary files a/doc/bot-preview.gif and /dev/null differ diff --git a/doc/diagram-code.png b/doc/diagram-code.png deleted file mode 100644 index 15a2363b1..000000000 Binary files a/doc/diagram-code.png and /dev/null differ diff --git a/doc/diagram-flow.png b/doc/diagram-flow.png new file mode 100644 index 000000000..611b32e0a Binary files /dev/null and b/doc/diagram-flow.png differ diff --git a/doc/diagram-logic.png b/doc/diagram-logic.png deleted file mode 100644 index f5e27ed7f..000000000 Binary files a/doc/diagram-logic.png and /dev/null differ diff --git a/doc/diagram-structure.png b/doc/diagram-structure.png new file mode 100644 index 000000000..7b4178497 Binary files /dev/null and b/doc/diagram-structure.png differ diff --git a/doc/preview-list-posts.gif b/doc/preview-list-posts.gif new file mode 100644 index 000000000..aa02a9e8d Binary files /dev/null and b/doc/preview-list-posts.gif differ diff --git a/doc/preview-main.png b/doc/preview-main.png new file mode 100644 index 000000000..c25da1a1a Binary files /dev/null and b/doc/preview-main.png differ diff --git a/doc/preview-one-post.gif b/doc/preview-one-post.gif new file mode 100644 index 000000000..fd655676c Binary files /dev/null and b/doc/preview-one-post.gif differ diff --git a/doc/pyinstabot-downloader.drawio b/doc/pyinstabot-downloader.drawio index ce195ccc9..2faf67129 100644 --- a/doc/pyinstabot-downloader.drawio +++ b/doc/pyinstabot-downloader.drawio @@ -1,376 +1,401 @@ - + - + - - + + + + + + + + + + + + + + - - + + - + - - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + - + + + + + + + + + + + + + + + + - - + + - - - - - - - + + - - + + - - + + - + - - + + - - + + - - + + - + - - + + + + + + + + + + + + + + + - - + + - - + + + + + + + - - + + - - + + - - + + - - + + - + - - - - - + + - + - - + + - + - - + + - + - - + + - - + + + + + - - + + - + + + + + + + + + + - + + + + + - - - - + + - + - - + - - + + - - + + - - + + - - - - + + - - + + - - + + - - - - - - - - - + + - - - - + + - - + + - - + + - + - - + + + - - - - + + - + - - - + + - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - - + + - - + + + + + + - - + + - - + + + + + + + - + - - - - + - - + + - - + + - - + + - + - + + + + - + + + + - - - - + + - - - - + + - - + + diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml deleted file mode 100644 index 07379c44c..000000000 --- a/docker-compose.dev.yml +++ /dev/null @@ -1,45 +0,0 @@ -# manifest for the development process with the local version of the vault ---- -version: '3' -services: - vault-server: - image: vault - container_name: vault - command: ["server"] - environment: - # yamllint disable-line rule:line-length - - 'VAULT_LOCAL_CONFIG={"backend": {"file": {"path": "/vault/data"}}, "default_lease_ttl": "1h", "max_lease_ttl": "720h", "listener": {"tcp": {"address": "0.0.0.0:8200", "tls_disable": "1"}}' - - VAULT_API_ADDR=http://0.0.0.0:8200 - - VAULT_PORT=8200 - volumes: - - ./vault/data:/vault/data - ports: - - "0.0.0.0:8200:8200" - cap_add: - - IPC_LOCK - - pyinstabot-downloader: - build: - context: . - dockerfile: Dockerfile - args: - PROJECT_NAME: pyinstabot-downloader - # yamllint disable-line rule:line-length - PROJECT_DESCRIPTION: "This project is a telegram bot that allows you to backup content from your Instagram profile to the Dropbox/Mega cloud or to the local filesystem." - PROJECT_VERSION: v2.0.0 - container_name: pyinstabot-downloader - restart: always - environment: - - BOT_NAME=pyinstabot-downloader - - VAULT_APPROLE_ID=${PB_VAULT_APPROLE_ID} - - VAULT_APPROLE_SECRETID=${PB_VAULT_APPROLE_SECRETID} - - VAULT_ADDR=http://vault-server:8200 - - LOGGER_LEVEL=DEBUG - - STORAGE_TYPE=local - - INSTAGRAM_SESSION=${INSTAGRAM_SESSION} - - STORAGE_EXCLUDE_TYPE= - - MESSAGES_CONFIG=configs/messages.json - depends_on: - - vault-server - volumes: - - ../.config/instaloader:/home/python/.config/instaloader diff --git a/docker-compose.prerelease.yml b/docker-compose.prerelease.yml deleted file mode 100644 index a877ca66a..000000000 --- a/docker-compose.prerelease.yml +++ /dev/null @@ -1,21 +0,0 @@ -# manifest for launching and testing the pre-release version ---- -version: '3' -services: - pyinstabot-downloader: - image: ghcr.io/obervinov/pyinstabot-downloader:release-v2.0.0 - container_name: pyinstabot-downloader - restart: always - pull_policy: always - environment: - - BOT_NAME=pyinstabot-downloader - - VAULT_APPROLE_ID=${PB_VAULT_APPROLE_ID} - - VAULT_APPROLE_SECRETID=${PB_VAULT_APPROLE_SECRETID} - - VAULT_ADDR=${VAULT_ADDR} - - LOGGER_LEVEL=INFO - - STORAGE_TYPE=mega - - INSTAGRAM_SESSION=${INSTAGRAM_SESSION} - - STORAGE_EXCLUDE_TYPE=.txt - - MESSAGES_CONFIG=configs/messages.json - volumes: - - ../.config/instaloader:/home/python/.config/instaloader diff --git a/docker-compose.release.yml b/docker-compose.release.yml deleted file mode 100644 index 9fb2c0860..000000000 --- a/docker-compose.release.yml +++ /dev/null @@ -1,20 +0,0 @@ -# the main manifest for launching the release version of the project ---- -version: '3' -services: - pyinstabot-downloader: - image: ghcr.io/obervinov/pyinstabot-downloader:v2.0.0 - container_name: pyinstabot-downloader - restart: always - environment: - - BOT_NAME=pyinstabot-downloader - - VAULT_APPROLE_ID=${PB_VAULT_APPROLE_ID} - - VAULT_APPROLE_SECRETID=${PB_VAULT_APPROLE_SECRETID} - - VAULT_ADDR=${VAULT_ADDR} - - LOGGER_LEVEL=INFO - - STORAGE_TYPE=mega - - INSTAGRAM_SESSION=${INSTAGRAM_SESSION} - - STORAGE_EXCLUDE_TYPE=.txt - - MESSAGES_CONFIG=configs/messages.json - volumes: - - ../.config/instaloader:/home/python/.config/instaloader diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..8813e447e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,67 @@ +# manifest for the development process with the local version of the vault +--- +version: '3' +services: + vault-server: + image: hashicorp/vault:1.14.1 + container_name: vault + command: ["server"] + environment: + # yamllint disable-line rule:line-length + - 'VAULT_LOCAL_CONFIG={"backend": {"file": {"path": "/vault/data"}}, "default_lease_ttl": "1h", "max_lease_ttl": "720h", "listener": {"tcp": {"address": "0.0.0.0:8200", "tls_disable": "1"}}' + - VAULT_API_ADDR=http://0.0.0.0:8200 + - VAULT_PORT=8200 + ports: + - "0.0.0.0:8200:8200" + cap_add: + - IPC_LOCK + + postgres: + image: postgres:latest + container_name: postgres + environment: + POSTGRES_DB: pyinstabot-downloader + POSTGRES_USER: python + POSTGRES_PASSWORD: python + ports: + - "0.0.0.0:5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + + pgadmin: + image: dpage/pgadmin4:latest + container_name: pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: admin@email.com + PGADMIN_DEFAULT_PASSWORD: adminpassword + ports: + - "5050:80" + depends_on: + - postgres + + pyinstabot-downloader: + build: + context: . + dockerfile: Dockerfile + args: + PROJECT_NAME: pyinstabot-downloader + PROJECT_DESCRIPTION: "This project is a Telegram bot that allows you to backup post content from your Instagram profile to Dropbox or Mega clouds." + PROJECT_VERSION: 2.1.0 + container_name: pyinstabot-downloader + restart: always + environment: + - PROJECT_ENVIRONMENT=dev + - TELEGRAM_BOT_NAME=pyinstabot-downloader + - VAULT_APPROLE_ID=${VAULT_APPROLE_ID} + - VAULT_APPROLE_SECRETID=${VAULT_APPROLE_SECRETID} + - VAULT_ADDR=${VAULT_ADDR} + - LOGGER_LEVEL=DEBUG + - MESSAGES_CONFIG=configs/messages.json + volumes: + - ../.config/instaloader:/home/pyinstabot-downloader/.config/instaloader + depends_on: + - vault-server + - postgres + +volumes: + postgres_data: diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 000000000..d55b1deb6 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,950 @@ +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "42.0.7" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477"}, + {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7"}, + {file = "cryptography-42.0.7-cp37-abi3-win32.whl", hash = "sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b"}, + {file = "cryptography-42.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678"}, + {file = "cryptography-42.0.7-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886"}, + {file = "cryptography-42.0.7-cp39-abi3-win32.whl", hash = "sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda"}, + {file = "cryptography-42.0.7-cp39-abi3-win_amd64.whl", hash = "sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68"}, + {file = "cryptography-42.0.7.tar.gz", hash = "sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "dropbox" +version = "11.36.2" +description = "Official Dropbox API Client" +optional = false +python-versions = "*" +files = [ + {file = "dropbox-11.36.2-py2-none-any.whl", hash = "sha256:afbfce2589b777ade1deaa2c186f3650c41e41cea0f1fac497a75112a171f8e2"}, + {file = "dropbox-11.36.2-py3-none-any.whl", hash = "sha256:a21e4d2bcbeb1d8067ff87969aea48792c9a8266182491153feff2be9c1b9c8f"}, + {file = "dropbox-11.36.2.tar.gz", hash = "sha256:d48d3d16d486c78b11c14a1c4a28a2611fbf5a0d0a358b861bfd9482e603c500"}, +] + +[package.dependencies] +requests = ">=2.16.2" +six = ">=1.12.0" +stone = ">=2" + +[[package]] +name = "emoji" +version = "2.12.1" +description = "Emoji for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "emoji-2.12.1-py3-none-any.whl", hash = "sha256:a00d62173bdadc2510967a381810101624a2f0986145b8da0cffa42e29430235"}, + {file = "emoji-2.12.1.tar.gz", hash = "sha256:4aa0488817691aa58d83764b6c209f8a27c0b3ab3f89d1b8dceca1a62e4973eb"}, +] + +[package.dependencies] +typing-extensions = ">=4.7.0" + +[package.extras] +dev = ["coverage", "pytest (>=7.4.4)"] + +[[package]] +name = "exceptiongroup" +version = "1.2.1" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "hvac" +version = "2.2.0" +description = "HashiCorp Vault API client" +optional = false +python-versions = "<4.0,>=3.8" +files = [ + {file = "hvac-2.2.0-py3-none-any.whl", hash = "sha256:f287a19940c6fc518c723f8276cc9927f7400734303ee5872ac2e84539466d8d"}, + {file = "hvac-2.2.0.tar.gz", hash = "sha256:e4b0248c5672cb9a6f5974e7c8f5271a09c6c663cbf8ab11733a227f3d2db2c2"}, +] + +[package.dependencies] +requests = ">=2.27.1,<3.0.0" + +[package.extras] +parser = ["pyhcl (>=0.4.4,<0.5.0)"] + +[[package]] +name = "idna" +version = "3.7" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, +] + +[[package]] +name = "importlib-metadata" +version = "7.1.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_metadata-7.1.0-py3-none-any.whl", hash = "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570"}, + {file = "importlib_metadata-7.1.0.tar.gz", hash = "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "instaloader" +version = "4.11" +description = "Download pictures (or videos) along with their captions and other metadata from Instagram." +optional = false +python-versions = ">=3.8" +files = [ + {file = "instaloader-4.11.tar.gz", hash = "sha256:7478a1f0ed5c05911832c50cb19747243a461b5d434907f9fdb7d2d750d1b4f5"}, +] + +[package.dependencies] +requests = ">=2.4" + +[package.extras] +browser-cookie3 = ["browser_cookie3 (>=0.19.1)"] + +[[package]] +name = "jaraco-classes" +version = "3.4.0" +description = "Utility functions for Python class constructs" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790"}, + {file = "jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd"}, +] + +[package.dependencies] +more-itertools = "*" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)"] + +[[package]] +name = "jeepney" +version = "0.8.0" +description = "Low-level, pure Python DBus protocol wrapper." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jeepney-0.8.0-py3-none-any.whl", hash = "sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755"}, + {file = "jeepney-0.8.0.tar.gz", hash = "sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806"}, +] + +[package.extras] +test = ["async-timeout", "pytest", "pytest-asyncio (>=0.17)", "pytest-trio", "testpath", "trio"] +trio = ["async_generator", "trio"] + +[[package]] +name = "keyring" +version = "24.3.1" +description = "Store and access your passwords safely." +optional = false +python-versions = ">=3.8" +files = [ + {file = "keyring-24.3.1-py3-none-any.whl", hash = "sha256:df38a4d7419a6a60fea5cef1e45a948a3e8430dd12ad88b0f423c5c143906218"}, + {file = "keyring-24.3.1.tar.gz", hash = "sha256:c3327b6ffafc0e8befbdb597cacdb4928ffe5c1212f7645f186e6d9957a898db"}, +] + +[package.dependencies] +importlib-metadata = {version = ">=4.11.4", markers = "python_version < \"3.12\""} +"jaraco.classes" = "*" +jeepney = {version = ">=0.4.2", markers = "sys_platform == \"linux\""} +pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""} +SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} + +[package.extras] +completion = ["shtab (>=1.1.0)"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)"] + +[[package]] +name = "logger" +version = "1.0.6" +description = "This is an additional implementation over the logging module. This module is designed for fast initialization and configuration of readable and structured logging." +optional = false +python-versions = "^3.9 || ^3.10 || ^3.11" +files = [] +develop = false + +[package.source] +type = "git" +url = "https://github.com/obervinov/logger-package.git" +reference = "v1.0.6" +resolved_reference = "7e29923209af8b9e1944441ad62f5ac3f26e823f" + +[[package]] +name = "mega-py" +version = "1.0.8" +description = "Python lib for the Mega.co.nz API" +optional = false +python-versions = "*" +files = [ + {file = "mega.py-1.0.8-py2.py3-none-any.whl", hash = "sha256:863b2dd59f8e639402fef3c67778fd63895fc1b678127e7006df1544f04e737c"}, + {file = "mega.py-1.0.8.tar.gz", hash = "sha256:0632664abda8b7e4d0bbd68460da4b331d3d7d63db9afe7d3ce6fcb4f137c1a9"}, +] + +[package.dependencies] +pathlib = "1.0.1" +pycryptodome = ">=3.9.6,<4.0.0" +requests = ">=0.10" +tenacity = ">=5.1.5,<6.0.0" + +[[package]] +name = "messages" +version = "1.0.4" +description = "This package helps to easily and quickly generate beautiful messages for telegram bots using templates described in json." +optional = false +python-versions = "^3.9 || ^3.10 || ^3.11" +files = [] +develop = false + +[package.dependencies] +emoji = "^2" + +[package.source] +type = "git" +url = "https://github.com/obervinov/messages-package.git" +reference = "v1.0.4" +resolved_reference = "c94d0f2f09ab4f0c88f732f9a6dfc8163202bb61" + +[[package]] +name = "mock" +version = "5.1.0" +description = "Rolling backport of unittest.mock for all Pythons" +optional = false +python-versions = ">=3.6" +files = [ + {file = "mock-5.1.0-py3-none-any.whl", hash = "sha256:18c694e5ae8a208cdb3d2c20a993ca1a7b0efa258c247a1e565150f477f83744"}, + {file = "mock-5.1.0.tar.gz", hash = "sha256:5e96aad5ccda4718e0a229ed94b2024df75cc2d55575ba5762d31f5767b8767d"}, +] + +[package.extras] +build = ["blurb", "twine", "wheel"] +docs = ["sphinx"] +test = ["pytest", "pytest-cov"] + +[[package]] +name = "more-itertools" +version = "10.2.0" +description = "More routines for operating on iterables, beyond itertools" +optional = false +python-versions = ">=3.8" +files = [ + {file = "more-itertools-10.2.0.tar.gz", hash = "sha256:8fccb480c43d3e99a00087634c06dd02b0d50fbf088b380de5a41a015ec239e1"}, + {file = "more_itertools-10.2.0-py3-none-any.whl", hash = "sha256:686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684"}, +] + +[[package]] +name = "packaging" +version = "24.0" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, +] + +[[package]] +name = "pathlib" +version = "1.0.1" +description = "Object-oriented filesystem paths" +optional = false +python-versions = "*" +files = [ + {file = "pathlib-1.0.1-py3-none-any.whl", hash = "sha256:f35f95ab8b0f59e6d354090350b44a80a80635d22efdedfa84c7ad1cf0a74147"}, + {file = "pathlib-1.0.1.tar.gz", hash = "sha256:6940718dfc3eff4258203ad5021090933e5c04707d5ca8cc9e73c94a7894ea9f"}, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "ply" +version = "3.11" +description = "Python Lex & Yacc" +optional = false +python-versions = "*" +files = [ + {file = "ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce"}, + {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, +] + +[[package]] +name = "psycopg2-binary" +version = "2.9.9" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win32.whl", hash = "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win32.whl", hash = "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"}, +] + +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + +[[package]] +name = "pycryptodome" +version = "3.20.0" +description = "Cryptographic library for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pycryptodome-3.20.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-win32.whl", hash = "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-win_amd64.whl", hash = "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818"}, + {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044"}, + {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4"}, + {file = "pycryptodome-3.20.0-cp35-abi3-win32.whl", hash = "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72"}, + {file = "pycryptodome-3.20.0-cp35-abi3-win_amd64.whl", hash = "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9"}, + {file = "pycryptodome-3.20.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a"}, + {file = "pycryptodome-3.20.0-pp27-pypy_73-win32.whl", hash = "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e"}, + {file = "pycryptodome-3.20.0.tar.gz", hash = "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7"}, +] + +[[package]] +name = "pytelegrambotapi" +version = "4.18.1" +description = "Python Telegram bot api." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytelegrambotapi-4.18.1-py3-none-any.whl", hash = "sha256:07951383c5831b1f810edaf01e06ee95a40253486d725780cf88de15aa0893ce"}, + {file = "pytelegrambotapi-4.18.1.tar.gz", hash = "sha256:6bf79a726624441e84724d933312edb3138ad22906ffea2fee09ee6846236ac0"}, +] + +[package.dependencies] +requests = "*" + +[package.extras] +aiohttp = ["aiohttp"] +aioredis = ["aioredis"] +coloredlogs = ["coloredlogs"] +fastapi = ["fastapi"] +json = ["ujson"] +pil = ["pillow"] +psutil = ["psutil"] +redis = ["redis (>=3.4.1)"] +uvicorn = ["uvicorn"] +watchdog = ["watchdog"] + +[[package]] +name = "pytest" +version = "8.2.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.2.1-py3-none-any.whl", hash = "sha256:faccc5d332b8c3719f40283d0d44aa5cf101cec36f88cde9ed8f2bc0538612b1"}, + {file = "pytest-8.2.1.tar.gz", hash = "sha256:5046e5b46d8e4cac199c373041f26be56fdb81eb4e67dc11d4e10811fc3408fd"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pywin32-ctypes" +version = "0.2.2" +description = "A (partial) reimplementation of pywin32 using ctypes/cffi" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pywin32-ctypes-0.2.2.tar.gz", hash = "sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60"}, + {file = "pywin32_ctypes-0.2.2-py3-none-any.whl", hash = "sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7"}, +] + +[[package]] +name = "requests" +version = "2.32.2" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, + {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "secretstorage" +version = "3.3.3" +description = "Python bindings to FreeDesktop.org Secret Service API" +optional = false +python-versions = ">=3.6" +files = [ + {file = "SecretStorage-3.3.3-py3-none-any.whl", hash = "sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99"}, + {file = "SecretStorage-3.3.3.tar.gz", hash = "sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77"}, +] + +[package.dependencies] +cryptography = ">=2.0" +jeepney = ">=0.6" + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "stone" +version = "3.3.6" +description = "Stone is an interface description language (IDL) for APIs." +optional = false +python-versions = "*" +files = [ + {file = "stone-3.3.6-py3-none-any.whl", hash = "sha256:f25c977936d7b5f75b9a953543257681eb19bcd7758f91b6b515f931b1bd1a66"}, + {file = "stone-3.3.6.tar.gz", hash = "sha256:7e96560bffdaf038d53ca7673cac8cabbaf824a74561564961c34237df901717"}, +] + +[package.dependencies] +ply = ">=3.4" +six = ">=1.12.0" + +[[package]] +name = "telegram" +version = "1.2.0" +description = "This is an additional implementation compared to the telebot module. This module is designed for quick initialization, authorization and rendering of various buttons/widgets for telegram bots." +optional = false +python-versions = "^3.9" +files = [] +develop = false + +[package.dependencies] +logger = {git = "https://github.com/obervinov/logger-package.git", tag = "v1.0.6"} +messages = {git = "https://github.com/obervinov/messages-package.git", tag = "v1.0.4"} +pyTelegramBotAPI = "^4" +pytest = "^8" +vault = {git = "https://github.com/obervinov/vault-package.git", tag = "v2.0.4"} + +[package.source] +type = "git" +url = "https://github.com/obervinov/telegram-package.git" +reference = "v1.2.0" +resolved_reference = "0a7d08cc3b215e6e7bfe13ccfe72f8190f5ae457" + +[[package]] +name = "tenacity" +version = "5.1.5" +description = "Retry code until it succeeeds" +optional = false +python-versions = "*" +files = [ + {file = "tenacity-5.1.5-py2.py3-none-any.whl", hash = "sha256:3a916e734559f1baa2cab965ee00061540c41db71c3bf25375b81540a19758fc"}, + {file = "tenacity-5.1.5.tar.gz", hash = "sha256:e664bd94f088b17f46da33255ae33911ca6a0fe04b156d334b601a4ef66d3c5f"}, +] + +[package.dependencies] +six = ">=1.9.0" + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.12.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, + {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, +] + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "users" +version = "2.0.5" +description = "This python module is a simple implementation of user management functionality for telegram bots, such as: authentication, authorization and request limiting." +optional = false +python-versions = "^3.9 || ^3.10 || ^3.11" +files = [] +develop = false + +[package.dependencies] +logger = {git = "https://github.com/obervinov/logger-package.git", tag = "v1.0.6"} +vault = {git = "https://github.com/obervinov/vault-package.git", tag = "v2.0.4"} + +[package.source] +type = "git" +url = "https://github.com/obervinov/users-package.git" +reference = "v2.0.5" +resolved_reference = "f4f4b758d394fd1820c9ee3ce264b5d18a439823" + +[[package]] +name = "vault" +version = "2.0.4" +description = "This is an additional implementation compared to the hvac module. The main purpose of which is to simplify the use and interaction with vault for my standard projects. This module contains a set of methods for working with secrets and quickly configuring Vault." +optional = false +python-versions = "^3.9 || ^3.10 || ^3.11" +files = [] +develop = false + +[package.dependencies] +hvac = "^2" +keyring = "^24" +logger = {git = "https://github.com/obervinov/logger-package.git", tag = "v1.0.6"} +python-dateutil = "^2" +SecretStorage = "^3" + +[package.source] +type = "git" +url = "https://github.com/obervinov/vault-package.git" +reference = "v2.0.4" +resolved_reference = "54a312b747ad84c391a837c5bddaed7a021c9d76" + +[[package]] +name = "zipp" +version = "3.19.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "zipp-3.19.0-py3-none-any.whl", hash = "sha256:96dc6ad62f1441bcaccef23b274ec471518daf4fbbc580341204936a5a3dddec"}, + {file = "zipp-3.19.0.tar.gz", hash = "sha256:952df858fb3164426c976d9338d3961e8e8b3758e2e059e0f754b8c4262625ee"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "e08ed1cdb707b44e9aeb7a92e5b75123b065a3481d5209d6c5c482a56d859a67" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..4df3fb73b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,33 @@ +[tool.poetry] +name = "pyinstabot-downloader" +version = "2.1.0" +description = "This project is a Telegram bot that allows you to backup post content from your Instagram profile to Dropbox or Mega clouds." +authors = ["Bervinov Oleg "] +maintainers = ["Bervinov Oleg "] +license = "MIT" +readme = "README.md" +homepage = "https://github.com/obervinov/pyinstabot-downloader" +repository = "https://github.com/obervinov/pyinstabot-downloader" +documentation = "https://github.com/obervinov/pyinstabot-downloader/blob/main/README.md" +keywords = ["telegram", "bot", "telebot", "telegram-bot", "instagram", "meganz", "dropbox", "backup", "pyinstabot", "pyinstabot-downloader"] +include = ["CHANGELOG.md"] +packages = [{include = "src"}] + +[tool.poetry.urls] +"Bug Tracker" = "https://github.com/obervinov/pyinstabot-downloader/issues" + +[tool.poetry.dependencies] +python = "^3.9" +instaloader = "^4" +dropbox = "^11" +'mega.py' = "^1.0.8" +psycopg2-binary = "^2" +mock = "^5" +logger = { git = "https://github.com/obervinov/logger-package.git", tag = "v1.0.6" } +vault = { git = "https://github.com/obervinov/vault-package.git", tag = "v2.0.4" } +users = { git = "https://github.com/obervinov/users-package.git", tag = "v2.0.5" } +telegram = { git = "https://github.com/obervinov/telegram-package.git", tag = "v1.2.0" } + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 72d8e92c6..000000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -instaloader==4.10.0 -dropbox==11.21.0 -mega.py==1.0.8 -tenacity==5.1.5 -messages @ git+https://github.com/obervinov/messages-package.git@v1.0.0 -users @ git+https://github.com/obervinov/users-package.git@v1.0.5 -logger @ git+https://github.com/obervinov/logger-package.git@v1.0.1 -telegram @ git+https://github.com/obervinov/telegram-package.git@v1.1.3 -vault @ git+https://github.com/obervinov/vault-package.git@v2.0.1 \ No newline at end of file diff --git a/src/bot.py b/src/bot.py index cdf9763ec..6267b0ffc 100644 --- a/src/bot.py +++ b/src/bot.py @@ -2,207 +2,594 @@ This module contains the main code for the bot to work and contains the main logic linking the additional modules. """ +from datetime import datetime, timedelta +import re +import threading +import time +import random +import string + +from mock import MagicMock from logger import log -import constants +from telegram import TelegramBot +from telegram import exceptions as TelegramExceptions +from users import Users +from vault import VaultClient +from configs.constants import (PROJECT_ENVIRONMENT, TELEGRAM_BOT_NAME, ROLES_MAP, QUEUE_FREQUENCY, STATUSES_MESSAGE_FREQUENCY) +from modules.database import DatabaseClient +from modules.exceptions import FailedMessagesStatusUpdater +from modules.tools import get_hash +from modules.downloader import Downloader +from modules.uploader import Uploader -# decorators -@constants.BOT.message_handler(commands=['start']) -def start_message( - message: constants.TELEGRAM_CLIENT.telegram_types.Message = None -) -> None: +# Vault client +vault = VaultClient(name=TELEGRAM_BOT_NAME) +# Telegram instance +telegram = TelegramBot(vault=vault) +# Telegram bot for decorators +bot = telegram.telegram_bot +# Users module with rate limits option +users_rl = Users(vault=vault) +# Users module without rate limits option +users = Users(vault=vault, rate_limits=False) + +# Client for download content from supplier +# If API disabled, the mock object will be used +downloader_api_enabled = vault.read_secret(path='configuration/downloader-api').get('enabled', False) +if downloader_api_enabled == 'True': + log.info('[Bot]: downloader API is enabled: %s', downloader_api_enabled) + downloader = Downloader(vault=vault) +else: + log.warning('[Bot]: downloader API is disabled, using mock object, because enabled flag is %s', downloader_api_enabled) + downloader = MagicMock() + downloader.get_post_content.return_value = { + 'post': f"mock_{''.join(random.choices(string.ascii_letters + string.digits, k=10))}", + 'owner': 'undefined', + 'type': 'fake', + 'status': 'completed' + } + +# Client for upload content to the cloud storage +# If API disabled, the mock object will be used +uploader_api_enabled = vault.read_secret(path='configuration/uploader-api').get('enabled', False) +if uploader_api_enabled == 'True': + log.info('[Bot]: uploader API is enabled: %s', uploader_api_enabled) + uploader = Uploader(vault=vault) +else: + log.warning('[Bot]: uploader API is disabled, using mock object, because enabled flag is %s', uploader_api_enabled) + uploader = MagicMock() + uploader.run_transfers.return_value = 'completed' + +# Client for communication with the database +database = DatabaseClient(vault=vault, environment=PROJECT_ENVIRONMENT) + + +# START HANDLERS BLOCK ############################################################################################################## +# Command handler for START command +@bot.message_handler(commands=['start']) +def start_command(message: telegram.telegram_types.Message = None) -> None: """ - The function for intercepting the start command sent to the bot. + Sends a startup message to the specified Telegram chat. Args: - :param message (telegram_types.Message): the message received by the bot. + message (telegram.telegram_types.Message): The message object containing information about the chat. Returns: None """ - if constants.AUTH_CLIENT.check_permissions( - message.chat.id - ) == "allow": - log.info( - '[%s] sending startup message in chat %s', - __name__, - message.chat.id - ) + if users.user_access_check(message.chat.id).get('access', None) == users.user_status_allow: + log.info('[Bot]: Processing `start` command for user %s...', message.chat.id) + + # Add user to the database + _ = database.add_user(user_id=message.chat.id, chat_id=message.chat.id) - constants.BOT.send_message( + # Main message + reply_markup = telegram.create_inline_markup(ROLES_MAP.keys()) + start_message = telegram.send_styled_message( chat_id=message.chat.id, - text=constants.MESSAGES_GENERATOR.render_template( - template_alias='hello_message', - username=message.from_user.username, - userid=message.chat.id - ) + messages_template={ + 'alias': 'start_message', + 'kwargs': {'username': message.from_user.username, 'userid': message.chat.id} + }, + reply_markup=reply_markup + ) + bot.pin_chat_message(start_message.chat.id, start_message.id) + bot.delete_message(message.chat.id, message.id) + update_status_message(user_id=message.chat.id) + else: + telegram.send_styled_message( + chat_id=message.chat.id, + messages_template={ + 'alias': 'reject_message', + 'kwargs': {'username': message.chat.username, 'userid': message.chat.id} + } ) -@constants.BOT.message_handler(regexp=r"^https://(www\.)?instagram.com/(?!p/)(?!reel/).*$") -def get_posts_account( - message: constants.TELEGRAM_CLIENT.telegram_types.Message = None -) -> None: +# Callback query handler for InlineKeyboardButton (BUTTONS) +@bot.callback_query_handler(func=lambda call: True) +def bot_callback_query_handler(call: telegram.callback_query = None) -> None: """ - A function for intercepting links sent to the bot to the Instagram profile. + The handler for the callback query from the user. + Mainly used to handle button presses. Args: - :param message (telegram_types.Message): the message received by the bot. + call (telegram.callback_query): The callback query object. Returns: None """ - if constants.AUTH_CLIENT.check_permissions( - message.chat.id - ) == "allow": - log.info( - '[%s] starting handler for profile url %s...', - __name__, - message.text + log.info('[Bot]: Processing button `%s` for user %s...', call.data, call.message.chat.id) + if users.user_access_check(call.message.chat.id, ROLES_MAP[call.data]).get('permissions', None) == users.user_status_allow: + if call.data == "Post": + button_post(call=call) + elif call.data == "Posts List": + button_posts_list(call=call) + else: + log.error('[Bot]: Handler for button %s not found', call.data) + else: + telegram.send_styled_message( + chat_id=call.message.chat.id, + messages_template={ + 'alias': 'permission_denied_message', + 'kwargs': {'username': call.message.chat.username, 'userid': call.message.chat.id} + } ) - editable_message = None - stats_message_id = None - account_name = message.text.split("/")[3].split("?")[0] - account_info = constants.DOWNLOADER_INSTANCE.get_download_info( - account=account_name + +# Handler for incorrect flow (UNKNOWN INPUT) +@bot.message_handler(regexp=r'.*') +def unknown_command(message: telegram.telegram_types.Message = None) -> None: + """ + Sends a message to the user if the command is not recognized. + + Args: + message (telegram.telegram_types.Message): The message object containing the unrecognized command. + + Returns: + None + """ + if users.user_access_check(message.chat.id).get('access', None) == users.user_status_allow: + log.error('[Bot]: Invalid command `%s` from user %s', message.text, message.chat.id) + telegram.send_styled_message( + chat_id=message.chat.id, + messages_template={'alias': 'unknown_command'} ) + else: + telegram.send_styled_message( + chat_id=message.chat.id, + messages_template={ + 'alias': 'reject_message', + 'kwargs': {'username': message.chat.username, 'userid': message.chat.id} + } + ) +# END HANDLERS BLOCK ############################################################################################################## - constants.BOT.send_message( - message.chat.id, - constants.MESSAGES_GENERATOR.render_template( - template_alias='account_info', - account_name=account_name, - shortcodes_count=account_info['shortcodes_total_count'] - ) + +# START BUTTONS BLOCK ############################################################################################################# +# Inline button handler for Post +def button_post(call: telegram.callback_query = None) -> None: + """ + The handler for the Post button. + + Args: + call (telegram.callback_query): The callback query object. + + Returns: + None + """ + user = users.user_access_check(call.message.chat.id, ROLES_MAP['Post']) + if user.get('permissions', None) == users.user_status_allow: + help_message = telegram.send_styled_message( + chat_id=call.message.chat.id, + messages_template={'alias': 'help_for_post'} + ) + bot.register_next_step_handler( + call.message, + process_one_post, + help_message + ) + else: + telegram.send_styled_message( + chat_id=call.message.chat.id, + messages_template={ + 'alias': 'permission_denied_message', + 'kwargs': {'username': call.message.chat.username, 'userid': call.message.chat.id} + } ) - for shortcode in account_info['shortcodes_for_download']: - # download the contents of an instagram post to a temporary folder - d_response = constants.DOWNLOADER_INSTANCE.get_post_content( - shortcode=shortcode - ) - # upload the received content to the destination storage - _ = constants.UPLOADER_INSTANCE.start_upload( - sub_dir_name=d_response['owner'] + +# Inline button handler for Posts List +def button_posts_list(call: telegram.callback_query = None) -> None: + """ + The handler for the Posts List button. + + Args: + call (telegram.callback_query): The callback query object. + + Returns: + None + """ + user = users.user_access_check(call.message.chat.id, ROLES_MAP['Posts List']) + if user.get('permissions', None) == users.user_status_allow: + help_message = telegram.send_styled_message( + chat_id=call.message.chat.id, + messages_template={'alias': 'help_for_posts_list'} + ) + bot.register_next_step_handler( + call.message, + process_list_posts, + help_message + ) + else: + telegram.send_styled_message( + chat_id=call.message.chat.id, + messages_template={ + 'alias': 'permission_denied_message', + 'kwargs': {'username': call.message.chat.username, 'userid': call.message.chat.id} + } + ) + + +# START BLOCK ADDITIONAL FUNCTIONS ###################################################################################################### +def update_status_message(user_id: str = None) -> None: + """ + Updates the status message for the user. + + Args: + user_id (str): The user id. + + Returns: + None + """ + try: + chat_id = user_id + exist_status_message = database.get_considered_message(message_type='status_message', chat_id=chat_id) + message_statuses = get_user_messages(user_id=user_id) + diff_between_messages = False + if exist_status_message: + # check difference between messages content + if exist_status_message[3] != get_hash(message_statuses): + diff_between_messages = True + + # if message already sended and expiring (because bot can edit message only first 48 hours) + # automatic renew message every 23 hours + if exist_status_message[2] < datetime.now() - timedelta(hours=23): + if exist_status_message[2] < datetime.now() - timedelta(hours=48): + log.warning('[Bot]: `status_message` for user %s old more than 48 hours, can not delete them', user_id) + else: + _ = bot.delete_message( + chat_id=chat_id, + message_id=exist_status_message[0] + ) + status_message = telegram.send_styled_message( + chat_id=chat_id, + messages_template={ + 'alias': 'message_statuses', + 'kwargs': message_statuses + } + ) + bot.pin_chat_message(status_message.chat.id, status_message.id) + database.keep_message( + message_id=status_message.message_id, + chat_id=status_message.chat.id, + message_type='status_message', + message_content=message_statuses + ) + log.info('[Bot]: `status_message` for user %s has been renewed', user_id) + elif message_statuses is not None and diff_between_messages: + log.info( + '[Bot]: `status_message` for user %s is outdated, updating %s -> %s...', + user_id, exist_status_message[3], get_hash(message_statuses) + ) + editable_message = telegram.send_styled_message( + chat_id=chat_id, + messages_template={ + 'alias': 'message_statuses', + 'kwargs': message_statuses + }, + editable_message_id=exist_status_message[0] + ) + database.keep_message( + message_id=editable_message.message_id, + chat_id=editable_message.chat.id, + message_type='status_message', + message_content=message_statuses + ) + log.info('[Bot]: `status_message` for user %s has been updated', user_id) + elif not diff_between_messages: + log.info('[Bot]: `status_message` for user %s is actual', user_id) + else: + status_message = telegram.send_styled_message( + chat_id=chat_id, + messages_template={ + 'alias': 'message_statuses', + 'kwargs': message_statuses + } ) - # render progressbar - progressbar = constants.MESSAGES_GENERATOR.render_progressbar( - total_count=account_info['shortcodes_total_count'], - current_count=account_info['shortcodes_exist_count'] + bot.pin_chat_message( + chat_id=status_message.chat.id, + message_id=status_message.id ) - account_info['shortcodes_exist_count'] = account_info['shortcodes_exist_count'] + 1 - stats_response = constants.MESSAGES_GENERATOR.render_template( - template_alias='account_stats_progress', - account_name=account_name, - posts_downloaded=account_info['shortcodes_exist_count'], - posts_count=account_info['shortcodes_total_count'], - progressbar=progressbar + database.keep_message( + message_id=status_message.message_id, + chat_id=status_message.chat.id, + message_type='status_message', + message_content=message_statuses ) - # check whether a message with stats has already been sent and whether we can edit it - if not editable_message: - stats_message_id = constants.BOT.send_message( - chat_id=message.chat.id, - text=stats_response - ).id - editable_message = True - elif editable_message: - constants.BOT.edit_message_text( - text=stats_response, - chat_id=message.chat.id, - message_id=stats_message_id - ) + log.info('[Bot]: `status_message` for user %s has been created', user_id) + except TypeError as exception: + exception_context = { + 'message': f"Failed to update the message with the status of received messages for user {user_id}", + 'exception': exception, + 'exist_status_message': exist_status_message, + 'message_statuses': message_statuses, + 'diff_between_messages': diff_between_messages + } + log.error(exception_context) - # when all messages are uploaded send a response with statistics - constants.BOT.edit_message_text( - text=constants.MESSAGES_GENERATOR.render_template( - template_alias='account_stats_done', - posts_downloaded=account_info['shortcodes_exist_count'], - posts_count=account_info['shortcodes_total_count'], - account_name=account_name, - progressbar=constants.MESSAGES_GENERATOR.render_progressbar( - total_count=account_info['shortcodes_total_count'], - current_count=account_info['shortcodes_exist_count'] - ) - ), + +def get_user_messages(user_id: str = None) -> dict: + """ + Returns the queue and processed posts for the user. + + Args: + user_id (str): The user id. + + Returns: + dict: The queue and processed posts for the user. + + Examples: + >>> get_user_messages(user_id='1234567890') + {'queue': 'queue is empty', 'processed': 'processed is empty'} + """ + queue_dict = database.get_user_queue(user_id=user_id) + processed_dict = database.get_user_processed(user_id=user_id) + + queue_string = '' + if queue_dict is not None: + sorted_data = sorted(queue_dict[user_id], key=lambda x: x['scheduled_time'], reverse=False) + for item in sorted_data: + queue_string = queue_string + f"+ {item['post_id']}: will be started {item['scheduled_time']}\n" + else: + queue_string = 'queue is empty' + + processed_string = '' + if processed_dict is not None: + sorted_data = sorted(processed_dict[user_id], key=lambda x: x['timestamp'], reverse=False) + for item in sorted_data: + processed_string = processed_string + f"* {item['post_id']}: {item['state']} at {item['timestamp']}\n" + else: + processed_string = 'processed is empty' + + return {'queue': queue_string, 'processed': processed_string} + + +def message_parser(message: telegram.telegram_types.Message = None) -> dict: + """ + Parses the message containing the Instagram post link and returns the data. + + Args: + message (telegram.telegram_types.Message): The message object containing the post link. + + Returns: + dict: The data containing the user id, post url, post id, post owner, link type, message id, and chat id. + """ + data = {} + if re.match(r'^https://www.instagram.com/(p|reel)/.*', message.text): + post_id = message.text.split('/')[4] + if len(post_id) == 11 and re.match(r'^[a-zA-Z0-9_-]+$', post_id): + data['user_id'] = message.chat.id + data['post_url'] = message.text + data['post_id'] = post_id + data['post_owner'] = 'undefined' + data['link_type'] = 'post' + data['message_id'] = message.id + data['chat_id'] = message.chat.id + else: + log.error('[Bot]: post id %s from user %s is wrong', post_id, message.chat.id) + telegram.send_styled_message( + chat_id=message.chat.id, + messages_template={'alias': 'url_error'} + ) + else: + log.error('[Bot]: post link %s from user %s is incorrect', message.text, message.chat.id) + telegram.send_styled_message( chat_id=message.chat.id, - message_id=stats_message_id - ) - log.info( - '[%s] all available posts from account %s has been downloaded', - __name__, - account_name + messages_template={'alias': 'url_error'} ) + return data +# END BLOCK ADDITIONAL FUNCTIONS ###################################################################################################### -@constants.BOT.message_handler(regexp="^https://www.instagram.com/(p|reel)/.*") -def get_post_account( - message: constants.TELEGRAM_CLIENT.telegram_types.Message = None +# START BLOCK PROCESSING FUNCTIONS #################################################################################################### +def process_one_post( + message: telegram.telegram_types.Message = None, + help_message: telegram.telegram_types.Message = None, + mode: str = 'single' ) -> None: """ - A function for intercepting links sent by a bot to an Instagram post. + Processes an Instagram post link sent by a user and adds it to the queue for download. Args: - :param message (telegram_types.Message): the message received by the bot. + message (telegram.telegram_types.Message): The Telegram message object containing the post link. + help_message (telegram.telegram_types.Message): The help message to be deleted. + mode (str, optional): The mode of processing. Defaults to 'single'. Returns: None """ - if constants.AUTH_CLIENT.check_permissions( - message.chat.id - ) == "allow": - log.info( - '[%s] starting handler for post url %s...', - __name__, - message.text - ) + # Check permissions + user = users_rl.user_access_check(message.chat.id, ROLES_MAP['Post']) + if user.get('permissions', None) == users_rl.user_status_allow: + data = message_parser(message) + rate_limit = user.get('rate_limits', {}).get('end_time', None) - shortcode = message.text.split("/")[4] - - # download history - # we check the whole history instead of getting the owner by a short code to reduce - # the frequency of requests to the instagram api - for owner in constants.VAULT_CLIENT.list_secrets(path='history/'): - for post in constants.VAULT_CLIENT.read_secret(path=f"history/{owner}"): - if post == shortcode and constants.VAULT_CLIENT.read_secret( - path=f"history/{owner}", - key=post - ) == 'downloaded': - constants.BOT.send_message( - chat_id=message.chat.id, - text=constants.MESSAGES_GENERATOR.render_template( - template_alias='post_already_downloaded', - post_id=shortcode, - owner=owner - ) - ) - log.warning( - '[%s] the post %s of the owner %s has already been downloaded, skipped.', - __name__, - post, - owner - ) - return - # download the contents of an instagram post to a temporary folder - d_response = constants.DOWNLOADER_INSTANCE.get_post_content( - shortcode=shortcode - ) - # upload the received content to the destination storage - u_response = constants.UPLOADER_INSTANCE.start_upload( - sub_dir_name=d_response['owner'] - ) - constants.BOT.send_message( + # Define time to process the message in queue + if rate_limit: + data['scheduled_time'] = rate_limit + else: + data['scheduled_time'] = datetime.now() + + # Check if the message is unique + if database.check_message_uniqueness(data['post_id'], data['user_id']): + _ = database.add_message_to_queue(data) + update_status_message(user_id=message.chat.id) + log.info('[Bot]: post %s from user %s has been added to the queue', message.text, message.chat.id) + else: + log.info('[Bot]: post %s from user %s already in queue or processed', data['post_id'], message.chat.id) + + # If it is not a list of posts - delete users message + if mode == 'single': + telegram.delete_message(message.chat.id, message.id) + if help_message is not None: + telegram.delete_message(message.chat.id, help_message.id) + else: + telegram.send_styled_message( chat_id=message.chat.id, - text=constants.MESSAGES_GENERATOR.render_template( - 'post_stats_info', - post_id=shortcode, - download_response=d_response, - upload_response=u_response + messages_template={ + 'alias': 'reject_message', + 'kwargs': {'username': message.chat.username, 'userid': message.chat.id} + } + ) + + +def process_list_posts( + message: telegram.telegram_types.Message = None, + help_message: telegram.telegram_types.Message = None +) -> None: + """ + Process a list of Instagram post links. + + Args: + message (telegram.telegram_types.Message, optional): The message containing the list of post links. Defaults to None. + help_message (telegram.telegram_types.Message, optional): The help message to be deleted. Defaults to None. + + Returns: + None + """ + user = users.user_access_check(message.chat.id, ROLES_MAP['Posts List']) + if user.get('permissions', None) == users.user_status_allow: + for link in message.text.split('\n'): + message.text = link + process_one_post( + message=message, + help_message=help_message, + mode='list' ) + telegram.delete_message(message.chat.id, message.id) + if help_message is not None: + telegram.delete_message(message.chat.id, help_message.id) + else: + telegram.send_styled_message( + chat_id=message.chat.id, + messages_template={ + 'alias': 'reject_message', + 'kwargs': {'username': message.chat.username, 'userid': message.chat.id} + } ) +# END BLOCK PROCESSING FUNCTIONS #################################################################################################### + + +# SPECIFIED THREADS ############################################################################################################### +def status_message_updater_thread() -> None: + """ + Handler thread for monitoring and timely updating of the widget with the status of messages sent by the user. + + Args: + None + + Returns: + None + """ + log.info('[Message-updater-thread]: started thread for `status_message` updater') + while True: + try: + time.sleep(STATUSES_MESSAGE_FREQUENCY) + if database.get_users(): + for user in database.get_users(): + user_id = user[0] + update_status_message(user_id=user_id) + # pylint: disable=broad-exception-caught + except Exception as exception: + exception_context = { + 'call': threading.current_thread().name, + 'message': 'Failed to update the message with the status of received messages ', + 'users': database.get_users(), + 'user': user, + 'exception': exception + } + raise FailedMessagesStatusUpdater(exception_context) from exception + + +def queue_handler_thread() -> None: + """ + Handler thread to process messages from the queue at the specified time. + + Args: + None + + Returns: + None + """ + log.info('[Queue-handler-thread]: started thread for queue handler') + # Verify scheduled timestamps in the users queue for cases when bot was down + database.verify_users_queue() + + while True: + time.sleep(QUEUE_FREQUENCY) + message = database.get_message_from_queue(datetime.now()) + + if message is not None: + download_status = message[9] + upload_status = message[10] + post_id = message[2] + owner_id = message[4] + + log.info('[Queue-handler-thread] starting handler for post url %s...', message[3]) + # download the contents of an instagram post to a temporary folder + if download_status != 'completed': + download_metadata = downloader.get_post_content(shortcode=post_id) + owner_id = download_metadata['owner'] + download_status = download_metadata['status'] + database.update_message_state_in_queue( + post_id=post_id, + state='processing', + download_status=download_status, + upload_status=upload_status, + post_owner=owner_id + ) + # upload the received content to the destination storage + if upload_status != 'completed': + upload_status = uploader.run_transfers(sub_directory=owner_id) + database.update_message_state_in_queue( + post_id=post_id, + state='processing', + download_status=download_status, + upload_status=upload_status, + post_owner=owner_id + ) + # mark item in queue as processed + if download_status == 'completed' and upload_status == 'completed': + database.update_message_state_in_queue( + post_id=post_id, + state='processed', + download_status=download_status, + upload_status=upload_status, + post_owner=owner_id + ) + log.info('[Queue-handler-thread] the post %s has been processed successfully', post_id) + else: + log.warning( + '[Queue-handler-thread] the post %s has not been processed yet (download: %s, uploader: %s)', + post_id, download_status, upload_status + ) + else: + log.info("[Queue-handler-thread] no messages in the queue for processing at the moment, waiting...") +# SPECIFIED THREADS ############################################################################################################### -# entrypoint def main(): """ The main entry point of the project. @@ -213,12 +600,19 @@ def main(): Returns: None """ + # Thread for processing queue + thread_queue_handler_thread = threading.Thread(target=queue_handler_thread, args=(), name="Thread-queue-handler") + thread_queue_handler_thread.start() + # Thread for update status message + thread_status_message = threading.Thread(target=status_message_updater_thread, args=(), name="Thread-message-updater") + thread_status_message.start() + # Run bot while True: - log.info( - 'Starting bot %s', - constants.BOT_NAME - ) - constants.BOT.polling() + try: + telegram.launch_bot() + except TelegramExceptions.FailedToCreateInstance as telegram_api_exception: + log.error('[Bot]: main thread failed, restart thread: %s', telegram_api_exception) + time.sleep(5) if __name__ == "__main__": diff --git a/src/configs/constants.py b/src/configs/constants.py new file mode 100644 index 000000000..090abc2ae --- /dev/null +++ b/src/configs/constants.py @@ -0,0 +1,19 @@ +""" +This module contains the constants for this python project. +""" +import os + +# environment variables +PROJECT_ENVIRONMENT = os.environ.get("PROJECT_ENVIRONMENT", "dev") +TELEGRAM_BOT_NAME = os.environ.get('TELEGRAM_BOT_NAME', 'pyinstabot-downloader') + +# permissions roles and buttons mapping +# 'button_title': 'role' +ROLES_MAP = { + 'Post': 'post', + 'Posts List': 'posts_list' +} + +# Queue handler +QUEUE_FREQUENCY = 60 +STATUSES_MESSAGE_FREQUENCY = 60 diff --git a/src/configs/databases.json b/src/configs/databases.json new file mode 100644 index 000000000..fbdefc789 --- /dev/null +++ b/src/configs/databases.json @@ -0,0 +1,73 @@ +{ + "Tables": [ + { + "name": "queue", + "description": "Messages received by the bot are placed in this table for further processing at the specified time by a separate thread", + "columns": [ + "id SERIAL PRIMARY KEY, ", + "user_id VARCHAR(255) NOT NULL, ", + "post_id VARCHAR(255) NOT NULL, ", + "post_url VARCHAR(255) NOT NULL, ", + "post_owner VARCHAR(255) NOT NULL, ", + "link_type VARCHAR(255) NOT NULL DEFAULT 'post', ", + "message_id VARCHAR(255) NOT NULL, ", + "chat_id VARCHAR(255) NOT NULL, ", + "scheduled_time TIMESTAMP NOT NULL, ", + "download_status VARCHAR(255) NOT NULL DEFAULT 'not started', ", + "upload_status VARCHAR(255) NOT NULL DEFAULT 'not started', ", + "timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ", + "state VARCHAR(255) NOT NULL DEFAULT 'waiting'" + ] + }, + { + "name": "processed", + "description": "After processing from the queue, the record should be moved to this table and enriched with additional data", + "columns": [ + "id SERIAL PRIMARY KEY, ", + "user_id VARCHAR(255) NOT NULL, ", + "post_id VARCHAR(255) NOT NULL, ", + "post_url VARCHAR(255) NOT NULL, ", + "post_owner VARCHAR(255) NOT NULL, ", + "link_type VARCHAR(255) NOT NULL DEFAULT 'post', ", + "message_id VARCHAR(255) NOT NULL, ", + "chat_id VARCHAR(255) NOT NULL, ", + "download_status VARCHAR(255) NOT NULL, ", + "upload_status VARCHAR(255) NOT NULL, ", + "timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ", + "state VARCHAR(255) NOT NULL DEFAULT 'processed'" + ] + }, + { + "name": "migrations", + "description": "Table to store the migration history of the database", + "columns": [ + "id SERIAL PRIMARY KEY, ", + "name VARCHAR(255) NOT NULL, ", + "version VARCHAR(255) NOT NULL, ", + "timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP" + ] + }, + { + "name": "messages", + "description": "The table stores the message ID and the chat ID", + "columns": [ + "id SERIAL PRIMARY KEY, ", + "message_id VARCHAR(255) NOT NULL, ", + "chat_id VARCHAR(255) NOT NULL, ", + "timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ", + "message_type VARCHAR(255) NOT NULL , ", + "producer VARCHAR(255) NOT NULL , ", + "message_content_hash VARCHAR(64) NOT NULL" + ] + }, + { + "name": "users", + "description": "The table stores the user ID and the chat ID for communication with the bot", + "columns": [ + "id SERIAL PRIMARY KEY, ", + "user_id VARCHAR(255) NOT NULL, ", + "chat_id VARCHAR(255) NOT NULL" + ] + } + ] +} diff --git a/src/configs/messages.json b/src/configs/messages.json index 359eb5aca..22127be8c 100644 --- a/src/configs/messages.json +++ b/src/configs/messages.json @@ -1 +1,36 @@ -{"templates":{"hello_message": {"text": "Hi, {0}({1})! {2}\nAccess for your account - allowed {3}\nBot functions:\n {4} Upload post content by instagram link to local fs, dropbox or mega cloud\n {4} Uploading all posts content by instagram profile-link to local fs, dropbox or mega cloud\nJust send link{5}", "args": ["username", "userid", ":raised_hand:", ":unlocked:", ":pushpin:", ":raising_hands:"]}, "account_info": {"text": "{0} Account information {1} read successfully.\n {2} Posts count: {3}\n", "args": [":glowing_star:", "account_name", ":night_with_stars:", "shortcodes_count"]}, "account_stats_progress": {"text": "{0} Account statistics {1}\n{2} posts downloaded: {3} of {4}\n{5}", "args": [":framed_picture:", "account_name", ":down_arrow:", "posts_downloaded", "posts_count", "progressbar"]}, "account_stats_done": {"text": "{0} {1} of {2} posts from account {3} has been downloaded\n{4}", "args": [":hot_beverage:", "posts_downloaded", "posts_count", "account_name", "progressbar"]}, "post_stats_info": {"text": "{0} The content of the post {1} has been processed\n\n{2} {3}\n\n{4} {5}", "args": [":framed_picture:", "post_id", ":down_arrow:", "download_response", ":up_arrow:", "upload_response"]}, "post_already_downloaded": {"text": "{0} The content of the post {1} of the owner {2} has already been downloaded.\n{3}The download was skipped.", "args": [":construction:", "post_id", "owner", ":boxing_glove:"]}}} +{ + "templates": { + "start_message": { + "text": "Hi, {0}({1})! {2}\nAccess for your account - allowed {3}\nYou can receive content in various ways. Just press the button:", + "args": ["username", "userid", ":raised_hand:", ":unlocked:"] + }, + "message_statuses": { + "text": "{0} Your last activity:\n\n{1} processed (last 10)\n{2}\n{3} in queue\n{4}", + "args": [":bar_chart:", ":check_mark_button:", "processed", ":shopping_cart:", "queue"] + }, + "reject_message": { + "text": "Greetings to {0} ({1})! \nThis is a private bot and it looks like you don't have the necessary access rights {3}\nHave a nice day and bye!", + "args": ["username", "userid", ":raised_hand:", ":no_entry:"] + }, + "permission_denied_message": { + "text": "{0} Sorry, you have not permissions for use this feature. Please, check you permission roles.\nMetadata:\nuserid: {1}\nusername: {2}", + "args": [":no_entry:", "userid", "username"] + }, + "help_for_post": { + "text": "{0} For a download post, just send a link to this post.\nfor example:\nhttps://www.instagram.com/p/QwErtY_1234", + "args": [":link:"] + }, + "help_for_posts_list": { + "text": "{0} To get a backup copy of the list of posts, send links to posts in a list (each new link with a new message line). This message will be automatically split into a number of messages equal to the number of links and processed in the order of the queue.\n {1} Example:\nhttps://www.instagram.com/p/QwEr_tY1234\nhttps://www.instagram.com/p/QwEr_tY1235\nhttps://www.instagram.com/p/QwEr_tY1236", + "args": [":information:", ":link:"] + }, + "unknown_command": { + "text": "{0} Invalid button command. Please use inline keyboard.", + "args": [":warning:"] + }, + "url_error": { + "text": "{0} Validation error. The link is incorrect.", + "args": [":warning:", ":link:"] + } + } +} diff --git a/src/constants.py b/src/constants.py deleted file mode 100644 index 623529a4c..000000000 --- a/src/constants.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -This module contains the constants for this python project. -""" -import os -from telegram import TelegramBot -from users import UsersAuth -from messages import Messages -from vault import VaultClient -from modules.downloader import Downloader -from modules.uploader import Uploader - - -# environment variables -LOGGER_LEVEL = os.environ.get( - 'LOGGER_LEVEL', - 'INFO' -) - -BOT_NAME = os.environ.get( - 'BOT_NAME', - 'pyinstabot-downloader' -) - -MESSAGES_CONFIG = os.environ.get( - 'MESSAGES_CONFIG', - 'src/configs/messages.json' -) - -STORAGE_TYPE = os.environ.get( - 'STORAGE_TYPE', - 'mega' -) - -TEMPORARY_DIR = os.environ.get( - 'TEMPORARY_DIR', - 'tmp/' -) - -INSTAGRAM_SESSION = os.environ.get( - 'INSTAGRAM_SESSION', - '.session' -) - -INSTAGRAM_USERAGENT = os.environ.get( - 'INSTAGRAM_USERAGENT', - None -) - -STORAGE_EXCLUDE_TYPE = os.environ.get( - 'STORAGE_EXCLUDE_TYPE', - '.txt' -) - - -# instances -VAULT_CLIENT = VaultClient( - name=BOT_NAME -) - -TELEGRAM_CLIENT = TelegramBot( - vault=VAULT_CLIENT -) - -BOT = TELEGRAM_CLIENT.telegram_bot - -AUTH_CLIENT = UsersAuth( - vault=VAULT_CLIENT -) - -MESSAGES_GENERATOR = Messages( - config_path=MESSAGES_CONFIG -) - -DOWNLOADER_INSTANCE = Downloader( - auth={ - 'sessionfile': INSTAGRAM_SESSION - }, - settings={ - 'savepath': TEMPORARY_DIR, - 'useragent': INSTAGRAM_USERAGENT - }, - vault=VAULT_CLIENT -) - -UPLOADER_INSTANCE = Uploader( - storage={ - 'type': STORAGE_TYPE, - 'temporary': TEMPORARY_DIR, - 'cloud_root_path': BOT_NAME, - 'exclude_type': STORAGE_EXCLUDE_TYPE - }, - vault=VAULT_CLIENT -) diff --git a/src/migrations/0001_vault_historical_data.py b/src/migrations/0001_vault_historical_data.py new file mode 100644 index 000000000..a15e41103 --- /dev/null +++ b/src/migrations/0001_vault_historical_data.py @@ -0,0 +1,71 @@ +# pylint: disable=C0103,R0914 +""" +Migrates historical data from the Vault to the processed table in the database. +https://github.com/obervinov/pyinstabot-downloader/issues/30 +""" +VERSION = '1.0' +NAME = '0001_vault_historical_data' + + +def execute(obj): + """ + Migrates historical data from the Vault to the processed table in the database. + + Args: + obj: An obj containing the database connection and cursor, as well as the Vault instance. + + Returns: + None + """ + # database settings + table_name = 'processed' + columns = 'user_id, post_id, post_url, post_owner, link_type, message_id, chat_id, download_status, upload_status, state' + + # information about owners + try: + owners = obj.vault.list_secrets(path='history/') + owners_counter = len(owners) + print(f"Founded {owners_counter} owners in history") + + # reade history form Vault + for owner in owners: + # information about owner posts + posts = obj.vault.read_secret(path=f"history/{owner}") + posts_counter = len(posts) + print(f"{NAME}: Founded {posts_counter} posts in history/{owner}") + + for post in posts: + user_id = next(iter(obj.vault.read_secret(path='configuration/users').keys())) + post_id = post + post_url = f"https://www.instagram.com/p/{post}" + post_owner = owner + link_type = 'post' + message_id = 'unknown' + chat_id = next(iter(obj.vault.read_secret(path='configuration/users').keys())) + download_status = 'completed' + upload_status = 'completed' + state = 'processed' + + values = ( + f"'{user_id}', " + f"'{post_id}', " + f"'{post_url}', " + f"'{post_owner}', " + f"'{link_type}', " + f"'{message_id}', " + f"'{chat_id}', " + f"'{download_status}', " + f"'{upload_status}', " + f"'{state}'" + ) + + print(f"{NAME}: Migrating {post_id} from history/{owner}") + obj.cursor.execute(f"INSERT INTO {table_name} ({columns}) VALUES ({values})") + obj.database_connection.commit() + print(f"{NAME}: Post {post_id} from history/{owner} has been added to processed table") + print(f"{NAME}: Migration has been completed") + # Will be fixed after the issue https://github.com/obervinov/vault-package/issues/46 is resolved + # pylint: disable=broad-exception-caught + except Exception as migration_error: + print(f"{NAME}: Migration cannot be completed due to an error: {migration_error}") + print(f"{NAME}: Perhaps the history is empty or the Vault secrets path does not exist. It's not critical for the bot.") diff --git a/src/modules/database.py b/src/modules/database.py new file mode 100644 index 000000000..ddcd52cfa --- /dev/null +++ b/src/modules/database.py @@ -0,0 +1,809 @@ +"""This module contains a class for interacting with a PostgreSQL database using psycopg2""" +import os +import sys +import importlib +import json +from typing import Union +from datetime import datetime, timedelta +import psycopg2 +from logger import log +from .tools import get_hash + + +class DatabaseClient: + """ + A class that represents a client for interacting with a PostgreSQL database. + """ + def __init__( + self, + vault: object = None, + environment: str = None + ) -> None: + """ + Initializes a new instance of the Database client. + + Args: + vault (object): An object representing a HashiCorp Vault client for retrieving secrets with the database configuration. + environment (str): The environment to use for the database connection. + + Attributes: + database_connection (psycopg2.extensions.connection): A connection to the PostgreSQL database. + cursor (psycopg2.extensions.cursor): A cursor for executing SQL queries on the database. + vault (object): An object representing a HashiCorp Vault client for retrieving secrets. + + Parameters: + host (str): The hostname of the database server. + port (int): The port number of the database server. + user (str): The username to use when connecting to the database. + password (str): The password to use when connecting to the database. + database (str): The name of the database to connect to. + log (object): An object representing a logger for logging messages. + + Returns: + None + + Examples: + To create a new instance of the Database class: + >>> from modules.database import Database + >>> from modules.vault import Vault + >>> vault = Vault() + >>> db = Database(vault=vault) + """ + if environment: + db_configuration = vault.read_secret(path=f"configuration/database-{environment}") + else: + db_configuration = vault.read_secret(path='configuration/database') + + self.database_connection = psycopg2.connect( + host=db_configuration['host'], + port=db_configuration['port'], + user=db_configuration['user'], + password=db_configuration['password'], + database=db_configuration['database'] + ) + log.info( + '[class.%s] Database: connected to the database %s:%s/%s', + __class__.__name__, db_configuration['host'], db_configuration['port'], db_configuration['database'] + ) + + self.cursor = self.database_connection.cursor() + self.vault = vault + + self._prepare_db() + self._migrations() + + def _prepare_db(self) -> None: + """ + Prepare the database by creating and initializing the necessary tables. + + Args: + None + + Parameters: + None + + Returns: + None + """ + # Read configuration file for database initialization + configuration_path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../configs/databases.json')) + with open(configuration_path, encoding='UTF-8') as config_file: + database_init_configuration = json.load(config_file) + + # Create databases if does not exist + for table in database_init_configuration.get('Tables', None): + self._create_table( + table_name=table['name'], + columns="".join(f"{column}" for column in table['columns']) + ) + log.info('[class.%s] Prepare Database: create table `%s` (if does not exist)', __class__.__name__, table['name']) + + # Write necessary data to the database (service records) + if database_init_configuration.get('DataSeeding', None): + # ! This code block needs to be improved after some service data will appear for filling, + # ! because this code creates duplicate lines each time the project is started. + for data in database_init_configuration['DataSeeding']: + self._insert( + table_name=data['table'], + columns=tuple(data['data'].keys()), + values=tuple(data['data'].values()) + ) + log.info('[class.%s] Prepare Database: data seeding has been added to the `%s` table', __class__.__name__, data['table']) + + def _migrations(self) -> None: + """ + Execute database migrations to update the database schema or data. + + Args: + None + + Parameters: + None + + Returns: + None + """ + log.info('[class.%s] Database Migrations: Preparing to execute database migrations...', __class__.__name__) + # Migrations directory + migrations_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../migrations')) + sys.path.append(migrations_dir) + + for migration_file in os.listdir(migrations_dir): + if migration_file.endswith('.py'): + migration_module_name = migration_file[:-3] + + if not self._is_migration_executed(migration_name=migration_module_name): + log.info('[class.%s] Database Migrations: executing the %s migration...', __class__.__name__, migration_module_name) + migration_module = importlib.import_module(name=migration_module_name) + migration_module.execute(self) + version = getattr(migration_module, 'VERSION', migration_module_name) + self._mark_migration_as_executed(migration_name=migration_module_name, version=version) + else: + log.info( + '[class.%s] Database Migrations: the %s has already been executed and was skipped', + __class__.__name__, migration_module_name + ) + + def _is_migration_executed( + self, + migration_name: str = None + ) -> bool: + """ + Check if a migration has already been executed. + + Args: + migration_name (str): The name of the migration to check. + + Returns: + bool: True if the migration has been executed, False otherwise. + """ + self.cursor.execute(f"SELECT id FROM migrations WHERE name = '{migration_name}'") + return self.cursor.fetchone() is not None + + def _mark_migration_as_executed( + self, + migration_name: str = None, + version: str = None + ) -> None: + """ + Inserts a migration into the migrations table to mark it as executed. + + Args: + migration_name (str): The name of the migration to mark as executed. + + Returns: + None + """ + self.cursor.execute(f"INSERT INTO migrations (name, version) VALUES ('{migration_name}', '{version}')") + self.database_connection.commit() + + def _create_table( + self, + table_name: str = None, + columns: str = None + ) -> None: + """ + Create a new table in the database with the given name and columns if it does not already exist. + + Args: + table_name (str): The name of the table to create. + columns (str): A string containing the column definitions for the table. + + Returns: + None + + Examples: + To create a new table called 'users' with columns 'id' and 'name', you can call the method like this: + >>> _create_table('users', 'id INTEGER PRIMARY KEY, name TEXT') + """ + self.cursor.execute(f"CREATE TABLE IF NOT EXISTS {table_name} ({columns})") + self.database_connection.commit() + + def _insert( + self, + table_name: str = None, + columns: tuple = None, + values: tuple = None + ) -> None: + """ + Inserts a new row into the specified table with the given columns and values. + + Args: + table_name (str): The name of the table to insert the row into. + columns (tuple): A tuple containing the names of the columns to insert the values into. + values (tuple): A tuple containing the values to insert into the table. + + Returns: + None + + Examples: + >>> db_client._insert( + ... table_name='users', + ... columns=('username', 'email'), + ... values=('john_doe', 'john_doe@example.com') + ... ) + """ + try: + sql_query = f"INSERT INTO {table_name} ({', '.join(columns)}) VALUES ({', '.join(['%s'] * len(columns))})" + self.cursor.execute(sql_query, values) + self.database_connection.commit() + except (psycopg2.Error, IndexError) as error: + log.error( + '[class.%s] an error occurred while inserting a row into the table %s: %s\nColumns: %s\nValues: %s\nQuery: %s', + __class__.__name__, table_name, error, columns, values, sql_query + ) + + def _select( + self, + table_name: str = None, + columns: tuple = None, + **kwargs + ) -> Union[list, None]: + """ + Selects rows from the specified table with the given columns based on the specified condition. + + Args: + table_name (str): The name of the table to select data from. + columns (tuple): A tuple containing the names of the columns to select. + + Keyword Args: + condition (str): The condition to use to select the data. + order_by (str): The column to use for ordering the data. + limit (int): The maximum number of rows to return. + + Returns: + list: a list of tuples containing the selected data. + or + None: if no data is found. + + Examples: + >>> _select(table_name='users', columns=('username', 'email'), condition="id=1") + [('john_doe', 'john_doe@exmaple.com')] + """ + # base query + sql_query = f"SELECT {', '.join(columns)} FROM {table_name}" + + if kwargs.get('condition', None): + sql_query += f" WHERE {kwargs.get('condition')}" + if kwargs.get('order_by', None): + sql_query += f" ORDER BY {kwargs.get('order_by')}" + if kwargs.get('limit', None): + sql_query += f" LIMIT {kwargs.get('limit')}" + + self.cursor.execute(sql_query) + return self.cursor.fetchall() + + def _update( + self, + table_name: str = None, + values: str = None, + condition: str = None + ) -> None: + """ + Update the specified table with the given values of values based on the specified condition. + + Args: + table_name (str): The name of the table to update. + values (str): The values of values to update in the table. + condition (str): The condition to use for updating the table. + + Returns: + None + + Examples: + >>> _update('users', "username='new_username', password='new_password'", "id=1") + """ + self.cursor.execute(f"UPDATE {table_name} SET {values} WHERE {condition}") + self.database_connection.commit() + + def _delete( + self, + table_name: str = None, + condition: str = None + ) -> None: + """ + Delete rows from a table based on a condition. + + Args: + table_name (str): The name of the table to delete rows from. + condition (str): The condition to use to determine which rows to delete. + + Returns: + None + + Examples: + To delete all rows from the 'users' table where the 'username' column is 'john': + >>> db._delete('users', "username='john'") + """ + self.cursor.execute(f"DELETE FROM {table_name} WHERE {condition}") + self.database_connection.commit() + + def add_message_to_queue( + self, + data: dict = None + ) -> str: + """ + Add a message to the queue table in the database. + + Args: + data (dict): A dictionary containing the message details. + + Parameters: + user_id (str): The user ID of the message sender. + post_id (str): The ID of the post the message is related to. + post_url (str): The URL of the post the message is related to. + post_owner (str): The username of the post owner. + link_type (str): The type of link in the message. + message_id (str): The ID of the message. + chat_id (str): The ID of the chat the message belongs to. + scheduled_time (str): The time the message is scheduled to be sent. + download_status (str): The status of the post downloading process. + upload_status (str): The status of the post uploading process. + + Returns: + str: A message indicating that the message was added to the queue. + + Examples: + >>> data = { + ... 'user_id': '12345', + ... 'post_id': '67890', + ... 'post_url': 'https://www.instagram.com/p/67890/', + ... 'post_owner': 'johndoe', + ... 'link_type': 'profile', + ... 'message_id': 'abcde', + ... 'chat_id': 'xyz', + ... 'scheduled_time': '2022-01-01 12:00:00' + ... 'download_status': 'not started', + ... 'upload_status': 'not started' + ... } + >>> database.add_message_to_queue(data=data) + 'abcde: added to queue' + """ + self._insert( + table_name='queue', + columns=( + "user_id", + "post_id", + "post_url", + "post_owner", + "link_type", + "message_id", + "chat_id", + "scheduled_time", + "download_status", + "upload_status" + ), + values=( + data.get('user_id', None), + data.get('post_id', None), + data.get('post_url', None), + data.get('post_owner', None), + data.get('link_type', None), + data.get('message_id', None), + data.get('chat_id', None), + data.get('scheduled_time', None), + data.get('download_status', 'not started'), + data.get('upload_status', 'not started'), + ) + ) + return f"{data.get('message_id', None)}: added to queue" + + def get_message_from_queue( + self, + scheduled_time: str = None + ) -> tuple: + """ + Get a one message from the queue table that is scheduled to be sent at the specified time. + The message will be returned before or equal to the specified timestamp in the argument. + + Args: + scheduled_time (str): The time at which the message is scheduled to be sent. + + Returns: + tuple: A tuple containing the message from the queue. + + Examples: + >>> database.get_message_from_queue('2022-01-01 12:00:00') + (1, '123456789', 'vahj5AN8aek', 'https://www.instagram.com/p/vahj5AN8aek', 'johndoe', 'post', '12345', '12346', '123456789', + datetime.datetime(2023, 11, 14, 21, 21, 22, 603440), 'None', 'None', datetime.datetime(2023, 11, 14, 21, 14, 26, 680024), 'waiting') + """ + message = self._select( + table_name='queue', + columns=("*",), + condition=f"scheduled_time <= '{scheduled_time}' AND state IN ('waiting', 'processing')", + limit=1 + ) + return message[0] if message else None + + def update_message_state_in_queue( + self, + post_id: str = None, + state: str = None, + **kwargs + ) -> str: + """ + Update the state of a message in the queue table and move it to the processed table if the state is 'processed'. + + Args: + post_id (str): The ID of the post. + state (str): The new state of the message. + + Keyword Arguments: + download_status (str): The status of the post downloading process. + upload_status (str): The status of the post uploading process. + post_owner (str): The ID of the post owner. + + Parameters: + table_name (str): The name of the table to update. + values (str): The new value for the state column. + condition (str): The condition to use to select the row to update. + + Returns: + str: A response message indicating the status of the update. + + Examples: + >>> database.update_message_state_in_queue( + post_id='123', + state='processed', + download_status='completed', + upload_status='completed', + post_owner='username123' + ) + '456: processed' + """ + values = f"state = '{state}'" + + if kwargs.get('post_owner'): + values += f", post_owner = '{kwargs.get('post_owner')}'" + if kwargs.get('download_status'): + values += f", download_status = '{kwargs.get('download_status')}'" + if kwargs.get('upload_status'): + values += f", upload_status = '{kwargs.get('upload_status')}'" + + self._update(table_name='queue', values=values, condition=f"post_id = '{post_id}'") + + if state == 'processed': + processed_message = self._select( + table_name='queue', + columns=("*",), + condition=f"post_id = '{post_id}'", + limit=1 + ) + self._insert( + table_name='processed', + columns=( + "user_id", + "post_id", + "post_url", + "post_owner", + "link_type", + "message_id", + "chat_id", + "download_status", + "upload_status", + "state" + ), + values=( + processed_message[0][1], + processed_message[0][2], + processed_message[0][3], + processed_message[0][4], + processed_message[0][5], + processed_message[0][6], + processed_message[0][7], + kwargs.get('download_status', 'pending'), + kwargs.get('upload_status', 'pending'), + state + ) + ) + self._delete(table_name='queue', condition=f"post_id = '{post_id}'") + response = f"{processed_message[0][6]}: processed" + else: + response = f"{post_id}: state updated" + + return response + + def verify_users_queue(self) -> None: + """ + Verify the queue for all users and reschedule messages if necessary. + If the message is not processed in time (for example, the bot was down), reschedule the time of the message processing. + + Args: + None + + Returns: + None + + Examples: + >>> verify_users_queue() + """ + log.info("[class.%s] Database: verifying the message of users in queue...", __class__.__name__) + users = self.get_users() + + for user in users: + user_id = user[0] + need_reschedule = False + full_queue = self._select( + table_name='queue', + columns=("id", "scheduled_time"), + condition=f"user_id = '{user_id}'", + order_by='scheduled_time ASC', + limit=1000 + ) + + for message in full_queue: + if message[1] < datetime.now() - timedelta(minutes=10): + need_reschedule = True + log.warning( + "[class.%s] Database: found a message in the queue that was not processed in time for user %s", + __class__.__name__, user_id + ) + break + + if need_reschedule: + log.warning("[class.%s] Database: rescheduling messages in the queue for user %s", __class__.__name__, user_id) + # The lag between the current time and the scheduled time of the message in the seconds + lag = None + # The difference in minutes between the current message and the previous message in the seconds + diff = None + # The new scheduled time for the message after rescheduling + new_schedule_time = None + # The previous scheduled time of the message for calculate the skew between the messages. For keep rate limit. + previous_schedule_time = None + # Reschedule the all messages in the queue + for message in full_queue: + schedule_time = message[1] + lag = (datetime.now() - schedule_time).total_seconds() + + # If haven't previous message value for compare difference between the messages + if not previous_schedule_time: + new_schedule_time = datetime.now() + self._update( + table_name='queue', + values=f"scheduled_time = '{new_schedule_time}'", + condition=f"id = '{message[0]}'" + ) + else: + diff = (schedule_time - previous_schedule_time).total_seconds() + # Add the difference in minutes between the current message and the previous message to the lag + skew = diff + lag + new_schedule_time = datetime.now() + timedelta(seconds=skew) + self._update( + table_name='queue', + values=f"scheduled_time = '{new_schedule_time}'", + condition=f"id = '{message[0]}'" + ) + previous_schedule_time = schedule_time + log.info("[class.%s] Database: rescheduled message %s: %s -> %s", __class__.__name__, message[0], message[1], new_schedule_time) + log.info("[class.%s] Database: users queue verification completed", __class__.__name__) + + def get_user_queue( + self, + user_id: str = None + ) -> Union[dict, None]: + """ + Get messages from the queue table for the specified user. + + Args: + user_id (str): The ID of the user. + + Returns: + dict: A dictionary containing messages from the queue for the specified user. + + Examples: + >>> get_user_queue(user_id='12345') + {'12345': [{'post_id': '123456789', 'scheduled_time': '2022-01-01 12:00:00'}]} + """ + result = {} + queue = self._select( + table_name='queue', + columns=("post_id", "scheduled_time"), + condition=f"user_id = '{user_id}'", + limit=1000 + ) + for message in queue: + if user_id not in result: + result[user_id] = [] + result[user_id].append({'post_id': message[0], 'scheduled_time': message[1]}) + return result if result else None + + def get_user_processed( + self, + user_id: str = None + ) -> Union[dict, None]: + """ + Get last ten messages from the processed table for the specified user. + It is used to display the last messages sent by the bot to the user. + + Args: + user_id (str): The ID of the user. + + Returns: + dict: A dictionary containing the last five messages from the processed table for the specified user. + + Examples: + >>> get_user_processed(user_id='12345') + {'12345': [{'post_id': '123456789', 'processed_time': '2022-01-01 12:00:00', 'state': 'completed'}]} + """ + result = {} + processed = self._select( + table_name='processed', + columns=("post_id", "timestamp", "state"), + condition=f"user_id = '{user_id}'", + order_by='timestamp DESC', + limit=10 + ) + for message in processed: + if user_id not in result: + result[user_id] = [] + result[user_id].append({'post_id': message[0], 'timestamp': message[1], 'state': message[2]}) + return result if result else None + + def check_message_uniqueness( + self, + post_id: str = None, + user_id: str = None + ) -> bool: + """ + Check if a message with the given post ID and chat ID already exists in the queue. + + Args: + post_id (str): The ID of the post. + user_id (str): The ID of the chat. + + Returns: + bool: True if the message is unique, False otherwise. + + Examples: + >>> check_message_uniqueness(post_id='12345', user_id='67890') + True + """ + queue = self._select( + table_name='queue', + columns=("id",), + condition=f"post_id = '{post_id}' AND user_id = '{user_id}'", + limit=1 + ) + processed = self._select( + table_name='processed', + columns=("id",), + condition=f"post_id = '{post_id}' AND user_id = '{user_id}'", + limit=1 + ) + if queue or processed: + return False + return True + + def keep_message( + self, + message_id: str = None, + chat_id: str = None, + message_type: str = None, + message_content: Union[str, dict] = None + ) -> str: + """ + Add a message to the messages table in the database. + It is used to store the last message sent to the user for updating the message in the future. + + Args: + message_id (str): The ID of the message. + chat_id (str): The ID of the chat. + message_type (str): The type of the message. + message_content (Union[str, dict]): The content of the message. + + Returns: + str: A message indicating that the message was added to the messages table. + + Examples: + >>> keep_message('12345', '67890', 'status_message', 'Hello, username\n...') + '12345 kept' or '12345 updated' + """ + message_content_hash = get_hash(message_content) + check_exist_message_type = self._select( + table_name='messages', + columns=("id", "message_id"), + condition=f"message_type = '{message_type}' AND chat_id = '{chat_id}'", + ) + if check_exist_message_type: + self._update( + table_name='messages', + values=( + f"message_content_hash = '{message_content_hash}', " + f"message_id = '{message_id}', " + f"timestamp = CURRENT_TIMESTAMP" + ), + condition=f"id = '{check_exist_message_type[0][0]}'" + ) + response = f"{message_id} updated" + else: + self._insert( + table_name='messages', + columns=("message_id", "chat_id", "message_type", "message_content_hash", "producer"), + values=(message_id, chat_id, message_type, message_content_hash, 'bot') + ) + response = f"{message_id} kept" + return response + + def add_user( + self, + user_id: str = None, + chat_id: str = None + ) -> str: + """ + Add a user to the users table in the database. + It is used to store the user ID and chat ID for sending messages to the user. + + Args: + user_id (str): The ID of the user. + chat_id (str): The ID of the chat. + + Returns: + str: A message indicating that the user was added to the users table or that the user already exists. + + Examples: + >>> add_user(user_id='12345', chat_id='67890') + '12345 added' + or + '12345 already exists' + """ + exist_user = self._select(table_name='users', columns=("user_id",), condition=f"user_id = '{user_id}'") + if exist_user and user_id in exist_user[0]: + result = f"{user_id} already exists" + else: + self._insert( + table_name='users', + columns=("chat_id", "user_id"), + values=(chat_id, user_id) + ) + result = f"{user_id} added" + return result + + def get_users(self) -> list: + """ + Get a list of all users in the database. + + Args: + None + + Returns: + list: A list of all users from the messages table. + + Examples: + >>> get_users() + # [('{user_id}', '{chat_id}')] + [('12345', '67890')] + """ + users = self._select( + table_name='users', + columns=("user_id", "chat_id"), + limit=1000 + ) + return users if users else None + + def get_considered_message( + self, + message_type: str = None, + chat_id: str = None + ) -> str: + """ + Get a message with specified type and chat ID from the messages table in the database. + + Args: + message_type (str): The type of the message. + chat_id (str): The ID of the chat. + + Returns: + tuple: A tuple containing the message from the messages table. + + Examples: + >>> current_message_id(message_type='status_message', chat_id='12345') + # ('message_id', 'chat_id', 'timestamp', 'message_content_hash') + ('123456789', '12345', datetime.datetime(2023, 11, 14, 21, 14, 26, 680024), '2ef7bde608ce5404e97d5f042f95f89f1c232871d3d7') + """ + message = self._select( + table_name='messages', + columns=("message_id", "chat_id", "timestamp", "message_content_hash",), + condition=f"message_type = '{message_type}' AND chat_id = '{chat_id}'", + limit=1 + ) + return message[0] if message else None diff --git a/src/modules/downloader.py b/src/modules/downloader.py index a6f2adad5..5a372d902 100644 --- a/src/modules/downloader.py +++ b/src/modules/downloader.py @@ -1,217 +1,149 @@ +# pylint: disable=duplicate-code """ -This module interacts with the instagram api and uploads content to a temporary local directory. -Supports downloading the content of the post by link, -the entire content of posts in the account, -getting information about the account -and saving the history of already downloaded messages in the vault. +This module interacts with the instagram api and uploads content to a temporary directory. +Supports downloading the content of the post by link and getting information about the account. https://instaloader.github.io/module/instaloader.html """ -import os +from typing import Union +from ast import literal_eval +import base64 import instaloader from logger import log +from .exceptions import WrongVaultInstance, FailedCreateDownloaderInstance, FailedAuthInstaloader +# pylint: disable=too-few-public-methods class Downloader: """ - The Instagram api instance is created by this class - and contains a set of all the necessary posts - for uploading content from Instagram accounts to local storage. + An Instagram API instance is created by this class and contains a set of all the necessary methods + to upload content from Instagram to a temporary directory. """ - def __init__( self, - auth: dict = None, - settings: dict = None, + configuration: dict = None, vault: object = None ) -> None: """ - The method for create a new instagram api client instance. + The method for create a new Instagram API client instance. Args: - :param auth (dict): dictionary with authorization parameters. + :param configuration (dict): dictionary with configuration parameters for Instagram API communication. :param username (str): username for authentication in the instagram api. :param password (str): password for authentication in the instagram api. - :param sessionfile (str): the path to the session file of the instagram. - :param anonymous (bool): access to open profiles without logging in to an account. - only for tests. - :param settings (dict): dictionary with settings instaloader parameters. - :param savepath (str): local directory for saving downloaded content. - :param useragent (str): user-agent header. - :param vault (object): instance of vault for recording or reading download history. + :param login-method (str): method for authentication in the instagram api. Can be: 'session', 'password', 'anonymous'. + :param session-file (str): the path for saving the session file. + :param user-agent (str): user-agent header. + :param fatal-status-codes (list): list of fatal status codes, this causes the thread executing this module's code to crash. + :param iphone-support (bool): enable or disable iphone http headers. + :param session-base64 (str): base64 encoded session file. + :param vault (object): instance of vault for reading configuration downloader-api. Returns: None + Attributes: + :attribute configuration (dict): dictionary with configuration parameters for instagram api communication. + :attribute instaloader (object): instance of the instaloader class for working + Examples: - >>> DOWNLOADER_INSTANCE = Downloader( - auth={ - 'anonymous': true - }, - settings={ - 'savepath': TEMPORARY_DIR, - 'useragent': INSTAGRAM_USERAGENT - }, - vault=VAULT_CLIENT - ) - >>> DOWNLOADER_INSTANCE = Downloader( - auth={ - 'sessionfile': INSTAGRAM_SESSION - }, - settings={ - 'savepath': TEMPORARY_DIR, - 'useragent': INSTAGRAM_USERAGENT - }, - vault=VAULT_CLIENT - ) - >>> DOWNLOADER_INSTANCE = Downloader( - auth={ - 'username': INSTAGRAM_USERNAME, - 'password': INSTAGRAM_PASSWORD - }, - settings={ - 'savepath': TEMPORARY_DIR, - 'useragent': INSTAGRAM_USERAGENT - }, - vault=VAULT_CLIENT - ) + >>> configuration = { + ... 'username': 'my_username', + ... 'password': 'my_password', + ... 'login-method': 'session', + ... 'session-file': '/path/to/session/file', + ... 'session-base64': '', + ... 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3' + ... } + >>> vault = Vault() + >>> downloader = Downloader(configuration, vault) """ - self.auth = auth - self.settings = settings - self.vault = vault + if not vault: + raise WrongVaultInstance("Wrong vault instance, you must pass the vault instance to the class argument.") + + if configuration: + self.configuration = configuration + elif not configuration: + self.configuration = vault.read_secret(path='configuration/downloader-api') + else: + raise FailedCreateDownloaderInstance( + "Failed to initialize the Downloader instance." + "Please check the configuration in class argument or the secret with the configuration in the Vault." + ) + log.info('[class.%s] Downloader: creating a new instance of the Downloader...', __class__.__name__) self.instaloader = instaloader.Instaloader( quiet=True, - user_agent=self.settings['useragent'], - dirname_pattern=f"{settings['savepath']}/{{profile}}", + user_agent=self.configuration.get('user-agent', None), + iphone_support=self.configuration.get('iphone-support', None), + dirname_pattern='data/{profile}', filename_pattern='{profile}_{shortcode}_{filename}', download_pictures=True, download_videos=True, - download_video_thumbnails=True, + download_video_thumbnails=False, + download_geotags=False, + download_comments=False, save_metadata=False, - compress_json=True, + compress_json=False, post_metadata_txt_pattern=None, storyitem_metadata_txt_pattern=None, check_resume_bbd=True, - fatal_status_codes=[400, 401, 429, 500] + fatal_status_codes=literal_eval(self.configuration.get('fatal-status-codes', '[]')) ) - - if self.auth.get('anonymous'): - auth_status = self._login( - method='anonymous' - ) - elif os.path.exists( - self.auth.get('sessionfile') - ): - auth_status = self._login( - method='session' - ) - else: - auth_status = self._login( - method='password' - ) + auth_status = self._login() log.info( - '[class.%s] downloader instance init with account %s: %s', - __class__.__name__, - self.auth['username'], - auth_status + '[class.%s] Downloader: downloader instance created successfully: %s in %s', + __class__.__name__, auth_status, self.configuration['username'] ) - def _login( - self, - method: str = None - ) -> str | None: + def _login(self) -> Union[str, None]: """ - The method for authentication in instagram api. + The method for authentication in Instagram API. Args: - :param method (str): authentication method 'password', 'session' or 'anonymous'. - + None Returns: - (str) success + (str) logged_in or None """ - if not self.auth.get('username') or not self.auth.get('anonymous'): - self.auth['username'] = self.vault.read_secret( - 'configuration/instagram', - 'username' - ) - - if method == 'session': + if self.configuration['login-method'] == 'session': + # If session-base64 defined in the configuration, then decode and save the session file. + if self.configuration.get('session-base64', None): + with open(self.configuration['session-file'], 'wb') as file: + file.write(base64.b64decode(self.configuration['session-base64'])) + # Otherwise, it expects the session file to be in the specified path. self.instaloader.load_session_from_file( - self.auth['username'], - self.auth['sessionfile'] - ) - log.info( - '[class.%s] session file was load success', - __class__.__name__ + self.configuration['username'], + self.configuration['session-file'] ) - return 'success' + log.info('[class.%s] Downloader: session file %s was load success', __class__.__name__, self.configuration['session-file']) + return 'logged_in' - if method == 'password': - if not self.auth.get('password'): - self.auth['password'] = self.vault.read_secret( - 'configuration/instagram', - 'password' - ) + if self.configuration['login-method'] == 'password': self.instaloader.login( - self.auth['username'], - self.auth['password'] - ) - self.instaloader.save_session_to_file( - self.auth['sessionfile'] + self.configuration['username'], + self.configuration['password'] ) + self.instaloader.save_session_to_file(self.configuration['session-file']) log.info( - '[class.%s] login with password was successful. Save session in %s', - __class__.__name__, - self.auth['sessionfile'] + '[class.%s] Downloader: login with password was successful. Save session in %s', + __class__.__name__, self.configuration['sessionfile'] ) - return 'success' + return 'logged_in' - if method == 'anonymous': - log.warning( - '[class.%s] initialization without logging into an account (anonymous)', - __class__.__name__ - ) + if self.configuration['login-method'] == 'anonymous': + log.warning('[class.%s] Downloader: initialization without authentication into an account (anonymous)', __class__.__name__) return None - return None - - def get_posts( - self, - username: str = None - ) -> list | None: - """ - The method for getting a list posts of instagram account. - - Args: - :param username (str): instagram account profile name. - - Returns: - (list) ['post_id_1', 'post_id_2', 'post_id_3'] - or - None - """ - posts_list = [] - profile = instaloader.Profile.from_username( - self.instaloader.context, - username - ) - log.info( - '[class.%s] the %s profile was read success', - __class__.__name__, - username + raise FailedAuthInstaloader( + "Failed to authenticate the Instaloader instance. Please check the configuration in the Vault or the class argument." ) - for post in profile.get_posts(): - posts_list.append(post.shortcode) - - return posts_list def get_post_content( self, shortcode: str = None - ) -> dict | None: + ) -> Union[dict, None]: """ - The method for getting the content of a post from a specified Instagram account. + The method for getting the content of a post from a specified Post ID. Args: :param shortcode (str): the ID of the record for downloading content. @@ -221,94 +153,17 @@ def get_post_content( 'post': shortcode, 'owner': post.owner_username, 'type': post.typename, - 'status': 'downloaded' + 'status': 'completed' } """ - post = instaloader.Post.from_shortcode( - self.instaloader.context, - shortcode - ) + log.info('[class.%s] Downloader: downloading the contents of the post %s...', __class__.__name__, shortcode) + post = instaloader.Post.from_shortcode(self.instaloader.context, shortcode) self.instaloader.download_post(post, '') - log.info( - '[class.%s] the contents of the %s have been successfully downloaded ' - 'to the temporary storage', - __class__.__name__, - shortcode - ) - self.vault.write_secret( - f'history/{post.owner_username}', - shortcode, - "downloaded" - ) - return { + log.info('[class.%s] Downloader: the contents of the post %s have been successfully downloaded', __class__.__name__, shortcode) + metadata = { 'post': shortcode, 'owner': post.owner_username, 'type': post.typename, - 'status': 'downloaded' - } - - def get_download_info( - self, - account: str = None - ) -> dict | None: - """ - The method for collecting all the necessary information - to download all posts from the specified account. - Checks the history of already uploaded posts - and provides information for cyclic downloading. - - Args: - :param account (str): instagram account name to check the uploaded history. - - Returns: - (dict) { - "shortcodes_for_download": fresh_shortcodes, - "shortcodes_total_count": len(account_shortcodes), - "shortcodes_exist": len(history_shortcodes), - "shortcodes_exist_count": len(history_shortcodes.keys()) - } - """ - log.info( - '[class.%s] excluding shortcodes that are already downloaded...', - __class__.__name__ - ) - # account_shortcodes - list of shortcodes received from instagram - account_shortcodes = self.get_posts( - username=account - ) - # fresh_shortcodes - list of shortcodes that have not been downloaded yet - fresh_shortcodes = [] - # history_shortcodes - dict of shortcodes that have already been previously uploaded - try: - history_shortcodes = self.vault.read_secret( - f'history/{account}' - ) - # pylint: disable=W0718 - # will be fixed after https://github.com/obervinov/vault-package/issues/31 - except Exception as secret_not_found: - history_shortcodes = {} - log.warning( - '[class.%s] secret history/%s does not exist: %s', - __class__.__name__, - account, - secret_not_found - ) - for shortcode in account_shortcodes: - if shortcode not in history_shortcodes.keys(): - fresh_shortcodes.append(shortcode) - log.info( - '[class.%s] account metadata:\n' - 'already downloaded shortcodes: %s\n' - 'fresh shortcodes: %s\n' - 'shortcodes for download: %s', - __class__.__name__, - history_shortcodes, - account_shortcodes, - fresh_shortcodes - ) - return { - "shortcodes_for_download": fresh_shortcodes, - "shortcodes_total_count": len(account_shortcodes), - "shortcodes_exist": len(history_shortcodes), - "shortcodes_exist_count": len(history_shortcodes.keys()) + 'status': 'completed' } + return metadata diff --git a/src/modules/exceptions.py b/src/modules/exceptions.py new file mode 100644 index 000000000..04f7c4cdb --- /dev/null +++ b/src/modules/exceptions.py @@ -0,0 +1,57 @@ +""" +This module contains custom exceptions that are used in the application. +""" + + +class FailedMessagesStatusUpdater(Exception): + """ + Exception raised when the status of the messages could not be updated. + """ + def __init__(self, message): + self.message = message + super().__init__(self.message) + + +class FailedCreateDownloaderInstance(Exception): + """ + Exception raised when the downloader instance could not be initialized. + """ + def __init__(self, message): + self.message = message + super().__init__(self.message) + + +class FailedInitUploaderInstance(Exception): + """ + Exception raised when the uploader instance could not be initialized. + """ + def __init__(self, message): + self.message = message + super().__init__(self.message) + + +class WrongVaultInstance(Exception): + """ + Exception raised when the vault instance is not correct. + """ + def __init__(self, message): + self.message = message + super().__init__(self.message) + + +class FailedAuthInstaloader(Exception): + """ + Exception raised when the authentication of the Instaloader instance failed. + """ + def __init__(self, message): + self.message = message + super().__init__(self.message) + + +class WrongStorageType(Exception): + """ + Exception raised when the storage type is not correct. + """ + def __init__(self, message): + self.message = message + super().__init__(self.message) diff --git a/src/modules/tools.py b/src/modules/tools.py new file mode 100644 index 000000000..751fc5877 --- /dev/null +++ b/src/modules/tools.py @@ -0,0 +1,24 @@ +"""This module contains the tools for this python project""" +from typing import Union +import hashlib + + +def get_hash(data: Union[str, dict] = None) -> str: + """ + Get a hash of the input data. + + Args: + data (Union[str, dict]): The data to hash. + + Returns: + str: A hash of the content. + + Examples: + >>> get_hash('Hello, world!') + '2ef7bde608ce5404e97d5f042f95f89f1c232871d3d7' + """ + hasher = hashlib.sha256() + if isinstance(data, dict): + data = str(data) + hasher.update(data.encode('utf-8')) + return hasher.hexdigest() diff --git a/src/modules/uploader.py b/src/modules/uploader.py index 5280e6303..945b19782 100644 --- a/src/modules/uploader.py +++ b/src/modules/uploader.py @@ -1,90 +1,93 @@ +# pylint: disable=R0801 """ This module processes the content uploaded from Instagram and uploads the found media files (image, video) to the destination storage. """ import os +from typing import Union import dropbox from mega import Mega from logger import log +from .exceptions import WrongVaultInstance, FailedInitUploaderInstance, WrongStorageType class Uploader: """ This class creates an instance with a connection - to the target storage for uploading local media content. + to the target storage for uploading media content. """ - def __init__( self, - storage: dict = None, + configuration: dict = None, vault: object = None ) -> None: """ - The method creates an instance with a connection - to the target storage for uploading local media content. + The method creates an instance with a connection to the target storage for uploading media content. Args: - :param storage (dict): dictionary with storage parameters. - :param type (str): type of storage for uploading content - 'local' or 'dropbox' or 'mega'. - :param temporary (str): the temporary directory from which you want - to read the content and delete it after uploading. - :param cloud_root_path (str): a subdirectory in the cloud storage for saving content + :param configuration (dict): dictionary with target storage parameters. + :param username (str): username for authentication in the target storage. + :param password (str): password for authentication in the target storage. + :param storage-type (str): type of storage for uploading content. Can be: 'dropbox', 'mega'. + :param exclude-types (str): exclude files with this type from uploading. Example: '.json, .txt'. + :param source-directory (str): the path to the local directory with media content for uploading. + :param destination-directory (str): a subdirectory in the cloud storage where the content will be uploaded. :param vault (object): instance of vault for reading authorization data. Returns: None Examples: - >>> UPLOADER_INSTANCE = Uploader( - storage={ - 'type': STORAGE_TYPE, - 'temporary': TEMPORARY_DIR, - 'cloud_root_path': BOT_NAME, - 'exclude_type': STORAGE_EXCLUDE_TYPE - }, - vault=VAULT_CLIENT - ) + >>> configuration = { + ... 'username': 'my_username', + ... 'password': 'my_password', + ... 'storage-type': 'dropbox', + ... 'exclude-types': '.json, .txt', + ... 'source-directory': '/path/to/source/directory', + ... 'destination-directory': '/path/to/destination/directory' + ... } + >>> vault = Vault() + >>> uploader = Uploader(configuration, vault) """ - self.storage = storage - self.temporary_dir = f"{os.getcwd()}/{self.storage['temporary']}" - self.vault = vault - - log.info( - '[class.%s] uploader instance init with "%s" target storage', - __class__.__name__, - storage['type'] - ) - - if self.storage['type'] == 'dropbox': - self.dropbox_client = dropbox.Dropbox( - oauth2_access_token=self.vault.read_secret( - 'configuration/dropbox', - 'token' - ), - timeout=60 - ) - - if self.storage['type'] == 'mega': - self.mega_client = Mega().login( - self.vault.read_secret( - 'configuration/mega', - 'username' - ), - self.vault.read_secret( - 'configuration/mega', - 'password' - ) + if not vault: + raise WrongVaultInstance("Wrong vault instance, you must pass the vault instance to the class argument.") + + if configuration: + self.configuration = configuration + elif not configuration: + self.configuration = vault.read_secret(path='configuration/uploader-api') + else: + raise FailedInitUploaderInstance( + "Failed to initialize the Uploader instance." + "Please check the configuration in class argument or the secret with the configuration in the Vault." ) + self.local_directory = f"{os.getcwd()}/{self.configuration['source-directory']}" + self.storage = self._init_storage_connection() self._check_incomplete_transfers() - def _check_incomplete_transfers( - self, - ) -> None: + def _init_storage_connection(self) -> object: + """ + The method for initializing a connection to the target storage. + + Args: + None + + Returns: + (object) connection object to the target storage. + """ + if self.configuration['storage-type'] == 'dropbox': + return dropbox.Dropbox(oauth2_access_token=self.configuration['password']) + + if self.configuration['storage-type'] == 'mega': + mega = Mega() + return mega.login(email=self.configuration['username'], password=self.configuration['password']) + + raise WrongStorageType("Wrong storage type, please check the configuration. 'storage-type' can be: 'dropbox', 'mega'.") + + def _check_incomplete_transfers(self) -> None: """ - The method for checking uploads in temp storage - that for some reason could not be uploaded to the cloud. + The method for checking uploads in temp directory that for some reason could not be uploaded to the target cloud storage. Args: None @@ -92,92 +95,62 @@ def _check_incomplete_transfers( Returns: None """ - log.info( - '[class.%s] checking the pending uploads in the temporary directory ...', - __class__.__name__ - ) - - for _, artifacts, _ in os.walk(self.temporary_dir): - for artifact in artifacts: - log.warning( - '[class.%s] an unloaded artifact was found %s', - __class__.__name__, - artifact - ) - self.start_upload( - os.path.join(artifact) - ) - - def start_upload( + log.info('[class.%s] Uploader: checking incomplete transfers in the temporary directory...', __class__.__name__) + for root, dirs, _ in os.walk(self.configuration['source-directory']): + for dir_name in dirs: + sub_directory = os.path.join(root, dir_name) + # Check the subdirectory for files + sub_files = [f for f in os.listdir(sub_directory) if os.path.isfile(os.path.join(sub_directory, f))] + if sub_files: + log.warning('[class.%s] Uploader: an unloaded artifact was found: %s', __class__.__name__, sub_directory) + self.run_transfers(sub_directory=sub_directory) + else: + log.info('[class.%s] Uploader: remove of an empty directory %s', __class__.__name__, sub_directory) + os.rmdir(sub_directory) + + def run_transfers( self, - sub_dir_name: str = None - ) -> dict: + sub_directory: str = None + ) -> str: """ - The method of preparing media files for transfer to the target storage (cloud or local). + External entrypoint method for uploading media files to the target cloud storage. Args: - :param sub_dir_name (str): the name of the subdirectory where the content is located - is equivalent to the record ID. + :param sub_directory (str): the name of the subdirectory in the source directory with media content. Returns: - (dict) { - '/root/path/shortcode/file1.jpeg': 'uploaded', - '/root/path/shortcode/file2.jpeg': None - } - - (explanation of values) - (str) 'uploaded' - (this means that the file has been successfully uploaded to the cloud) - (str) 'None' - (this means that an error has occurred the file is not uploaded to the cloud) - (str) 'saved' - (this means that the file must remain in the local (temporary directory)) - (and it is not required to perform any actions with it) + (str) 'completed' + (this means that the file has been successfully uploaded to the cloud) + (str) 'not_completed' + (this means that an error has occurred the file is not uploaded to the cloud) """ transfers = {} - - log.info( - '[class.%s] preparing media files for transfer to the "%s"', - __class__.__name__, - self.storage['type'] - ) - - for root, _, files in os.walk( - f'{self.temporary_dir}{sub_dir_name}' - ): + result = "" + log.info('[class.%s] Uploader: preparing media files for transfer to the %s cloud...', __class__.__name__, self.configuration['storage-type']) + for root, _, files in os.walk(f"{self.configuration['source-directory']}{sub_directory}"): for file in files: - if self.storage['exclude_type'] and self.storage['exclude_type'] in file: - os.remove( - os.path.join(root, file) - ) + if file.split('.')[-1] in self.configuration.get('exclude-types', None): + os.remove(os.path.join(root, file)) else: - transfers[file] = self.file_upload( - os.path.join(root, file), - sub_dir_name + transfers[file] = self.upload_to_cloud( + source=os.path.join(root, file), + destination=root.split('/')[1] ) if transfers[file] == 'uploaded': - os.remove( - os.path.join(root, file) - ) - - if len(os.listdir(f'{self.temporary_dir}{sub_dir_name}')) == 0: - os.rmdir(f'{self.temporary_dir}{sub_dir_name}') - - log.info( - '[class.%s] All TRANSFERS: %s', - __class__.__name__, - transfers - ) - return transfers - - def file_upload( + os.remove(os.path.join(root, file)) + result = 'completed' + else: + result = 'not_completed' + log.info('[class.%s] Uploader: list of all transfers %s', __class__.__name__, transfers) + return result + + def upload_to_cloud( self, source: str = None, destination: str = None - ) -> str | None: + ) -> Union[str, None]: """ - The method of uploading the contents of the target directory - to the cloud or local directory. + The method of uploading the contents of the source directory to the target cloud storage. Args: :param source (str): the path to the local file to transfer to the target storage. @@ -188,68 +161,28 @@ def file_upload( or None """ - log.info( - '[class.%s] starting upload file %s to %s//:%s', - __class__.__name__, - source, - self.storage['type'], - destination - ) - - if self.storage['type'] == "local": - return "saved" - - if self.storage['type'] == 'mega': - directory = f"{self.storage['cloud_root_path']}/{destination}" - try: - mega_folder = self.mega_client.find( - directory, - exclude_deleted=True - ) - if not mega_folder: - self.mega_client.create_folder( - directory - ) - response = self.mega_client.upload( - source, - mega_folder[0] - ) - log.info( - '[class.%s] %s successful transferred', - __class__.__name__, - response - ) - return "uploaded" - - # pylint: disable=W0718 - # because the mega library does not contain exceptions - except Exception as mega_exception: - log.error( - '[class.%s] error when uploading via the mega api: %s', - __class__.__name__, - mega_exception - ) - log.warning( - '[class.%s] trying again file_upload()', - __class__.__name__, - ) - self.file_upload( - source, - destination, - ) - - if self.storage['type'] == 'dropbox': + log.info('[class.%s] starting upload file %s to %s://%s', __class__.__name__, source, self.configuration['storage-type'], destination) + response = None + result = None + + if self.configuration['storage-type'] == 'mega': + directory = f"{self.configuration['destination-directory']}/{destination}" + log.info('[class.%s] Uploader: trying found mega folder %s...', __class__.__name__, directory) + mega_folder = self.storage.find(directory, exclude_deleted=True) + if not mega_folder: + self.storage.create_folder(directory) + mega_folder = self.storage.find(directory, exclude_deleted=True) + log.info('[class.%s] Uploader: mega folder not found, created new folder %s', __class__.__name__, mega_folder) + else: + log.info('[class.%s] Uploader: mega folder %s was found', __class__.__name__, mega_folder) + response = self.storage.upload(filename=source, dest=mega_folder[0]) + result = "uploaded" + + if self.configuration['storage-type'] == 'dropbox': with open(source, 'rb') as file_transfer: - response = self.dropbox_client.files_upload( - file_transfer.read(), - f'/{destination}/{source.split("/")[-1]}' - ) - log.info( - '[class.%s] %s successful transferred', - __class__.__name__, - response - ) + response = self.storage.files_upload(file_transfer.read(), f"/{destination}/{source.split('/')[-1]}") file_transfer.close() - return "uploaded" + result = "uploaded" - return None + log.info('[class.%s] Uploader: %s successful transferred', __class__.__name__, response) + return result diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..7b9771665 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,4 @@ +# Prepare development environment +```bash +export TG_TOKEN=change_ne && export TG_USERID=123456 pytest --verbose -s . +`````` \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..88ae83150 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,5 @@ +""" +This is just a special file that tells pip that your main module is in this folder +No need to add anything here. Feel free to delete this line when you make your own package +Leave it empty +""" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..d2930de7a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,190 @@ +""" +This module stores fixtures for performing tests. +""" +import os +import sys +import subprocess +import time +import requests +import pytest +# pylint: disable=E0401 +from vault import VaultClient + + +def pytest_configure(config): + """ + Configure Pytest by adding a custom marker for setting the execution order of tests. + + This function is called during Pytest's configuration phase and is used to extend Pytest's + functionality by adding custom markers. In this case, it adds a "order" marker to specify + the execution order of tests. + + Parameters: + - config (object): The Pytest configuration object. + + Example Usage: + @pytest.mark.order(1) + def test_example(): + # test code + """ + config.addinivalue_line("markers", "order: Set the execution order of tests") + + +@pytest.fixture(name="prepare_dev_environment", scope='session') +def fixture_prepare_dev_environment(): + """ + Prepare a local environment or ci environment and return the URL of the Vault server + """ + if not os.getenv("CI"): + if not os.getenv("TG_USERID"): + print("You need to set the TG_USER_ID environment variable to run the tests (telegram user-id)") + sys.exit(1) + if not os.getenv("TG_TOKEN"): + print("You need to set the TG_TOKEN environment variable to run the tests (telegram token)") + sys.exit(1) + command = ( + "vault=$(docker ps -a | grep vault | awk '{print $1}') && " + "bot=$(docker ps -a | grep pyinstabot-downloader | awk '{print $1}') && " + "[ -n '$vault' ] && docker container rm -f $vault && " + "[ -n '$bot' ] && docker container rm -f $bot && " + "docker compose -f docker-compose.dev.yml up -d" + ) + with subprocess.Popen(command, shell=True): + print("Running dev environment...") + return 'ready' + return None + + +@pytest.fixture(name="vault_url", scope='session') +def fixture_vault_url(prepare_dev_environment): + """Prepare a local environment or ci environment and return the URL of the Vault server""" + _ = prepare_dev_environment + # prepare vault for local environment + if not os.getenv("CI"): + url = "http://0.0.0.0:8200" + # prepare vault for ci environment + else: + url = "http://localhost:8200" + # checking the availability of the vault server + while True: + try: + response = requests.get(url=url, timeout=3) + if 200 <= response.status_code < 500: + break + except requests.exceptions.RequestException as exception: + print(f"Waiting for the vault server: {exception}") + time.sleep(5) + return url + + +@pytest.fixture(name="name", scope='session') +def fixture_name(): + """Returns the project name""" + return "pyinstabot-downloader" + + +@pytest.fixture(name="policy_path", scope='session') +def fixture_policy_path(): + """Returns the policy path""" + return "tests/vault/policy.hcl" + + +@pytest.fixture(name="vault_approle", scope='session') +def fixture_vault_approle(vault_url, name, policy_path): + """Prepare a temporary Vault instance and return the Vault client""" + configurator = VaultClient( + url=vault_url, + name=name, + new=True + ) + namespace = configurator.create_namespace( + name=name + ) + policy = configurator.create_policy( + name=name, + path=policy_path + ) + return configurator.create_approle( + name=name, + path=namespace, + policy=policy + ) + + +@pytest.fixture(name="vault_instance", scope='session') +def fixture_vault_instance(vault_url, vault_approle, name): + """Returns an initialized vault instance""" + return VaultClient( + url=vault_url, + name=name, + approle=vault_approle + ) + + +@pytest.fixture(name="vault_configuration_data", scope='session') +def fixture_vault_configuration_data(vault_instance): + """ + This function sets up a database configuration in the vault_instance object. + + Args: + vault_instance: An instance of the Vault class. + + Returns: + None + """ + database = { + 'host': 'postgres', + 'port': '5432', + 'user': 'python', + 'password': 'python', + 'database': 'pyinstabot-downloader' + } + for key, value in database.items(): + _ = vault_instance.write_secret( + path='configuration/database', + key=key, + value=value + ) + + _ = vault_instance.write_secret( + path='configuration/telegram', + key='token', + value=os.getenv("TG_TOKEN") + ) + + user_attributes = { + "status": "allowed", + "roles": [ + "get_post_role", + "get_posts_list_role", + "get_queue_role", + "get_history_role" + ], + "requests": { + "requests_per_day": 10, + "requests_per_hour": 1, + "random_shift_minutes": 15 + } + } + user_id = os.getenv("TG_USERID") + for key, value in user_attributes.items(): + _ = vault_instance.write_secret( + path=f'configuration/users/{user_id}', + key=key, + value=value + ) + + test_owner = { + "eiD5aech8Oh": "downloaded", + "eiD5aech8Oa": "downloaded", + "eiD5aech8Oq": "downloaded", + "eiD5aech8Ol": "downloaded", + "eiD5aech8Op": "downloaded", + "eiD5aech8Oy": "downloaded" + } + for key, value in test_owner.items(): + _ = vault_instance.write_secret( + path='history/testOwner', + key=key, + value=value + ) diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100644 index 000000000..ab49ceeb7 --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,21 @@ +""" +A test for quick setup of the dev environment for testing the release. +""" +import subprocess +import pytest + + +@pytest.mark.order(1) +def test_init_dev_environment(vault_configuration_data, vault_approle): + """ + Check the function for the user who is allow access to the bot + """ + _ = vault_configuration_data + command = ( + "export VAULT_ADDR=http://vault-server:8200 && " + f"export VAULT_APPROLE_ID={vault_approle['id']} && " + f"export VAULT_APPROLE_SECRETID={vault_approle['secret-id']} && " + "docker compose -f docker-compose.yml up -d --force-recreate --build pyinstabot-downloader" + ) + with subprocess.Popen(command, shell=True): + print("Running docker-compose.yml...") diff --git a/tests/vault/policy.hcl b/tests/vault/policy.hcl new file mode 100644 index 000000000..2a97205a6 --- /dev/null +++ b/tests/vault/policy.hcl @@ -0,0 +1,57 @@ + +# Operations for pytest +# Allow read access to retrieve the token using approle +path "auth/token/lookup" { + capabilities = ["read"] +} + +# Operations for pytest +# Allow updating capabilities for token revocation after creating and testing approle +path "auth/token/revoke" { + capabilities = ["update"] +} + +# Operations for the module +# Enable read access for self-lookup with tokens +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +# Operations for pytest +# Allow read, create or update operations on the pytest path +path "sys/mounts/pyinstabot-downloader" { + capabilities = ["read", "create", "update"] +} + +# Operations for the module +# Read and update namespace configuration +path "pyinstabot-downloader/config" { + capabilities = ["read", "list", "update"] +} + +# Operations for the module +# Work with secret application data +path "pyinstabot-downloader/data/configuration/*" { + capabilities = ["create", "read", "update", "list"] +} + +# Operations for the module +# Work with secret event data +path "pyinstabot-downloader/data/data/*" { + capabilities = ["create", "read", "update", "list"] +} + +# Allowed to read bot history +path "pyinstabot-downloader/metadata/history/*" { + capabilities = ["read", "list"] +} + +# Allowed to create, read, update, and list bot history +path "pyinstabot-downloader/data/history/*" { + capabilities = ["create", "read", "list", "update"] +} + +# Allowed to read and list of user configurations +path "pyinstabot-downloader/metadata/configuration/users" { + capabilities = ["read", "list"] +} diff --git a/vault/policy.hcl b/vault/policy.hcl index bf774e4bb..43e19b641 100644 --- a/vault/policy.hcl +++ b/vault/policy.hcl @@ -23,22 +23,29 @@ path "pyinstabot-downloader/configuration/*" { capabilities = ["read", "list"] } -# Allowed to read bot configuration +# Allowed to read other configurations path "pyinstabot-downloader/data/configuration/*" { capabilities = ["read", "list"] } -# Allowed to read bot history +# Allowed read and write of bot data (!!!deprecated after https://github.com/obervinov/users-package/issues/41!!!) +path "pyinstabot-downloader/data/data/*" { + capabilities = ["read", "list", "create", "update"] +} + + + +# Allowed to read bot history (!!!deprecated after migration to database!!!) path "pyinstabot-downloader/metadata/history/*" { capabilities = ["read", "list"] } -# Allowed to create, read, update, and list bot history +# Allowed to create, read, update, and list bot history (!!!deprecated after migration to database!!!) path "pyinstabot-downloader/data/history/*" { capabilities = ["create", "read", "list", "update"] } -# Allowed to read and record security events by a bot +# Allowed to read and record security events by a bot (!!!deprecated after v2.1.0 !!!) path "pyinstabot-downloader/data/events/*" { capabilities = ["read", "list", "create", "update"] }