From e060278dcae17d2a58ff962d02ccc020df3918df Mon Sep 17 00:00:00 2001 From: tangyoha Date: Mon, 15 May 2023 21:26:32 +0800 Subject: [PATCH] fix: fix rclone error (#72) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 修复 * #69 ## 新功能 * 支持docker下rclone调用 ## Fix * #69 ## Feat * Support rclone call under docker --- .github/workflows/docker-publish.yml | 13 ++++++++++-- .gitignore | 2 ++ Dockerfile | 12 ++++++++++- README.md | 2 +- README_CN.md | 2 +- docker-compose.yaml | 6 ++++++ media_downloader.py | 20 +++++++++++------- module/app.py | 2 ++ module/bot.py | 5 ++++- module/cloud_drive.py | 31 ++++++++++++++++++++++------ module/pyrogram_extension.py | 9 ++++++++ 11 files changed, 84 insertions(+), 20 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8acfcdbb..d3fbebfc 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -13,6 +13,7 @@ jobs: outputs: compile_image_exists: ${{ steps.check-image.outputs.exists }} requirements_modified: ${{ steps.check-requirements.outputs.modified }} + check-dockerfile: ${{ steps.check-dockerfile.outputs.dockerfile_modified }} steps: - name: Checkout @@ -30,10 +31,17 @@ jobs: - name: Check if requirements.txt has been modified id: check-requirements run: | - MODIFIED=$(git diff --name-only HEAD~1 HEAD | grep -w 'requirements.txt' && echo "true" || echo "false") + MODIFIED=$(git diff --name-only HEAD~1 HEAD | grep -w 'requirements.txt' > /dev/null && echo "true" || echo "false") echo "modified=$MODIFIED" >> $GITHUB_ENV echo "modified=$MODIFIED" >> $GITHUB_OUTPUT + - name: Check if Dockerfile has been modified + id: check-dockerfile + run: | + DOCKERFILE_MODIFIED=$(git diff --name-only HEAD~1 HEAD | grep -w 'Dockerfile' > /dev/null && echo "true" || echo "false") + echo "dockerfile_modified=$DOCKERFILE_MODIFIED" >> $GITHUB_ENV + echo "dockerfile_modified=$DOCKERFILE_MODIFIED" >> $GITHUB_OUTPUT + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -48,9 +56,10 @@ jobs: run: | echo ${{ steps.check-image.outputs.exists }} echo ${{ steps.check-requirements.outputs.modified }} + echo ${{ steps.check-dockerfile.outputs.dockerfile_modified }} - name: Build and push compile-image - if: ${{ !(steps.check-image.outputs.exists == 'true' && steps.check-requirements.outputs.modified == 'false') }} + if: ${{ !(steps.check-image.outputs.exists == 'true' && steps.check-requirements.outputs.modified == 'false' && steps.check-dockerfile.outputs.dockerfile_modified == 'false') }} uses: docker/build-push-action@v4 with: context: . diff --git a/.gitignore b/.gitignore index c26d8fa0..04e2b11e 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,5 @@ local_test/ TODO.md log/ temp/ +config.yaml +data.yaml diff --git a/Dockerfile b/Dockerfile index 6bc541e6..af03fdcb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,11 +8,21 @@ RUN apk add --no-cache --virtual .build-deps gcc musl-dev \ && pip install --trusted-host pypi.python.org -r requirements.txt \ && apk del .build-deps && rm -rf requirements.txt +# install rclone +RUN apk add --no-cache ca-certificates && \ + wget https://downloads.rclone.org/rclone-current-linux-amd64.zip && \ + unzip rclone-current-linux-amd64.zip && \ + mv rclone-*-linux-amd64/rclone /app/rclone && \ + rm -rf rclone-*-linux-amd64 && \ + rm -rf rclone-current-linux-amd64.zip + FROM python:3.11.2-alpine As runtime-image WORKDIR /app -COPY --from=tangyoha/telegram_media_downloader_compile:latest /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages +COPY --from=tangyoha/telegram_media_downloader_compile:master_fix_rclone_error /app/rclone /app/rclone/rclone + +COPY --from=tangyoha/telegram_media_downloader_compile:master_fix_rclone_error /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages COPY config.yaml data.yaml setup.py media_downloader.py /app/ COPY module /app/module diff --git a/README.md b/README.md index e106e667..c79badb1 100644 --- a/README.md +++ b/README.md @@ -268,4 +268,4 @@ Help us keep Telegram Media Downloader open and inclusive. Please read and follo

Code style: black Code style: black -

\ No newline at end of file +

diff --git a/README_CN.md b/README_CN.md index 39bda3d0..97c07d1b 100644 --- a/README_CN.md +++ b/README_CN.md @@ -269,4 +269,4 @@ proxy:

Code style: black Code style: black -

\ No newline at end of file +

diff --git a/docker-compose.yaml b/docker-compose.yaml index cf6fe5ca..85522996 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,10 +7,16 @@ services: ports: # Here is what you need to edit - "5000:5000" + #environment: + # - http_proxy=http://192.168.101.30:10811 + # - https_proxy=http://192.168.101.30:10811 volumes: # Here is what you need to edit - "./downloads/:/app/downloads/" + # Rclone + - "$HOME/.config/rclone/:$HOME/.config/rclone/" + # The following is what you don't need to edit - "./config.yaml:/app/config.yaml" - "./data.yaml:/app/data.yaml" diff --git a/media_downloader.py b/media_downloader.py index fc109fcd..9c633a09 100644 --- a/media_downloader.py +++ b/media_downloader.py @@ -287,6 +287,14 @@ async def download_task( download_status, ) + # rclone upload + if ( + not node.upload_telegram_chat_id + and download_status is DownloadStatus.SuccessDownload + ): + if await app.upload_file(file_name): + node.upload_success_count += 1 + await report_bot_download_status( node.bot, node, @@ -294,13 +302,6 @@ async def download_task( file_size, ) - # rclone upload - if ( - not node.upload_telegram_chat_id - and download_status is DownloadStatus.SuccessDownload - ): - await app.upload_file(file_name) - # pylint: disable = R0915,R0914 @@ -480,7 +481,10 @@ async def worker(client: pyrogram.client.Client): message = item[0] node: TaskNode = item[1] - await download_task(client, message, node) + if node.client: + await download_task(node.client, message, node) + else: + await download_task(client, message, node) except Exception as e: logger.exception(f"{e}") diff --git a/module/app.py b/module/app.py index 68d2dc43..b7f122dd 100644 --- a/module/app.py +++ b/module/app.py @@ -89,6 +89,8 @@ def __init__( self.failed_forward_task: int = 0 self.skip_forward_task: int = 0 self.is_running: bool = False + self.client = None + self.upload_success_count: int = 0 def is_finish(self): """If is finish""" diff --git a/module/bot.py b/module/bot.py index e22c1060..7a5ccd1e 100644 --- a/module/bot.py +++ b/module/bot.py @@ -350,6 +350,7 @@ async def direct_download( chat_id: Union[str, int], message: pyrogram.types.Message, download_message: pyrogram.types.Message, + client: pyrogram.Client = None, ): """Direct Download""" @@ -368,6 +369,8 @@ async def direct_download( task_id=_bot.gen_task_id(), ) + node.client = client + _bot.add_task_node(node) await _bot.add_download_task( @@ -393,7 +396,7 @@ async def download_forward_media( """ if message.media and getattr(message, message.media.value): - await direct_download(_bot, message.from_user.id, message, message) + await direct_download(_bot, message.from_user.id, message, message, client) return await client.send_message( diff --git a/module/cloud_drive.py b/module/cloud_drive.py index 339a9c33..6e80ab4b 100644 --- a/module/cloud_drive.py +++ b/module/cloud_drive.py @@ -86,8 +86,9 @@ def zip_file(local_file_path: str) -> str: @staticmethod async def rclone_upload_file( drive_config: CloudDriveConfig, save_path: str, local_file_path: str - ): + ) -> bool: """Use Rclone upload file""" + upload_status: bool = False try: remote_dir = ( drive_config.remote_dir @@ -125,19 +126,24 @@ async def rclone_upload_file( os.remove(local_file_path) if drive_config.before_upload_file_zip: os.remove(zip_file_path) + upload_status = True await proc.wait() except Exception as e: logger.error(f"{e.__class__} {e}") + return False + + return upload_status @staticmethod def aligo_upload_file( drive_config: CloudDriveConfig, save_path: str, local_file_path: str ): """aliyun upload file""" + upload_status: bool = False if not drive_config.aligo: logger.warning("please config aligo! see README.md") - return + return False try: remote_dir = ( @@ -174,13 +180,18 @@ def aligo_upload_file( if drive_config.before_upload_file_zip: os.remove(zip_file_path) + upload_status = True + except Exception as e: logger.error(f"{e.__class__} {e}") + return False + + return upload_status @staticmethod async def upload_file( drive_config: CloudDriveConfig, save_path: str, local_file_path: str - ): + ) -> bool: """Upload file Parameters ---------- @@ -192,13 +203,21 @@ async def upload_file( local_file_path: str Local file path + + Returns + ------- + bool + True or False """ if not drive_config.enable_upload_file: - return + return False + ret: bool = False if drive_config.upload_adapter == "rclone": - await CloudDrive.rclone_upload_file( + ret = await CloudDrive.rclone_upload_file( drive_config, save_path, local_file_path ) elif drive_config.upload_adapter == "aligo": - CloudDrive.aligo_upload_file(drive_config, save_path, local_file_path) + ret = CloudDrive.aligo_upload_file(drive_config, save_path, local_file_path) + + return ret diff --git a/module/pyrogram_extension.py b/module/pyrogram_extension.py index f1d73985..8397fe35 100644 --- a/module/pyrogram_extension.py +++ b/module/pyrogram_extension.py @@ -408,6 +408,14 @@ async def report_bot_status( f"└─ ⏩ {_t('Skipped')}: {node.skip_forward_task}\n" ) + upload_msg_detail_str: str = "" + + if node.upload_success_count: + upload_msg_detail_str = ( + f"\n📥 {_t('Upload')}\n" + f"└─ ✅ {_t('Success')}: {node.upload_success_count}\n" + ) + download_result_str = "" download_result = get_download_result() if node.chat_id in download_result: @@ -441,6 +449,7 @@ async def report_bot_status( f"├─ ❌ {_t('Failed')}: {node.failed_download_task}\n" f"└─ ⏩ {_t('Skipped')}: {node.skip_download_task}\n" f"{node.forward_msg_detail_str}" + f"{upload_msg_detail_str}" f"{download_result_str}\n```" )