Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: download error #88

Merged
merged 6 commits into from
Jun 24, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions media_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,8 +261,11 @@ async def _get_media_meta(

async def add_download_task(message: pyrogram.types.Message, node: TaskNode):
"""Add Download task"""
if message.empty:
return False
await queue.put((message, node))
node.total_task += 1
return True


async def download_task(
Expand Down Expand Up @@ -511,8 +514,8 @@ async def download_chat_task(
)

for message in skipped_messages:
await add_download_task(message, node)
chat_download_config.total_task += 1
if not await add_download_task(message, node):
chat_download_config.downloaded_ids.append(message.id)

async for message in messages_iter: # type: ignore
meta_data = MetaData()
Expand All @@ -525,13 +528,13 @@ async def download_chat_task(
caption = app.get_caption_name(node.chat_id, message.media_group_id)
set_meta_data(meta_data, message, caption)

if not app.need_skip_message(chat_download_config, message.id, meta_data):
await add_download_task(message, node)
chat_download_config.total_task += 1
else:
if not app.need_skip_message(
chat_download_config, message.id, meta_data
) and await add_download_task(message, node):
chat_download_config.downloaded_ids.append(message.id)

chat_download_config.need_check = True
chat_download_config.total_task = node.total_task
node.is_running = True


Expand Down
3 changes: 2 additions & 1 deletion module/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -576,13 +576,14 @@ def update_config(self, immediate: bool = True):
range(before_last_read_message_id, value.last_read_message_id + 1)
)
unfinished_ids -= set(value.downloaded_ids)
unfinished_ids -= set({0})

self.chat_download_config[key].ids_to_retry = list(unfinished_ids)

if idx >= len(self.app_data["chat"]):
self.app_data["chat"].append({})

if len(value.downloaded_ids) > 0:
if value.finish_task:
self.config["chat"][idx]["last_read_message_id"] = (
value.last_read_message_id + 1
)
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#https://github.com/tangyoha/pyrogram/archive/refs/tags/v2.0.69.zip
#pyrogram=2.0.106
#pyrogram==2.0.106
https://github.com/tangyoha/pyrogram/archive/refs/heads/master_v2.0.69_1.zip
PyYAML==6.0
rich==12.5.1
Expand Down
5 changes: 5 additions & 0 deletions tests/module/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,13 @@ def test_app(self):
app.chat_download_config[123].last_read_message_id = 13
app.chat_download_config[123].failed_ids.append(6)
app.chat_download_config[123].ids_to_retry.append(7)
# download success
app.chat_download_config[123].downloaded_ids.append(8)
app.chat_download_config[123].finish_task += 1
# download success
app.chat_download_config[123].downloaded_ids.append(10)
app.chat_download_config[123].finish_task += 1
# not exist message
app.chat_download_config[123].downloaded_ids.append(13)
app.config["chat"] = [{"chat_id": 123, "last_read_message_id": 5}]

Expand Down
1 change: 1 addition & 0 deletions tests/test_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ def __init__(self, **kwargs):
self.media_group_id = kwargs.get("media_group_id", None)
self.caption = kwargs.get("caption", None)
self.text = None
self.empty = kwargs.get("empty", False)

if kwargs.get("dis_chat") == None:
self.chat = Chat(
Expand Down
23 changes: 6 additions & 17 deletions tests/test_media_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,11 @@
)
from module.app import Application, DownloadStatus, TaskNode
from module.cloud_drive import CloudDriveConfig
from module.pyrogram_extension import record_download_status, reset_download_cache
from module.pyrogram_extension import (
get_extension,
record_download_status,
reset_download_cache,
)

from .test_common import (
Chat,
Expand Down Expand Up @@ -359,6 +363,7 @@ async def edit_message_text(self, *args, **kwargs):


@mock.patch("media_downloader.get_extension", new=get_extension)
@mock.patch("module.pyrogram_extension.get_extension", new=get_extension)
@mock.patch("media_downloader.fetch_message", new=new_fetch_message)
@mock.patch("media_downloader.get_chat_history_v2", new=get_chat_history)
@mock.patch("media_downloader.RETRY_TIME_OUT", new=0)
Expand Down Expand Up @@ -972,22 +977,6 @@ def test_issues_311(self):

self.assertEqual(res, (DownloadStatus.SkipDownload, None))

@mock.patch(
"media_downloader._exec_loop",
new=raise_keyboard_interrupt,
)
@mock.patch("media_downloader.pyrogram.Client", new=MockClient)
@mock.patch("media_downloader.RETRY_TIME_OUT", new=1)
@mock.patch("media_downloader.logger")
def test_main(self, mock_logger):
rest_app(MOCK_CONF)

main()

mock_logger.success.assert_called_with(
"Updated last read message_id to config file,total download 0, total upload file 0"
)

@mock.patch("media_downloader.pyrogram.Client", new=MockClient)
@mock.patch("media_downloader.RETRY_TIME_OUT", new=1)
@mock.patch("media_downloader.logger")
Expand Down