From d480753bf54b2a5676242430917a365ead79e9b5 Mon Sep 17 00:00:00 2001 From: Xientra Date: Mon, 20 Feb 2023 14:35:32 +0000 Subject: [PATCH 1/5] feat: download file as .part & resume .part download from where left off --- src/TSRDownload.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/TSRDownload.py b/src/TSRDownload.py index 70b9fa2..ed099b5 100644 --- a/src/TSRDownload.py +++ b/src/TSRDownload.py @@ -1,4 +1,4 @@ -import requests, time, json, os +import requests, time, json, os, re from TSRUrl import TSRUrl from logger import Logger from exceptions import * @@ -25,17 +25,36 @@ def download(self) -> bool: time.sleep(timeToSleep / 1000) downloadUrl = self.__getDownloadUrl() - request = self.session.get(downloadUrl, stream=True) - fileName = request.headers["Content-Disposition"][ - 22:-1 - ] # Remove 'attachment; filename="' from header - file = open(f"{CONFIG['downloadDirectory']}/{fileName}", "wb") + fileName = self.__getFileName(downloadUrl) + + startingBytes = ( + os.path.getsize(f"{CONFIG['downloadDirectory']}/{fileName}.part") + if os.path.exists(f"{CONFIG['downloadDirectory']}/{fileName}.part") + else 0 + ) + request = self.session.get( + downloadUrl, + stream=True, + headers={"Range": f"bytes={startingBytes}-"}, + ) + file = open(f"{CONFIG['downloadDirectory']}/{fileName}.part", "wb") for chunk in request.iter_content(1024 * 1024): file.write(chunk) file.close() + os.rename( + f"{CONFIG['downloadDirectory']}/{fileName}.part", + f"{CONFIG['downloadDirectory']}/{fileName}", + ) return True + @classmethod + def __getFileName(self, downloadUrl: str) -> str: + return re.search( + '(?<=filename=").+(?=")', + requests.get(downloadUrl, stream=True).headers["Content-Disposition"], + )[0] + @classmethod def __getDownloadUrl(self) -> str: response = self.session.get( From 2f5276b0311c606a6b2758d52cb3139c8c7caf20 Mon Sep 17 00:00:00 2001 From: Xientra Date: Mon, 20 Feb 2023 18:07:55 +0000 Subject: [PATCH 2/5] feat: save urls to file & load urls when starting script Saves running downloads & queued downloads to a file, which is loaded when the script it ran. --- src/TSRDownload.py | 2 +- src/main.py | 29 +++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/TSRDownload.py b/src/TSRDownload.py index ed099b5..da7ccb6 100644 --- a/src/TSRDownload.py +++ b/src/TSRDownload.py @@ -39,7 +39,7 @@ def download(self) -> bool: ) file = open(f"{CONFIG['downloadDirectory']}/{fileName}.part", "wb") - for chunk in request.iter_content(1024 * 1024): + for chunk in request.iter_content(1024 * 128): file.write(chunk) file.close() os.rename( diff --git a/src/main.py b/src/main.py index 5e636f1..d49e6ce 100644 --- a/src/main.py +++ b/src/main.py @@ -10,6 +10,7 @@ CONFIG: CONFIG_DICT = json.load( open(os.path.dirname(os.path.abspath(__file__)) + "/config.json", "r") ) + URLS_PATH = os.path.dirname(os.path.abspath(__file__)) + "/urls.txt" lastPastedText = "" runningDownloads: list[str] = [] downloadQueue: list[str] = [] @@ -23,9 +24,19 @@ def processTarget(url: TSRUrl): def callback(url: TSRUrl): runningDownloads.remove(url.url) + updateUrlFile() if len(runningDownloads) == 0: Logger.info("All downloads have been completed") + def updateUrlFile(): + open(URLS_PATH, "w").write("\n".join([*runningDownloads, *downloadQueue])) + + if os.path.exists(URLS_PATH): + for url in open(URLS_PATH, "r").read().split("\n"): + if url.strip() == "" or url in downloadQueue: + continue + downloadQueue.append(url.strip()) + while True: pastedText = clipboard.paste() if lastPastedText == pastedText: @@ -44,14 +55,6 @@ def callback(url: TSRUrl): Logger.info("Queue is now empty") else: lastPastedText = pastedText - if pastedText in runningDownloads: - Logger.info(f"Url is already being downloaded: {pastedText}") - continue - if pastedText in downloadQueue: - Logger.info( - f"Url is already in queue (#{downloadQueue.index(pastedText)}): {pastedText}" - ) - try: url = TSRUrl(pastedText) requirements = TSRUrl.getRequiredItems(url) @@ -60,6 +63,15 @@ def callback(url: TSRUrl): Logger.info(f"{url.url} has {len(requirements)} requirements") for url in [url, *requirements]: + if url.url in runningDownloads: + Logger.info(f"Url is already being downloaded: {url.url}") + continue + if url.url in downloadQueue: + Logger.info( + f"Url is already in queue (#{downloadQueue.index(url.url)}): {url.url}" + ) + continue + if len(runningDownloads) == CONFIG["maxDownloads"]: Logger.info( f"Added url to queue (#{len(downloadQueue)}): {url.url}" @@ -69,6 +81,7 @@ def callback(url: TSRUrl): runningDownloads.append(url.url) pool = Pool(1) pool.apply_async(processTarget, args=[url], callback=callback) + updateUrlFile() except InvalidURL: pass From 623c3bc0077408c7d10f5bbc279393075c2bd1ac Mon Sep 17 00:00:00 2001 From: Xientra Date: Mon, 20 Feb 2023 18:21:01 +0000 Subject: [PATCH 3/5] chore: add urls file to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5521712..63e6499 100644 --- a/.gitignore +++ b/.gitignore @@ -165,3 +165,6 @@ poetry.toml # ruff .ruff_cache/ + +# Script Created Files +src/urls.txt From c73efb4d6b91b06d44e9647b06738bebefa5a63f Mon Sep 17 00:00:00 2001 From: Xientra Date: Tue, 21 Feb 2023 20:03:56 +0000 Subject: [PATCH 4/5] feat: add configuration to toggle the saving & loading of active & queued downloads --- README.md | 1 + src/config.json | 3 ++- src/main.py | 5 +++-- src/typings.py | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d25e250..e3e9a88 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ With this tool you can download multiple items at once, to download items copy t | - | - | - | | downloadDirectory | Path to a directory where the files will be downloaded to. | string | | maxDownloads | Limits the max amount of active downloads to the value set. | integer | +| saveDownloadQueue | Toggles saving & loading of active downloads & queued downloads. | boolean | ## Installing Requirements diff --git a/src/config.json b/src/config.json index 20e9466..330cb7d 100644 --- a/src/config.json +++ b/src/config.json @@ -1,4 +1,5 @@ { "downloadDirectory": "./", - "maxDownloads": 4 + "maxDownloads": 4, + "saveDownloadQueue": true } \ No newline at end of file diff --git a/src/main.py b/src/main.py index d49e6ce..64902f6 100644 --- a/src/main.py +++ b/src/main.py @@ -29,9 +29,10 @@ def callback(url: TSRUrl): Logger.info("All downloads have been completed") def updateUrlFile(): - open(URLS_PATH, "w").write("\n".join([*runningDownloads, *downloadQueue])) + if CONFIG["saveDownloadQueue"]: + open(URLS_PATH, "w").write("\n".join([*runningDownloads, *downloadQueue])) - if os.path.exists(URLS_PATH): + if os.path.exists(URLS_PATH) and CONFIG["saveDownloadQueue"]: for url in open(URLS_PATH, "r").read().split("\n"): if url.strip() == "" or url in downloadQueue: continue diff --git a/src/typings.py b/src/typings.py index 61a076d..8da1155 100644 --- a/src/typings.py +++ b/src/typings.py @@ -1,5 +1,6 @@ import typing CONFIG_DICT = typing.TypedDict( - "Config Dict", {"downloadDirectory": str, "maxDownloads": int} + "Config Dict", + {"downloadDirectory": str, "maxDownloads": int, "saveDownloadQueue": bool}, ) From 29cc500230032e688b54ea805d52ae2cf63f47fd Mon Sep 17 00:00:00 2001 From: Xientra Date: Tue, 9 May 2023 16:38:47 +0100 Subject: [PATCH 5/5] refactor: remove unused import --- src/TSRDownload.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/TSRDownload.py b/src/TSRDownload.py index f7d55ab..5267c9c 100644 --- a/src/TSRDownload.py +++ b/src/TSRDownload.py @@ -1,5 +1,4 @@ -import requests, time, json, os, re -from TSRSession import TSRSession +import requests, time, os, re from TSRUrl import TSRUrl from logger import logger from exceptions import *