networkutil: Add new download_file utility function (#423)

* networkutil: Add new download_file utility function

* DXVK: Use download_file util function

* Luxtorpeda: Use download_file util function

* networkutil: Fix type hint

* DXVK: Typing and formatting

* networkutil: Catch more errors when making request

* networkutil: Make `download_file` lambda optional

* networkutil: Warn if filesize is 0 instead of returning `False` in `download_file`

* networkutil: Add safety checks to prevent crashing if buffer_size is 0

* raise exceptions in download_file and handle them in ctmods

* networkutil: Note exceptions for download_file

* ctmod: Add message_box_message connection for reporting errors

* ctmod: Update download_file error dialog title

* ctmod: Translate error dialog strings

* minor stylistic changes

* Use double quotes to allow for single quote in string

* ctmod_luxtorpeda: Clean imports

* ctmod_z0dxvk: Clean imports

* ctmod_z0dxvk: Apply stylistic change

---------

Co-authored-by: DavidoTek <54072917+DavidoTek@users.noreply.github.com>

pupgui2/networkutil.py

@@ -0,0 +1,108 @@
+import os
+from PySide6.QtCore import Property
+import requests
+
+from typing import Callable
+
+
+def download_file(url: str, destination: str, progress_callback: Callable[[int], None] | Callable[..., None] = lambda *args, **kwargs: None, download_cancelled: Property | None = None, buffer_size: int = 65536, stream: bool = True, known_size: int = 0):
+    """
+    Download a file from a given URL using `requests` to a destination directory with download progress, with some optional parameters:
+    * `progress_callback`: Function or Lambda that gets called with the download progress each time it changes
+    * `download_cancelled`: Qt Property that can stop the download
+    * `buffer_size`: Size of chunks to download the file in
+    * `stream`: Lazily parse response - If response headers won't contain `'Content-Length'` and the file size is not known ahead of time, set this to `False` to get file size from response content length
+    * `known_size`: If size is known ahead of time, this can be given to calculate download progress in place of Content-Length header (e.g. where it may be missing)
+
+    Returns `True` if download succeeds, `False` otherwise.
+
+    Raises: `OSError`, `requests.ConnectionError`, `requests.Timeout`
+    Return Type: bool
+    """
+
+    # Try to get the data for the file we want
+    try:
+        response: requests.Response = requests.get(url, stream=stream)
+    except (OSError, requests.ConnectionError, requests.Timeout) as e:
+        print(f"Error: Failed to make request to URL '{url}', cannot complete download! Reason: {e}")
+        raise e
+
+    progress_callback(1)  # 1 = download started
+
+    # Figure out file size for reporting download progress    
+    if stream and response.headers.get('Transfer-Encoding', '').lower() == 'chunked':
+        print("Warning: Using 'stream=True' in request but 'Transfer-Encoding' in Response is 'Chunked', so we may not get 'Content-Length' to parse file size!")
+
+    # Sometimes ctmods can have access to the asset size, so they can give it to us
+    # If it is not specified, or if it is zero/Falsey, try to get it from the response
+    file_size = known_size
+    if not known_size:
+        file_size = int(response.headers.get('Content-Length', 0))
+
+        # Sometimes Content-Length is not sent (such as for DXVK Async), so use response length in that case
+        # See: https://stackoverflow.com/questions/53797628/request-has-no-content-length#53797919
+        #
+        # Only get response.content if we aren't streaming so that we don't hold up the entire function,
+        # and defeating the point of streaming to begin with
+        if not stream:
+            file_size = len(response.content)
+
+    if file_size <= 0:
+        print('Warning: Failed to get file size, the progress bar may not display accurately!')
+
+    if buffer_size <= 0:
+        print(f"Warning: Buffer Size was '{buffer_size}', defaulting to '65536'")
+        buffer_size = 65536
+
+    # NOTE: If we don't get a known_size or if we can't get the size from Cotent-Length or the response size,
+    #       we cannot report download progress!
+    #
+    #       Right now, only GitLab doesn't give us Content-Length because it uses Chunked Transfer-Encoding,
+    #       but ctmods should be able to get the size and pass it as known_size.
+    #
+    #       If we ever make it this far without a file_size (e.g. we are stream=True and we don't get a
+    #       Content-Length, or len(response.content) is 0), then then the progress bar will stall at 1% until
+    #       the download finishes where it will jump to 99%, until extraction completes.
+    try:
+        chunk_count = int(file_size / buffer_size)
+    except ZeroDivisionError as e:
+        print(f'Error: Could not calculate chunk_count, {e}')
+        print('Defaulting to chunk count of 1')
+        chunk_count = 1
+
+    current_chunk = 1
+
+    # Get download filepath and download directory path without filename
+    destination_file_path: str = os.path.expanduser(destination)
+    destination_dir_path: str = os.path.dirname(destination_file_path)
+
+    # Create download path if it doesn't exist (and make sure we have permission to do so)
+    try:
+        os.makedirs(destination_dir_path, exist_ok=True)
+    except OSError as e:
+        print(f'Error: Failed to create path to destination directory, cannot complete download! Reason: {e}')
+        raise e
+
+    # Download file and return progress to any given callback
+    with open(destination, 'wb') as destination_file:
+        for chunk in response.iter_content(chunk_size=buffer_size):
+            chunk: bytes
+
+            if download_cancelled:
+                progress_callback(-2)  # -2 = Download cancelled
+                return False
+
+            if not chunk:
+                continue
+
+            _ = destination_file.write(chunk)
+            destination_file.flush()
+
+            download_progress = int(min(current_chunk / chunk_count * 98.0, 98.0))  # 1...98 = Download in progress
+            progress_callback(download_progress)
+
+            current_chunk += 1
+
+    progress_callback(99)  # 99 = Download completed successfully
+    return True
+

pupgui2/resources/ctmods/ctmod_luxtorpeda.py

@@ -8,8 +8,10 @@
 from PySide6.QtCore import QObject, QCoreApplication, Signal, Property
 from PySide6.QtWidgets import QMessageBox
 
-from pupgui2.util import extract_tar, write_tool_version
-from pupgui2.util import build_headers_with_authorization, create_missing_dependencies_message, fetch_project_release_data, fetch_project_releases
+from pupgui2.networkutil import download_file
+from pupgui2.util import extract_tar, write_tool_version, fetch_project_releases
+from pupgui2.util import fetch_project_release_data, build_headers_with_authorization
+from pupgui2.util import create_missing_dependencies_message
 
 
 CT_NAME = 'Luxtorpeda'
@@ -54,35 +56,30 @@ def __set_download_progress_percent(self, value : int):
         self.p_download_progress_percent = value
         self.download_progress_percent.emit(value)
 
-    def __download(self, url, destination):
+    def __download(self, url: str, destination: str, known_size: int = 0):
         """
         Download files from url to destination
         Return Type: bool
         """
-        try:
-            file = requests.get(url, stream=True)
-        except OSError:
-            return False
 
-        self.__set_download_progress_percent(1) # 1 download started
-        f_size = int(file.headers.get('content-length'))
-        c_count = int(f_size / self.BUFFER_SIZE)
-        c_current = 1
-        destination = os.path.expanduser(destination)
-        os.makedirs(os.path.dirname(destination), exist_ok=True)
-        with open(destination, 'wb') as dest:
-            for chunk in file.iter_content(chunk_size=self.BUFFER_SIZE):
-                if self.download_canceled:
-                    self.download_canceled = False
-                    self.__set_download_progress_percent(-2) # -2 download canceled
-                    return False
-                if chunk:
-                    dest.write(chunk)
-                    dest.flush()
-                self.__set_download_progress_percent(int(min(c_current / c_count * 98.0, 98.0))) # 1-98, 100 after extract
-                c_current += 1
-        self.__set_download_progress_percent(99) # 99 download complete
-        return True
+        try:
+            return download_file(
+                url=url,
+                destination=destination,
+                progress_callback=self.__set_download_progress_percent,
+                download_cancelled=self.download_canceled,
+                buffer_size=self.BUFFER_SIZE,
+                stream=True,
+                known_size=known_size
+            )
+        except Exception as e:
+            print(f"Failed to download tool {CT_NAME} - Reason: {e}")
+
+            self.message_box_message.emit(
+                self.tr("Download Error!"),
+                self.tr("Failed to download tool '{CT_NAME}'!\n\nReason: {EXCEPTION}".format(CT_NAME=CT_NAME, EXCEPTION=e)),
+                QMessageBox.Icon.Warning
+            )
 
     def __fetch_github_data(self, tag):
         """
@@ -131,7 +128,7 @@ def get_tool(self, version, install_dir, temp_dir):
             return False
 
         luxtorpeda_tar = os.path.join(temp_dir, data['download'].split('/')[-1])
-        if not self.__download(url=data['download'], destination=luxtorpeda_tar):
+        if not self.__download(url=data['download'], destination=luxtorpeda_tar, known_size=data.get('size', 0)):
             return False
 
         luxtorpeda_dir = os.path.join(install_dir, self.extract_dir_name)

pupgui2/resources/ctmods/ctmod_z0dxvk.py

@@ -3,13 +3,14 @@
 # Copyright (C) 2022 DavidoTek, partially based on AUNaseef's protonup
 
 import os
+from PySide6.QtWidgets import QMessageBox
 import requests
 
-from typing import Dict
-
 from PySide6.QtCore import QObject, QCoreApplication, Signal, Property
 
-from pupgui2.util import extract_tar, get_launcher_from_installdir, fetch_project_releases, fetch_project_release_data, build_headers_with_authorization
+from pupgui2.networkutil import download_file
+from pupgui2.util import extract_tar, get_launcher_from_installdir, fetch_project_releases
+from pupgui2.util import fetch_project_release_data, build_headers_with_authorization
 from pupgui2.datastructures import Launcher
 
 
@@ -26,6 +27,7 @@ class CtInstaller(QObject):
 
     p_download_progress_percent = 0
     download_progress_percent = Signal(int)
+    message_box_message = Signal((str, str, QMessageBox.Icon))
 
     def __init__(self, main_window = None):
         super(CtInstaller, self).__init__()
@@ -50,39 +52,32 @@ def __set_download_progress_percent(self, value : int):
         self.p_download_progress_percent = value
         self.download_progress_percent.emit(value)
 
-    def __download(self, url, destination):
+    def __download(self, url: str, destination: str, known_size: int = 0):
         """
         Download files from url to destination
         Return Type: bool
         """
-        try:
-            file = self.rs.get(url, stream=True)
-        except OSError:
-            return False
-
-        self.__set_download_progress_percent(1) # 1 download started
-        # https://stackoverflow.com/questions/53797628/request-has-no-content-length#53797919
-        f_size = len(file.content)
-        c_count = int(f_size / self.BUFFER_SIZE)
-        c_current = 1
-        destination = os.path.expanduser(destination)
-        os.makedirs(os.path.dirname(destination), exist_ok=True)
-        with open(destination, 'wb') as dest:
-            for chunk in file.iter_content(chunk_size=self.BUFFER_SIZE):
-                if self.download_canceled:
-                    self.download_canceled = False
-                    self.__set_download_progress_percent(-2) # -2 download canceled
-                    return False
-                if chunk:
-                    dest.write(chunk)
-                    dest.flush()
-                self.__set_download_progress_percent(int(min(c_current / c_count * 98.0, 98.0))) # 1-98, 100 after extract
-                c_current += 1
-        self.__set_download_progress_percent(99) # 99 download complete
-        return True
-
-    def __fetch_data(self, tag: str = '') -> Dict:
 
+        try:
+            return download_file(
+                url=url,
+                destination=destination,
+                progress_callback=self.__set_download_progress_percent,
+                download_cancelled=self.download_canceled,
+                buffer_size=self.BUFFER_SIZE,
+                stream=True,
+                known_size=known_size
+            )
+        except Exception as e:
+            print(f"Failed to download tool {CT_NAME} - Reason: {e}")
+
+            self.message_box_message.emit(
+                self.tr("Download Error!"),
+                self.tr("Failed to download tool '{CT_NAME}'!\n\nReason: {EXCEPTION}".format(CT_NAME=CT_NAME, EXCEPTION=e)),
+                QMessageBox.Icon.Warning
+            )
+
+    def __fetch_data(self, tag: str = '') -> dict:
         """
         Fetch release information
         Return Type: dict
@@ -119,7 +114,7 @@ def get_tool(self, version, install_dir, temp_dir):
 
         # Should be updated to support Heroic, like ctmod_d8vk
         dxvk_tar = os.path.join(temp_dir, data['download'].split('/')[-1])
-        if not self.__download(url=data['download'], destination=dxvk_tar):
+        if not self.__download(url=data['download'], destination=dxvk_tar, known_size=data.get('size', 0)):
             return False
 
         dxvk_dir = self.get_extract_dir(install_dir)

pupgui2/util.py

@@ -637,7 +637,7 @@ def fetch_project_release_data(release_url: str, release_format: str, rs: reques
     Fetch information about a given release based on its tag, with an optional condition lambda.
     Return Type: dict
     Content(s):
-        'version', 'date', 'download'
+        'version', 'date', 'download', 'size' (if available)
     """
 
     date_key: str = ''