Merge pull request #184 from kloptops/main

Added better handling of downloading/installing absurdly large ports.

PortMaster/pylibs/harbourmaster/config.py

@@ -25,6 +25,9 @@
 HM_TESTING=False
 HM_PERFTEST=False
 
+## Maximum temporary size is 100 mb, this can cause errors on TrimUI and muOS.
+HM_MAX_TEMP_SIZE = 1024 * 1024 * 100
+
 ################################################################################
 ## The following code is a simplification of the PortMaster toolsloc and whichsd code.
 HM_DEFAULT_PORTS_DIR   = Path("/roms/ports")
@@ -253,6 +256,7 @@
     'HM_SCRIPTS_DIR',
     'HM_SORT_ORDER',
     'HM_SOURCE_DEFAULTS',
+    'HM_MAX_TEMP_SIZE',
     'HM_TESTING',
     'HM_TOOLS_DIR',
     'HM_UPDATE_FREQUENCY',

PortMaster/pylibs/harbourmaster/harbour.py

@@ -1331,7 +1331,7 @@ def _fix_permissions(self, path_check=None):
             logger.error(f"Failed to fix permissions: {err}")
             return
 
-    def _install_theme(self, download_file):
+    def _install_theme(self, download_file, do_delete=False):
         """
         Installs a theme file.
         """
@@ -1362,11 +1362,14 @@ def _install_theme(self, download_file):
         with open(theme_dir / "theme.md5", 'w') as fh:
             fh.write(hash_file(download_file))
 
+        if do_delete:
+            download_file.unlink()
+
         self.callback.message_box(_("Theme {download_name!r} installed successfully.").format(download_name=download_file.name))
 
         return 0
 
-    def _install_portmaster(self, download_file):
+    def _install_portmaster(self, download_file, do_delete=False):
         """
         Installs a new version of PortMaster
         """
@@ -1419,11 +1422,12 @@ def _install_portmaster(self, download_file):
             self._fix_permissions(self.tools_dir)
 
         finally:
-            ...
+            if do_delete:
+                download_file.unlink()
 
         return 0
 
-    def _install_port(self, download_info):
+    def _install_port(self, download_info, do_delete=False):
         """
         Installs a port.
 
@@ -1460,14 +1464,21 @@ def _install_port(self, download_info):
                 self.callback.message(_("Installing {download_name}.").format(download_name=port_nice_name))
 
                 total_files = len(zf.infolist())
+
+                # Not naming any names, but this is necessary for ports with many files
+                count_skip = 1
+                if total_files > 400:
+                    count_skip = (total_files // 400)
+
                 for file_number, file_info in enumerate(zf.infolist()):
                     if file_info.file_size == 0:
                         compress_saving = 100
                     else:
                         compress_saving = file_info.compress_size / file_info.file_size * 100
 
-                    self.callback.progress(_("Installing"), file_number+1, total_files, '%')
-                    self.callback.message(f"- {file_info.filename}")
+                    if (file_number % count_skip) == 0 or (file_number + 1) == total_files:
+                        self.callback.progress(_("Installing"), file_number + 1, total_files, '%')
+                        self.callback.message(f"- {file_info.filename}")
 
                     is_script = (
                         file_info.filename.endswith('.sh') and
@@ -1538,9 +1549,6 @@ def _install_port(self, download_info):
                 json.dump(port_info, fh, indent=4)
 
             # Remove the zip file if it is in the self.temp_dir
-            if str(download_info['zip_file']).startswith(str(self.temp_dir)):
-                download_info['zip_file'].unlink()
-
             is_successs = True
 
             self.platform.port_install(port_info['name'], port_info, undo_data)
@@ -1557,14 +1565,17 @@ def _install_port(self, download_info):
             pass
 
         finally:
+            if do_delete:
+                download_info['zip_file'].unlink()
+
             if not is_successs:
                 if len(undo_data) > 0:
                     logger.error("Installation failed, removing installed files.")
                     self.callback.message(_("Installation failed, removing files..."))
 
                     for undo_file in undo_data[::-1]:
-                        logger.info(f"Removing {undo_file.relative_to(self.ports_dir)}")
-                        self.callback.message(f"- {str(undo_file.relative_to(self.ports_dir))}")
+                        logger.info(f"Removing {str(self._ports_dir_relative_to(undo_file))}")
+                        self.callback.message(f"- {str(self._ports_dir_relative_to(self.ports_dir))}")
 
                         if undo_file.is_file():
                             undo_file.unlink()
@@ -1803,13 +1814,13 @@ def install_port(self, port_name, md5_source=None):
 
             with self.callback.enable_cancellable(False):
                 if name_cleaner(download_info['name']).endswith('.theme.zip'):
-                    return self._install_theme(download_info['zip_file'])
+                    return self._install_theme(download_info['zip_file'], do_delete=True)
 
                 elif name_cleaner(download_info['name']) == 'portmaster.zip':
-                    return self._install_portmaster(download_info['zip_file'])
+                    return self._install_portmaster(download_info['zip_file'], do_delete=True)
 
                 else:
-                    return self._install_port(download_info)
+                    return self._install_port(download_info, do_delete=True)
 
         # Special case for a local file.
         if port_name.startswith('./') or port_name.startswith('../') or port_name.startswith('/'):
@@ -1883,12 +1894,12 @@ def install_port(self, port_name, md5_source=None):
             # print(f"Download Info: {download_info.to_dict()}")
             with self.callback.enable_cancellable(False):
                 if source.clean_name(port_name).endswith('.theme.zip'):
-                    return self._install_theme(download_info)
+                    return self._install_theme(download_info, do_delete=True)
 
                 elif source.clean_name(port_name) == 'portmaster.zip':
-                    return self._install_portmaster(download_info)
+                    return self._install_portmaster(download_info, do_delete=True)
 
-                return self._install_port(download_info)
+                return self._install_port(download_info, do_delete=True)
 
         self.callback.message_box(_("Unable to find a source for {port_name}").format(port_name=port_name))
 

PortMaster/pylibs/harbourmaster/source.py

@@ -982,7 +982,12 @@ def download(self, port_name, temp_dir=None, md5_result=None):
             return None
 
         if temp_dir is None:
-            temp_dir = self.hm.temp_dir
+            file_size = self._data[port_name]['size']
+
+            if file_size > HM_MAX_TEMP_SIZE:
+                temp_dir = self.hm.ports_dir
+            else:
+                temp_dir = self.hm.temp_dir
 
         md5_result[0] = self._data[port_name]['md5']
         zip_file = download(temp_dir / port_name, self._data[port_name]['url'], self._data[port_name]['md5'], callback=self.hm.callback)

PortMaster/pylibs/harbourmaster/util.py

@@ -375,6 +375,14 @@ def download(file_name, file_url, md5_source=None, md5_result=None, callback=Non
             if callback is not None:
                 callback.progress(_("Downloading file."), length, total_length, 'data')
 
+    except CancelEvent as err:
+        if file_name.is_file():
+            file_name.unlink()
+
+        logger.error(f"Requests error: {err}")
+
+        raise
+
     except requests.RequestException as err:
         if file_name.is_file():
             file_name.unlink()