From 6612ba579fac74fa573e9e3090ce0e24e0085749 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sat, 16 Nov 2024 15:05:13 +0800 Subject: [PATCH] Added better handling of downloading/installing absurdly large ports. --- PortMaster/pylibs/harbourmaster/config.py | 4 ++ PortMaster/pylibs/harbourmaster/harbour.py | 45 ++++++++++++++-------- PortMaster/pylibs/harbourmaster/source.py | 7 +++- PortMaster/pylibs/harbourmaster/util.py | 8 ++++ 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/PortMaster/pylibs/harbourmaster/config.py b/PortMaster/pylibs/harbourmaster/config.py index 1b6057c..971f4ec 100755 --- a/PortMaster/pylibs/harbourmaster/config.py +++ b/PortMaster/pylibs/harbourmaster/config.py @@ -25,6 +25,9 @@ HM_TESTING=False HM_PERFTEST=False +## Maximum temporary size is 100 mb, this can cause errors on TrimUI and muOS. +HM_MAX_TEMP_SIZE = 1024 * 1024 * 100 + ################################################################################ ## The following code is a simplification of the PortMaster toolsloc and whichsd code. HM_DEFAULT_PORTS_DIR = Path("/roms/ports") @@ -253,6 +256,7 @@ 'HM_SCRIPTS_DIR', 'HM_SORT_ORDER', 'HM_SOURCE_DEFAULTS', + 'HM_MAX_TEMP_SIZE', 'HM_TESTING', 'HM_TOOLS_DIR', 'HM_UPDATE_FREQUENCY', diff --git a/PortMaster/pylibs/harbourmaster/harbour.py b/PortMaster/pylibs/harbourmaster/harbour.py index e793d36..450e265 100644 --- a/PortMaster/pylibs/harbourmaster/harbour.py +++ b/PortMaster/pylibs/harbourmaster/harbour.py @@ -1331,7 +1331,7 @@ def _fix_permissions(self, path_check=None): logger.error(f"Failed to fix permissions: {err}") return - def _install_theme(self, download_file): + def _install_theme(self, download_file, do_delete=False): """ Installs a theme file. """ @@ -1362,11 +1362,14 @@ def _install_theme(self, download_file): with open(theme_dir / "theme.md5", 'w') as fh: fh.write(hash_file(download_file)) + if do_delete: + download_file.unlink() + self.callback.message_box(_("Theme {download_name!r} installed successfully.").format(download_name=download_file.name)) return 0 - def _install_portmaster(self, download_file): + def _install_portmaster(self, download_file, do_delete=False): """ Installs a new version of PortMaster """ @@ -1419,11 +1422,12 @@ def _install_portmaster(self, download_file): self._fix_permissions(self.tools_dir) finally: - ... + if do_delete: + download_file.unlink() return 0 - def _install_port(self, download_info): + def _install_port(self, download_info, do_delete=False): """ Installs a port. @@ -1460,14 +1464,21 @@ def _install_port(self, download_info): self.callback.message(_("Installing {download_name}.").format(download_name=port_nice_name)) total_files = len(zf.infolist()) + + # Not naming any names, but this is necessary for ports with many files + count_skip = 1 + if total_files > 400: + count_skip = (total_files // 400) + for file_number, file_info in enumerate(zf.infolist()): if file_info.file_size == 0: compress_saving = 100 else: compress_saving = file_info.compress_size / file_info.file_size * 100 - self.callback.progress(_("Installing"), file_number+1, total_files, '%') - self.callback.message(f"- {file_info.filename}") + if (file_number % count_skip) == 0 or (file_number + 1) == total_files: + self.callback.progress(_("Installing"), file_number + 1, total_files, '%') + self.callback.message(f"- {file_info.filename}") is_script = ( file_info.filename.endswith('.sh') and @@ -1538,9 +1549,6 @@ def _install_port(self, download_info): json.dump(port_info, fh, indent=4) # Remove the zip file if it is in the self.temp_dir - if str(download_info['zip_file']).startswith(str(self.temp_dir)): - download_info['zip_file'].unlink() - is_successs = True self.platform.port_install(port_info['name'], port_info, undo_data) @@ -1557,14 +1565,17 @@ def _install_port(self, download_info): pass finally: + if do_delete: + download_info['zip_file'].unlink() + if not is_successs: if len(undo_data) > 0: logger.error("Installation failed, removing installed files.") self.callback.message(_("Installation failed, removing files...")) for undo_file in undo_data[::-1]: - logger.info(f"Removing {undo_file.relative_to(self.ports_dir)}") - self.callback.message(f"- {str(undo_file.relative_to(self.ports_dir))}") + logger.info(f"Removing {str(self._ports_dir_relative_to(undo_file))}") + self.callback.message(f"- {str(self._ports_dir_relative_to(self.ports_dir))}") if undo_file.is_file(): undo_file.unlink() @@ -1803,13 +1814,13 @@ def install_port(self, port_name, md5_source=None): with self.callback.enable_cancellable(False): if name_cleaner(download_info['name']).endswith('.theme.zip'): - return self._install_theme(download_info['zip_file']) + return self._install_theme(download_info['zip_file'], do_delete=True) elif name_cleaner(download_info['name']) == 'portmaster.zip': - return self._install_portmaster(download_info['zip_file']) + return self._install_portmaster(download_info['zip_file'], do_delete=True) else: - return self._install_port(download_info) + return self._install_port(download_info, do_delete=True) # Special case for a local file. if port_name.startswith('./') or port_name.startswith('../') or port_name.startswith('/'): @@ -1883,12 +1894,12 @@ def install_port(self, port_name, md5_source=None): # print(f"Download Info: {download_info.to_dict()}") with self.callback.enable_cancellable(False): if source.clean_name(port_name).endswith('.theme.zip'): - return self._install_theme(download_info) + return self._install_theme(download_info, do_delete=True) elif source.clean_name(port_name) == 'portmaster.zip': - return self._install_portmaster(download_info) + return self._install_portmaster(download_info, do_delete=True) - return self._install_port(download_info) + return self._install_port(download_info, do_delete=True) self.callback.message_box(_("Unable to find a source for {port_name}").format(port_name=port_name)) diff --git a/PortMaster/pylibs/harbourmaster/source.py b/PortMaster/pylibs/harbourmaster/source.py index 99f6782..6de6e3c 100644 --- a/PortMaster/pylibs/harbourmaster/source.py +++ b/PortMaster/pylibs/harbourmaster/source.py @@ -982,7 +982,12 @@ def download(self, port_name, temp_dir=None, md5_result=None): return None if temp_dir is None: - temp_dir = self.hm.temp_dir + file_size = self._data[port_name]['size'] + + if file_size > HM_MAX_TEMP_SIZE: + temp_dir = self.hm.ports_dir + else: + temp_dir = self.hm.temp_dir md5_result[0] = self._data[port_name]['md5'] zip_file = download(temp_dir / port_name, self._data[port_name]['url'], self._data[port_name]['md5'], callback=self.hm.callback) diff --git a/PortMaster/pylibs/harbourmaster/util.py b/PortMaster/pylibs/harbourmaster/util.py index b2e25fb..3543440 100644 --- a/PortMaster/pylibs/harbourmaster/util.py +++ b/PortMaster/pylibs/harbourmaster/util.py @@ -375,6 +375,14 @@ def download(file_name, file_url, md5_source=None, md5_result=None, callback=Non if callback is not None: callback.progress(_("Downloading file."), length, total_length, 'data') + except CancelEvent as err: + if file_name.is_file(): + file_name.unlink() + + logger.error(f"Requests error: {err}") + + raise + except requests.RequestException as err: if file_name.is_file(): file_name.unlink()