From d34724aab11c4895cc6f5ccaf779b1673ea9c826 Mon Sep 17 00:00:00 2001 From: Josh Richards Date: Sun, 23 Jun 2024 11:03:10 -0400 Subject: [PATCH] feat(updater): download resume and progress logging Signed-off-by: Josh Richards --- index.php | 85 ++++++++++++++++++++++++++++++++++++++++-------- lib/Updater.php | 85 ++++++++++++++++++++++++++++++++++++++++-------- updater.phar | Bin 1173225 -> 1175688 bytes 3 files changed, 144 insertions(+), 26 deletions(-) diff --git a/index.php b/index.php index 1fe13825..dd0205e8 100644 --- a/index.php +++ b/index.php @@ -567,30 +567,55 @@ private function getUpdateServerResponse(): array { /** * Downloads the nextcloud folder to $DATADIR/updater-$instanceid/downloads/$filename * + * Logs download progress + * Resumes incomplete downloads if possible + * Supports outbound proxy usage + * Logs download statistics upon completion + * + * TODO: Provide download progress in real-time (in both CLI and Web modes) + * * @throws \Exception */ public function downloadUpdate(): void { $this->silentLog('[info] downloadUpdate()'); $response = $this->getUpdateServerResponse(); - - $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; - if (file_exists($storageLocation)) { - $this->silentLog('[info] storage location exists'); - $this->recursiveDelete($storageLocation); - } - $state = mkdir($storageLocation, 0750, true); - if ($state === false) { - throw new \Exception('Could not mkdir storage location'); - } - if (!isset($response['url']) || !is_string($response['url'])) { throw new \Exception('Response from update server is missing url'); } - $fp = fopen($storageLocation . basename($response['url']), 'w+'); + $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; + $saveLocation = $storageLocation . basename($response['url']); + $ch = curl_init($response['url']); + + if (!file_exists($storageLocation)) { + $state = mkdir($storageLocation, 0750, true); + if ($state === false) { + throw new \Exception('Could not mkdir storage location'); + } + $this->silentLog('[info] storage location created'); + } else { + $this->silentLog('[info] storage location already exists'); + // clean-up leftover extracted content from any prior runs, but leave any downloaded Archives alone + if (file_exists($storageLocation . 'nextcloud/')) { + $this->silentLog('[info] extracted Archive location exists'); + $this->recursiveDelete($storageLocation . 'nextcloud/'); + } + // see if there's an existing incomplete download to resume + if (is_file($saveLocation)) { + $size = filesize($saveLocation); + $range = $size . '-'; + curl_setopt($ch, CURLOPT_RANGE, $range); + $this->silentLog('[info] previous download found; resuming from ' . $this->formatBytes($size)); + } + } + + $fp = fopen($saveLocation, 'a'); curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_NOPROGRESS => false, + CURLOPT_PROGRESSFUNCTION => array($this, 'downloadProgressCallback'), CURLOPT_FILE => $fp, CURLOPT_USERAGENT => 'Nextcloud Updater', ]); @@ -607,7 +632,7 @@ public function downloadUpdate(): void { throw new \Exception('Curl error: ' . curl_error($ch)); } $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpCode !== 200) { + if ($httpCode !== 200 && $httpCode !== 206) { $statusCodes = [ 400 => 'Bad request', 401 => 'Unauthorized', @@ -634,13 +659,47 @@ public function downloadUpdate(): void { $message .= ' - URL: ' . htmlentities($response['url']); throw new \Exception($message); + } else { + // download succeeded + $info = curl_getinfo($ch); + $this->silentLog("[info] download stats: size=" . $this->formatBytes($info['size_download']) . " bytes; total_time=" . round($info['total_time'], 2) . " secs; avg speed=" . $this->formatBytes($info['speed_download']) . "/sec"); } + curl_close($ch); fclose($fp); $this->silentLog('[info] end of downloadUpdate()'); } + private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { + static $previousProgress = 0; + + if ($download_size !== 0) { + $progress = round($downloaded * 100 / $download_size); + if ($progress > $previousProgress) { + $previousProgress = $progress; + // log every 2% increment for the first 10% then only log every 10% increment after that + if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) { + $this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . " of " . $this->formatBytes($download_size) . ")"); + } + } + } + } + + private function formatBytes(int $bytes, int $precision = 2): string { + $units = array('B', 'KB', 'MB', 'GB', 'TB'); + + $bytes = max($bytes, 0); + $pow = floor(($bytes ? log($bytes) : 0) / log(1024)); + $pow = min($pow, count($units) - 1); + + // Uncomment one of the following alternatives + $bytes /= pow(1024, $pow); + // $bytes /= (1 << (10 * $pow)); + + return round($bytes, $precision) . $units[(int)$pow]; + } + /** * @throws \Exception */ diff --git a/lib/Updater.php b/lib/Updater.php index a7e61722..6740048c 100644 --- a/lib/Updater.php +++ b/lib/Updater.php @@ -529,30 +529,55 @@ private function getUpdateServerResponse(): array { /** * Downloads the nextcloud folder to $DATADIR/updater-$instanceid/downloads/$filename * + * Logs download progress + * Resumes incomplete downloads if possible + * Supports outbound proxy usage + * Logs download statistics upon completion + * + * TODO: Provide download progress in real-time (in both CLI and Web modes) + * * @throws \Exception */ public function downloadUpdate(): void { $this->silentLog('[info] downloadUpdate()'); $response = $this->getUpdateServerResponse(); - - $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; - if (file_exists($storageLocation)) { - $this->silentLog('[info] storage location exists'); - $this->recursiveDelete($storageLocation); - } - $state = mkdir($storageLocation, 0750, true); - if ($state === false) { - throw new \Exception('Could not mkdir storage location'); - } - if (!isset($response['url']) || !is_string($response['url'])) { throw new \Exception('Response from update server is missing url'); } - $fp = fopen($storageLocation . basename($response['url']), 'w+'); + $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; + $saveLocation = $storageLocation . basename($response['url']); + $ch = curl_init($response['url']); + + if (!file_exists($storageLocation)) { + $state = mkdir($storageLocation, 0750, true); + if ($state === false) { + throw new \Exception('Could not mkdir storage location'); + } + $this->silentLog('[info] storage location created'); + } else { + $this->silentLog('[info] storage location already exists'); + // clean-up leftover extracted content from any prior runs, but leave any downloaded Archives alone + if (file_exists($storageLocation . 'nextcloud/')) { + $this->silentLog('[info] extracted Archive location exists'); + $this->recursiveDelete($storageLocation . 'nextcloud/'); + } + // see if there's an existing incomplete download to resume + if (is_file($saveLocation)) { + $size = filesize($saveLocation); + $range = $size . '-'; + curl_setopt($ch, CURLOPT_RANGE, $range); + $this->silentLog('[info] previous download found; resuming from ' . $this->formatBytes($size)); + } + } + + $fp = fopen($saveLocation, 'a'); curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_NOPROGRESS => false, + CURLOPT_PROGRESSFUNCTION => array($this, 'downloadProgressCallback'), CURLOPT_FILE => $fp, CURLOPT_USERAGENT => 'Nextcloud Updater', ]); @@ -569,7 +594,7 @@ public function downloadUpdate(): void { throw new \Exception('Curl error: ' . curl_error($ch)); } $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpCode !== 200) { + if ($httpCode !== 200 && $httpCode !== 206) { $statusCodes = [ 400 => 'Bad request', 401 => 'Unauthorized', @@ -596,13 +621,47 @@ public function downloadUpdate(): void { $message .= ' - URL: ' . htmlentities($response['url']); throw new \Exception($message); + } else { + // download succeeded + $info = curl_getinfo($ch); + $this->silentLog("[info] download stats: size=" . $this->formatBytes($info['size_download']) . " bytes; total_time=" . round($info['total_time'], 2) . " secs; avg speed=" . $this->formatBytes($info['speed_download']) . "/sec"); } + curl_close($ch); fclose($fp); $this->silentLog('[info] end of downloadUpdate()'); } + private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { + static $previousProgress = 0; + + if ($download_size !== 0) { + $progress = round($downloaded * 100 / $download_size); + if ($progress > $previousProgress) { + $previousProgress = $progress; + // log every 2% increment for the first 10% then only log every 10% increment after that + if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) { + $this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . " of " . $this->formatBytes($download_size) . ")"); + } + } + } + } + + private function formatBytes(int $bytes, int $precision = 2): string { + $units = array('B', 'KB', 'MB', 'GB', 'TB'); + + $bytes = max($bytes, 0); + $pow = floor(($bytes ? log($bytes) : 0) / log(1024)); + $pow = min($pow, count($units) - 1); + + // Uncomment one of the following alternatives + $bytes /= pow(1024, $pow); + // $bytes /= (1 << (10 * $pow)); + + return round($bytes, $precision) . $units[(int)$pow]; + } + /** * @throws \Exception */ diff --git a/updater.phar b/updater.phar index 4a38828cc0af74490154147eb84ec8d19751da8d..4c8bdc28358d4d50b8f23f100a9645a544cb3b7e 100755 GIT binary patch delta 2694 zcmai0U2IfE6t=Z$?^0+70#P93P`B(ZY~5`EZQ5=tEl>!hh0^*%ZJN7xXLoP7duMZJ zZhw@zA`h5IL1OJBzM#f84e{rp8sm$JF(#UbF~%27NQ{X-DJDitOguApcl!gxCg*PN zoZmU;n{V!~Uk|)=W8jsm4QVU=SX(yR(UD2hEAQ-G{r=m?s`n3%)clxgI&)9(r+&j1 zZ+-vj{@#;K|GXY7s^5S2U;Fm_V(*2;-WrrzlzUO?Q0_yyALRj*dK3dCgt88$0cAbP z29!pWrlrMRQdhUJy{$EuOz!R2*RdZmwkKvI@_G+xm&xNKy!75rX*g_$kPW)gDl4(!=_z_nHea0Eay?qLk8@I5z74nS!U>N}*_q|s)grMl@Tlpu3cEldif?Hq^3Xqc6im$jBuztsWhjqH^L?^c{_rxH zk$=yVEAqPqa9+bsaA;PrDe9qx@JtJXU)8-6>tZ$xKW#V;Q20(L)sVUgEebSWnk!az_ar9 z6pqPsn2Vh@3>=O8>m#y3ZJ==rJOU9HlUNS(Gl@tn2G609y#58*E|YJQ`(Zf-IF){^ zEL-0Am~53#%#+5_+E{AWsw0Nl8mCmDLgcAOBUp|bY~FTrcR~;dhQm_;E0-L#`DE61 zXf(WXGUaj&Bh2k*vlIuVZ1DfJcDfWJ?3r#(St)a>%59AV8q4>bNlpbTifGu%$Dx00 zWN`Sz=;X+Op@Bnj2t4&1^?Z*NJvwDGf5qX*Dp+>~RjXa;IYlr@SmbTq+LL8o!4!S7 zf^ysh=8x%c#+-cN64}zU0&j7Mm=T${Kpv7;FOzz?UOi4%vL+&ImA;}qJT}xndUSY5S(%<^&PH`rn5lvlJV*L$_nVHBHmx&}*s4H> zj}9J!rLC6)|OEur6r^;nKqD&?$c_R}sG;W5p z>UtM~N|?^1;sYJTQxl8oYQ`hWy2!*+acB=5IknKwoXP<&V*US@TiL8lp1>H*0Rx2O zjs-F)yI&-vR4H+7-=oj;BNmS!ML85+F7i^lPg2nbi4Ce<$2tkFz)!7nPZUTG4n z&`hiorkI^k<0%=MVl4<)kX$JQ2b1gy;hgZaMptLlvrYySVW4Ax7<4FivQi#$#%^d! zCLyt=KL3{M47{KK#?qGp}j?siAM{XD3KkJZjcf@ zF0h;y_2vR|omp^7RUf5Tb~3X9Sy$vup^zH%pt{i(i~=dddlJr`#pYBM-AbWOMRlbG z+5^ifMHyut;K)#}SR5yHx5K>R-nks91)_HY{BpYku(B*d{my(-G1aKh*0JW8ym^jn ztep$V18?D(TgguvZ2HiYF@1Eyu=|HE2(jYi)Z5JY-Hp#AVqaSXDLpED55GhLt||9a;X3H2nuRX2 zfsf-iiij!K33>87>0bI^fovPue#YDR^Bb={+;eN;t8E99Z=O5($)~^6;>Y4Z-|@cx E0Cm)482|tP delta 384 zcmeBp=>GDo`-J&C#wMl~mPv+*smZA+6IUCf@?Qm^@Hcxf-nA7-Wq<$>{oe2YpUI~7 zfdYvWA%Y;f^JQ{C^ECVRY4(gj%ml>DK+FQftU$~L#Oy%K0mPg@%mu{UK+FTgyg3tn@dF1=7tbNb01!OH39dj<=Q|)rqAsYRGVHi zNl