From d34724aab11c4895cc6f5ccaf779b1673ea9c826 Mon Sep 17 00:00:00 2001 From: Josh Richards Date: Sun, 23 Jun 2024 11:03:10 -0400 Subject: [PATCH 1/4] feat(updater): download resume and progress logging Signed-off-by: Josh Richards --- index.php | 85 ++++++++++++++++++++++++++++++++++++++++-------- lib/Updater.php | 85 ++++++++++++++++++++++++++++++++++++++++-------- updater.phar | Bin 1173225 -> 1175688 bytes 3 files changed, 144 insertions(+), 26 deletions(-) diff --git a/index.php b/index.php index 1fe13825..dd0205e8 100644 --- a/index.php +++ b/index.php @@ -567,30 +567,55 @@ private function getUpdateServerResponse(): array { /** * Downloads the nextcloud folder to $DATADIR/updater-$instanceid/downloads/$filename * + * Logs download progress + * Resumes incomplete downloads if possible + * Supports outbound proxy usage + * Logs download statistics upon completion + * + * TODO: Provide download progress in real-time (in both CLI and Web modes) + * * @throws \Exception */ public function downloadUpdate(): void { $this->silentLog('[info] downloadUpdate()'); $response = $this->getUpdateServerResponse(); - - $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; - if (file_exists($storageLocation)) { - $this->silentLog('[info] storage location exists'); - $this->recursiveDelete($storageLocation); - } - $state = mkdir($storageLocation, 0750, true); - if ($state === false) { - throw new \Exception('Could not mkdir storage location'); - } - if (!isset($response['url']) || !is_string($response['url'])) { throw new \Exception('Response from update server is missing url'); } - $fp = fopen($storageLocation . basename($response['url']), 'w+'); + $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; + $saveLocation = $storageLocation . basename($response['url']); + $ch = curl_init($response['url']); + + if (!file_exists($storageLocation)) { + $state = mkdir($storageLocation, 0750, true); + if ($state === false) { + throw new \Exception('Could not mkdir storage location'); + } + $this->silentLog('[info] storage location created'); + } else { + $this->silentLog('[info] storage location already exists'); + // clean-up leftover extracted content from any prior runs, but leave any downloaded Archives alone + if (file_exists($storageLocation . 'nextcloud/')) { + $this->silentLog('[info] extracted Archive location exists'); + $this->recursiveDelete($storageLocation . 'nextcloud/'); + } + // see if there's an existing incomplete download to resume + if (is_file($saveLocation)) { + $size = filesize($saveLocation); + $range = $size . '-'; + curl_setopt($ch, CURLOPT_RANGE, $range); + $this->silentLog('[info] previous download found; resuming from ' . $this->formatBytes($size)); + } + } + + $fp = fopen($saveLocation, 'a'); curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_NOPROGRESS => false, + CURLOPT_PROGRESSFUNCTION => array($this, 'downloadProgressCallback'), CURLOPT_FILE => $fp, CURLOPT_USERAGENT => 'Nextcloud Updater', ]); @@ -607,7 +632,7 @@ public function downloadUpdate(): void { throw new \Exception('Curl error: ' . curl_error($ch)); } $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpCode !== 200) { + if ($httpCode !== 200 && $httpCode !== 206) { $statusCodes = [ 400 => 'Bad request', 401 => 'Unauthorized', @@ -634,13 +659,47 @@ public function downloadUpdate(): void { $message .= ' - URL: ' . htmlentities($response['url']); throw new \Exception($message); + } else { + // download succeeded + $info = curl_getinfo($ch); + $this->silentLog("[info] download stats: size=" . $this->formatBytes($info['size_download']) . " bytes; total_time=" . round($info['total_time'], 2) . " secs; avg speed=" . $this->formatBytes($info['speed_download']) . "/sec"); } + curl_close($ch); fclose($fp); $this->silentLog('[info] end of downloadUpdate()'); } + private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { + static $previousProgress = 0; + + if ($download_size !== 0) { + $progress = round($downloaded * 100 / $download_size); + if ($progress > $previousProgress) { + $previousProgress = $progress; + // log every 2% increment for the first 10% then only log every 10% increment after that + if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) { + $this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . " of " . $this->formatBytes($download_size) . ")"); + } + } + } + } + + private function formatBytes(int $bytes, int $precision = 2): string { + $units = array('B', 'KB', 'MB', 'GB', 'TB'); + + $bytes = max($bytes, 0); + $pow = floor(($bytes ? log($bytes) : 0) / log(1024)); + $pow = min($pow, count($units) - 1); + + // Uncomment one of the following alternatives + $bytes /= pow(1024, $pow); + // $bytes /= (1 << (10 * $pow)); + + return round($bytes, $precision) . $units[(int)$pow]; + } + /** * @throws \Exception */ diff --git a/lib/Updater.php b/lib/Updater.php index a7e61722..6740048c 100644 --- a/lib/Updater.php +++ b/lib/Updater.php @@ -529,30 +529,55 @@ private function getUpdateServerResponse(): array { /** * Downloads the nextcloud folder to $DATADIR/updater-$instanceid/downloads/$filename * + * Logs download progress + * Resumes incomplete downloads if possible + * Supports outbound proxy usage + * Logs download statistics upon completion + * + * TODO: Provide download progress in real-time (in both CLI and Web modes) + * * @throws \Exception */ public function downloadUpdate(): void { $this->silentLog('[info] downloadUpdate()'); $response = $this->getUpdateServerResponse(); - - $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; - if (file_exists($storageLocation)) { - $this->silentLog('[info] storage location exists'); - $this->recursiveDelete($storageLocation); - } - $state = mkdir($storageLocation, 0750, true); - if ($state === false) { - throw new \Exception('Could not mkdir storage location'); - } - if (!isset($response['url']) || !is_string($response['url'])) { throw new \Exception('Response from update server is missing url'); } - $fp = fopen($storageLocation . basename($response['url']), 'w+'); + $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; + $saveLocation = $storageLocation . basename($response['url']); + $ch = curl_init($response['url']); + + if (!file_exists($storageLocation)) { + $state = mkdir($storageLocation, 0750, true); + if ($state === false) { + throw new \Exception('Could not mkdir storage location'); + } + $this->silentLog('[info] storage location created'); + } else { + $this->silentLog('[info] storage location already exists'); + // clean-up leftover extracted content from any prior runs, but leave any downloaded Archives alone + if (file_exists($storageLocation . 'nextcloud/')) { + $this->silentLog('[info] extracted Archive location exists'); + $this->recursiveDelete($storageLocation . 'nextcloud/'); + } + // see if there's an existing incomplete download to resume + if (is_file($saveLocation)) { + $size = filesize($saveLocation); + $range = $size . '-'; + curl_setopt($ch, CURLOPT_RANGE, $range); + $this->silentLog('[info] previous download found; resuming from ' . $this->formatBytes($size)); + } + } + + $fp = fopen($saveLocation, 'a'); curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_NOPROGRESS => false, + CURLOPT_PROGRESSFUNCTION => array($this, 'downloadProgressCallback'), CURLOPT_FILE => $fp, CURLOPT_USERAGENT => 'Nextcloud Updater', ]); @@ -569,7 +594,7 @@ public function downloadUpdate(): void { throw new \Exception('Curl error: ' . curl_error($ch)); } $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpCode !== 200) { + if ($httpCode !== 200 && $httpCode !== 206) { $statusCodes = [ 400 => 'Bad request', 401 => 'Unauthorized', @@ -596,13 +621,47 @@ public function downloadUpdate(): void { $message .= ' - URL: ' . htmlentities($response['url']); throw new \Exception($message); + } else { + // download succeeded + $info = curl_getinfo($ch); + $this->silentLog("[info] download stats: size=" . $this->formatBytes($info['size_download']) . " bytes; total_time=" . round($info['total_time'], 2) . " secs; avg speed=" . $this->formatBytes($info['speed_download']) . "/sec"); } + curl_close($ch); fclose($fp); $this->silentLog('[info] end of downloadUpdate()'); } + private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { + static $previousProgress = 0; + + if ($download_size !== 0) { + $progress = round($downloaded * 100 / $download_size); + if ($progress > $previousProgress) { + $previousProgress = $progress; + // log every 2% increment for the first 10% then only log every 10% increment after that + if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) { + $this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . " of " . $this->formatBytes($download_size) . ")"); + } + } + } + } + + private function formatBytes(int $bytes, int $precision = 2): string { + $units = array('B', 'KB', 'MB', 'GB', 'TB'); + + $bytes = max($bytes, 0); + $pow = floor(($bytes ? log($bytes) : 0) / log(1024)); + $pow = min($pow, count($units) - 1); + + // Uncomment one of the following alternatives + $bytes /= pow(1024, $pow); + // $bytes /= (1 << (10 * $pow)); + + return round($bytes, $precision) . $units[(int)$pow]; + } + /** * @throws \Exception */ diff --git a/updater.phar b/updater.phar index 4a38828cc0af74490154147eb84ec8d19751da8d..4c8bdc28358d4d50b8f23f100a9645a544cb3b7e 100755 GIT binary patch delta 2694 zcmai0U2IfE6t=Z$?^0+70#P93P`B(ZY~5`EZQ5=tEl>!hh0^*%ZJN7xXLoP7duMZJ zZhw@zA`h5IL1OJBzM#f84e{rp8sm$JF(#UbF~%27NQ{X-DJDitOguApcl!gxCg*PN zoZmU;n{V!~Uk|)=W8jsm4QVU=SX(yR(UD2hEAQ-G{r=m?s`n3%)clxgI&)9(r+&j1 zZ+-vj{@#;K|GXY7s^5S2U;Fm_V(*2;-WrrzlzUO?Q0_yyALRj*dK3dCgt88$0cAbP z29!pWrlrMRQdhUJy{$EuOz!R2*RdZmwkKvI@_G+xm&xNKy!75rX*g_$kPW)gDl4(!=_z_nHea0Eay?qLk8@I5z74nS!U>N}*_q|s)grMl@Tlpu3cEldif?Hq^3Xqc6im$jBuztsWhjqH^L?^c{_rxH zk$=yVEAqPqa9+bsaA;PrDe9qx@JtJXU)8-6>tZ$xKW#V;Q20(L)sVUgEebSWnk!az_ar9 z6pqPsn2Vh@3>=O8>m#y3ZJ==rJOU9HlUNS(Gl@tn2G609y#58*E|YJQ`(Zf-IF){^ zEL-0Am~53#%#+5_+E{AWsw0Nl8mCmDLgcAOBUp|bY~FTrcR~;dhQm_;E0-L#`DE61 zXf(WXGUaj&Bh2k*vlIuVZ1DfJcDfWJ?3r#(St)a>%59AV8q4>bNlpbTifGu%$Dx00 zWN`Sz=;X+Op@Bnj2t4&1^?Z*NJvwDGf5qX*Dp+>~RjXa;IYlr@SmbTq+LL8o!4!S7 zf^ysh=8x%c#+-cN64}zU0&j7Mm=T${Kpv7;FOzz?UOi4%vL+&ImA;}qJT}xndUSY5S(%<^&PH`rn5lvlJV*L$_nVHBHmx&}*s4H> zj}9J!rLC6)|OEur6r^;nKqD&?$c_R}sG;W5p z>UtM~N|?^1;sYJTQxl8oYQ`hWy2!*+acB=5IknKwoXP<&V*US@TiL8lp1>H*0Rx2O zjs-F)yI&-vR4H+7-=oj;BNmS!ML85+F7i^lPg2nbi4Ce<$2tkFz)!7nPZUTG4n z&`hiorkI^k<0%=MVl4<)kX$JQ2b1gy;hgZaMptLlvrYySVW4Ax7<4FivQi#$#%^d! zCLyt=KL3{M47{KK#?qGp}j?siAM{XD3KkJZjcf@ zF0h;y_2vR|omp^7RUf5Tb~3X9Sy$vup^zH%pt{i(i~=dddlJr`#pYBM-AbWOMRlbG z+5^ifMHyut;K)#}SR5yHx5K>R-nks91)_HY{BpYku(B*d{my(-G1aKh*0JW8ym^jn ztep$V18?D(TgguvZ2HiYF@1Eyu=|HE2(jYi)Z5JY-Hp#AVqaSXDLpED55GhLt||9a;X3H2nuRX2 zfsf-iiij!K33>87>0bI^fovPue#YDR^Bb={+;eN;t8E99Z=O5($)~^6;>Y4Z-|@cx E0Cm)482|tP delta 384 zcmeBp=>GDo`-J&C#wMl~mPv+*smZA+6IUCf@?Qm^@Hcxf-nA7-Wq<$>{oe2YpUI~7 zfdYvWA%Y;f^JQ{C^ECVRY4(gj%ml>DK+FQftU$~L#Oy%K0mPg@%mu{UK+FTgyg3tn@dF1=7tbNb01!OH39dj<=Q|)rqAsYRGVHi zNl Date: Thu, 4 Jul 2024 11:03:12 -0400 Subject: [PATCH 2/4] fix: adjust download resume variable/property handling Signed-off-by: Josh Richards --- index.php | 7 +++---- lib/Updater.php | 7 +++---- updater.phar | Bin 1175688 -> 1175703 bytes 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/index.php b/index.php index dd0205e8..d67ffea1 100644 --- a/index.php +++ b/index.php @@ -69,6 +69,7 @@ class Updater { private bool $updateAvailable = false; private ?string $requestID = null; private bool $disabled = false; + private int $previousProgress = 0; /** * Updater constructor @@ -672,12 +673,10 @@ public function downloadUpdate(): void { } private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { - static $previousProgress = 0; - if ($download_size !== 0) { $progress = round($downloaded * 100 / $download_size); - if ($progress > $previousProgress) { - $previousProgress = $progress; + if ($progress > $this->previousProgress) { + $this->previousProgress = $progress; // log every 2% increment for the first 10% then only log every 10% increment after that if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) { $this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . " of " . $this->formatBytes($download_size) . ")"); diff --git a/lib/Updater.php b/lib/Updater.php index 6740048c..a44c2880 100644 --- a/lib/Updater.php +++ b/lib/Updater.php @@ -31,6 +31,7 @@ class Updater { private bool $updateAvailable = false; private ?string $requestID = null; private bool $disabled = false; + private int $previousProgress = 0; /** * Updater constructor @@ -634,12 +635,10 @@ public function downloadUpdate(): void { } private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { - static $previousProgress = 0; - if ($download_size !== 0) { $progress = round($downloaded * 100 / $download_size); - if ($progress > $previousProgress) { - $previousProgress = $progress; + if ($progress > $this->previousProgress) { + $this->previousProgress = $progress; // log every 2% increment for the first 10% then only log every 10% increment after that if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) { $this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . " of " . $this->formatBytes($download_size) . ")"); diff --git a/updater.phar b/updater.phar index 4c8bdc28358d4d50b8f23f100a9645a544cb3b7e..505e37ab3a0ae6a49b8666bccca5e4a6b9e393e3 100755 GIT binary patch delta 266 zcmeBp=sx|S`-J&CsfkIcDJCh2#)hWJ6IUCf@?Qm^@HcxfzO-fe6(E;0+0;HzfO9qj z7=Y+3=?9COr`fkpvu6ZiCLm@8Viq7~1!6WJW(Q&pAm#*OE+FOxVjdvo1!6uR<_BVd z?bGZ9Iav5oj7`jqOcE7RGK)$o)u&&y6Exa>&slIAFArxyQD#|UNvc9-Udi;FS%OmA zrv(aL@R;5{OE8PABqOs}*KYd9S%Q*K_I9q>f=Qt==5JOQpS`i+UM{=b`pReQC;#OO Nt1G^qWvZ;Nbz^_>h zU;v{3ny$OlJk7p+nmr>BGXXI(5VHU=D-g2*F*^`*05K;Ja{)0o5c2>rFA(zqF+UIs zY@cQ?$ic#AU|?)vVqvO~l37$zsXqOpouJY7d(MK}c(?Be6ujiYR$P);l9@bxL7<@2 z^v+p=*%Adssb!h@rNse7`RPTe#l@Nm)m)rFFnwX5pvZQe*@FI|GTB9nPZms7w);8d WjI@iv!p6Yed!I1@J>c%->jVJJuVNVh From 113588737eca03f189cbff41716679fe8ec5988a Mon Sep 17 00:00:00 2001 From: Josh Richards Date: Thu, 4 Jul 2024 12:56:54 -0400 Subject: [PATCH 3/4] fix: cast to int Signed-off-by: Josh Richards --- index.php | 2 +- lib/Updater.php | 2 +- updater.phar | Bin 1175703 -> 1175708 bytes 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/index.php b/index.php index d67ffea1..d2e76a50 100644 --- a/index.php +++ b/index.php @@ -674,7 +674,7 @@ public function downloadUpdate(): void { private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { if ($download_size !== 0) { - $progress = round($downloaded * 100 / $download_size); + $progress = (int)round($downloaded * 100 / $download_size); if ($progress > $this->previousProgress) { $this->previousProgress = $progress; // log every 2% increment for the first 10% then only log every 10% increment after that diff --git a/lib/Updater.php b/lib/Updater.php index a44c2880..5fa9944d 100644 --- a/lib/Updater.php +++ b/lib/Updater.php @@ -636,7 +636,7 @@ public function downloadUpdate(): void { private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { if ($download_size !== 0) { - $progress = round($downloaded * 100 / $download_size); + $progress = (int)round($downloaded * 100 / $download_size); if ($progress > $this->previousProgress) { $this->previousProgress = $progress; // log every 2% increment for the first 10% then only log every 10% increment after that diff --git a/updater.phar b/updater.phar index 505e37ab3a0ae6a49b8666bccca5e4a6b9e393e3..0138aea312305ea68ccc2938aded16fc7c4d2a82 100755 GIT binary patch delta 480 zcma*kPbfq|90zc3J^$9)op+H$OAelY+k5lx?G&~s%fSJ0kZ5+^8|}qGQaDm_Bekzk z>t@SAlHQ4fB3CEnp!T4o92{Jn%x*4wGxh2B`_=Sqrk`I-pA^&k`AE`Zf-si~B4juNHZDCjb|!ysZ@X^*Bo?xu zQ+iYD?pO>2V=-eE8*pIx@An;Uc6Du|+kYB1VDLI>JkQmb*_E}P;QGjTm~HF5-#WW( Z(SPnXmyY|b&zXE*dm*IukRF*F`2|-6md5}9 delta 498 zcmbR9(0%$t_X+cPQWKL>Q%q74jSWqcC$2U|<-ZC-;cxa}jIm+iE3}`yd4+8!Bg?M< zxtz(S_JIPNvl+kuL}y7qSlm3#zI~cKBM>tIF*6Xe05K~NvjH(X5OV-AClGT1F*gwN z05LBR^KGAI&;Rj+MoOxIsfAIBfkC37g>hPPqG5`unTctNv8hR_g+-F3Ns?ufxoM)M z!S?kW0!EBPE1$lRS0HeE6qi656O*C&^oN218qKRTq6cA8kG@LFdC7`Hb zYMPRoW^Q1VmSmY|Y@TRjXlZI_m};0}k!+ThYLRMeZfItbY?zj`9cUmoBgm|aVgm9k z#(E%A`2>_T(o7ALOcGO5EDcP}Op{Ga%ngmr(hO4!3{A}~Ez-==%#$sQOe_u3KrWJM zcjFh>?#3?|Y9eT;o1S89Vs2!TsF0FbR8pxv{i2p2AFrt4?$XirzD z;}x6Ee~6o7d$hY?RPgp)vjmTY%9y`dVSM(+hI_f}a_cLfv7h{xFRaQ0j2U+)Unc<9 CC78JY From 2c6045ef329238b8a848ba8e274cbeeaf6466aa2 Mon Sep 17 00:00:00 2001 From: Josh Richards Date: Mon, 15 Jul 2024 21:17:49 -0400 Subject: [PATCH 4/4] fix: reset download progress just in case Signed-off-by: Josh Richards --- index.php | 1 + lib/Updater.php | 1 + updater.phar | Bin 1175708 -> 1175739 bytes 3 files changed, 2 insertions(+) diff --git a/index.php b/index.php index d2e76a50..2989407f 100644 --- a/index.php +++ b/index.php @@ -587,6 +587,7 @@ public function downloadUpdate(): void { $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; $saveLocation = $storageLocation . basename($response['url']); + $this->previousProgress = 0; $ch = curl_init($response['url']); diff --git a/lib/Updater.php b/lib/Updater.php index 5fa9944d..851dab83 100644 --- a/lib/Updater.php +++ b/lib/Updater.php @@ -549,6 +549,7 @@ public function downloadUpdate(): void { $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; $saveLocation = $storageLocation . basename($response['url']); + $this->previousProgress = 0; $ch = curl_init($response['url']); diff --git a/updater.phar b/updater.phar index 0138aea312305ea68ccc2938aded16fc7c4d2a82..74b60a255a2634ff7993eff2b64f00a0c522417c 100755 GIT binary patch delta 528 zcma*kJ!lj`6bEp3@6@~8*GzLoNOXe%PvqjhotfQPG+9V-tAdaSHY&pI&P;Mu^ezSj z(ZV7mMNk4ag@UbUA(CRj6r!XEcIi?H*NKHF2!fS!cIkZZM^jIuFiRE(-oGwS&uYW&=93F$KK3D-@46O|<(m`|A?j7rYa zkfvc8uq1Tzwr-5%!OxjBTgmE4?qT5@2a8@78VK0xq;$}fVF5y%9 z!gBSzb{^H5y78$}%5TM3#gpao^y+fA-=4eMlWX0T`~8L9%2H4E`_8Q6<{!53$7I5u e{QTvI|C)3TonL$J_omNnKHRm<-{Sde^M3&mgQ6e+ delta 474 zcma*kPbfq|90zc3_hH$!JMSWGwB%s(XM1n{y`91qwTp5*XW50_iTOre<3lzn94mDTkg?|#^T{^O|eJmEj>8rQtN_jdF6s!jj8*;p*~ So9{E_zK)ZC+CXk(a^xE`jg@Wy