Skip to content

Commit

Permalink
Merge #3666 Resume failed downloads
Browse files Browse the repository at this point in the history
  • Loading branch information
@HebaruSan
HebaruSan committed Sep 13, 2022
2 parents 7e2636f + c5a6b77 commit 140ab77
Show file tree
Hide file tree
Showing 33 changed files with 664 additions and 236 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ All notable changes to this project will be documented in this file.
- [Multiple] Improvements for failed repo updates (#3645 by: HebaruSan; reviewed: techman83)
- [GUI] Highlight incompatible mods recursively (#3651 by: HebaruSan; reviewed: techman83)
- [GUI] Support mouse back/forward buttons (#3655 by: HebaruSan; reviewed: techman83)
- [Core] Resume failed downloads (#3666 by: HebaruSan; reviewed: techman83)

## Bugfixes

Expand Down
23 changes: 23 additions & 0 deletions Core/Extensions/IOExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,5 +54,28 @@ public static DriveInfo GetDrive(this DirectoryInfo dir)
.OrderByDescending(dr => dr.RootDirectory.FullName.Length)
.FirstOrDefault();

/// <summary>
/// A version of Stream.CopyTo with progress updates.
/// </summary>
/// <param name="src">Stream from which to copy</param>
/// <param name="dest">Stream to which to copy</param>
/// <param name="progress">Callback to notify as we traverse the input, called with count of bytes received</param>
public static void CopyTo(this Stream src, Stream dest, IProgress<long> progress)
{
// CopyTo says its default buffer is 81920, but we want more than 1 update for a 100 KiB file
const int bufSize = 8192;
var buffer = new byte[bufSize];
long total = 0;
while (true)
{
var bytesRead = src.Read(buffer, 0, bufSize);
if (bytesRead == 0)
{
break;
}
dest.Write(buffer, 0, bytesRead);
progress.Report(total += bytesRead);
}
}
}
}
52 changes: 29 additions & 23 deletions Core/ModuleInstaller.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,20 +155,11 @@ public void InstallList(ICollection<CkanModule> modules, RelationshipResolverOpt

foreach (CkanModule module in modsToInstall)
{
User.RaiseMessage(" * {0}", Cache.DescribeAvailability(module));
if (!Cache.IsMaybeCachedZip(module))
{
User.RaiseMessage(" * {0} {1} ({2}, {3})",
module.name,
module.version,
module.download.Host,
CkanModule.FmtSize(module.download_size)
);
downloads.Add(module);
}
else
{
User.RaiseMessage(Properties.Resources.ModuleInstallerModuleCached, module.name, module.version);
}
}

if (ConfirmPrompt && !User.RaiseYesNoDialog("Continue?"))
Expand Down Expand Up @@ -1053,12 +1044,21 @@ public void Upgrade(IEnumerable<CkanModule> modules, IDownloader netAsyncDownloa
{
if (!Cache.IsMaybeCachedZip(module))
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingUncached,
module.name,
module.version,
module.download.Host,
CkanModule.FmtSize(module.download_size)
);
var inProgressFile = new FileInfo(Cache.GetInProgressFileName(module));
if (inProgressFile.Exists)
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingResuming,
module.name, module.version,
module.download.Host,
CkanModule.FmtSize(module.download_size - inProgressFile.Length));
}
else
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingUncached,
module.name, module.version,
module.download.Host,
CkanModule.FmtSize(module.download_size));
}
}
else
{
Expand Down Expand Up @@ -1086,13 +1086,19 @@ public void Upgrade(IEnumerable<CkanModule> modules, IDownloader netAsyncDownloa
{
if (!Cache.IsMaybeCachedZip(module))
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingUncached,
module.name,
installed.version,
module.version,
module.download.Host,
CkanModule.FmtSize(module.download_size)
);
var inProgressFile = new FileInfo(Cache.GetInProgressFileName(module));
if (inProgressFile.Exists)
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingResuming,
module.name, installed.version, module.version,
module.download.Host, CkanModule.FmtSize(module.download_size - inProgressFile.Length));
}
else
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingUncached,
module.name, installed.version, module.version,
module.download.Host, CkanModule.FmtSize(module.download_size));
}
}
else
{
Expand Down
108 changes: 3 additions & 105 deletions Core/Net/Net.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;

using Autofac;
using ChinhDo.Transactions.FileManager;
using log4net;

using CKAN.Configuration;

namespace CKAN
Expand All @@ -25,7 +27,7 @@ public class Net
private const int MaxRetries = 3;
private const int RetryDelayMilliseconds = 100;

private static readonly ILog log = LogManager.GetLogger(typeof(Net));
private static readonly ILog log = LogManager.GetLogger(typeof(Net));

public static readonly Dictionary<string, Uri> ThrottledHosts = new Dictionary<string, Uri>()
{
Expand Down Expand Up @@ -355,109 +357,5 @@ public static Uri GetRawUri(Uri remoteUri)
return remoteUri;
}
}

/// <summary>
/// A WebClient with some CKAN-sepcific adjustments:
/// - A user agent string (required by GitHub API policy)
/// - Sets the Accept header to a given MIME type (needed to get raw files from GitHub API)
/// - Times out after a specified amount of time in milliseconds, 100 000 milliseconds (=100 seconds) by default (https://stackoverflow.com/a/3052637)
/// - Handles permanent redirects to the same host without clearing the Authorization header (needed to get files from renamed GitHub repositories via API)
/// </summary>
private sealed class RedirectingTimeoutWebClient : WebClient
{
/// <summary>
/// Initialize our special web client
/// </summary>
/// <param name="timeout">Timeout for the request in milliseconds, defaulting to 100 000 (=100 seconds)</param>
/// <param name="mimeType">A mime type sent with the "Accept" header</param>
public RedirectingTimeoutWebClient(int timeout = 100000, string mimeType = "")
{
this.timeout = timeout;
this.mimeType = mimeType;
}

protected override WebRequest GetWebRequest(Uri address)
{
// Set user agent and MIME type for every request. including redirects
Headers.Add("User-Agent", UserAgentString);
if (!string.IsNullOrEmpty(mimeType))
{
log.InfoFormat("Setting MIME type {0}", mimeType);
Headers.Add("Accept", mimeType);
}
if (permanentRedirects.TryGetValue(address, out Uri redirUri))
{
// Obey a previously received permanent redirect
address = redirUri;
}
var request = base.GetWebRequest(address);
if (request is HttpWebRequest hwr)
{
// GitHub API tokens cannot be passed via auto-redirect
hwr.AllowAutoRedirect = false;
hwr.Timeout = timeout;
}
return request;
}

protected override WebResponse GetWebResponse(WebRequest request)
{
if (request == null)
return null;
var response = base.GetWebResponse(request);
if (response == null)
return null;

if (response is HttpWebResponse hwr)
{
int statusCode = (int)hwr.StatusCode;
var location = hwr.Headers["Location"];
if (statusCode >= 300 && statusCode <= 399 && location != null)
{
log.InfoFormat("Redirecting to {0}", location);
hwr.Close();
var redirUri = new Uri(request.RequestUri, location);
if (Headers.AllKeys.Contains("Authorization")
&& request.RequestUri.Host != redirUri.Host)
{
log.InfoFormat("Host mismatch, purging token for redirect");
Headers.Remove("Authorization");
}
// Moved or PermanentRedirect
if (statusCode == 301 || statusCode == 308)
{
permanentRedirects.Add(request.RequestUri, redirUri);
}
return GetWebResponse(GetWebRequest(redirUri));
}
}
return response;
}

private int timeout;
private string mimeType;
private static readonly Dictionary<Uri, Uri> permanentRedirects = new Dictionary<Uri, Uri>();
}

// HACK: The ancient WebClient doesn't support setting the request type to HEAD and WebRequest doesn't support
// setting the User-Agent header.
// Maybe one day we'll be able to use HttpClient (https://msdn.microsoft.com/en-us/library/system.net.http.httpclient%28v=vs.118%29.aspx)
private sealed class RedirectWebClient : WebClient
{
public RedirectWebClient()
{
Headers.Add("User-Agent", UserAgentString);
}

protected override WebRequest GetWebRequest(Uri address)
{
var webRequest = (HttpWebRequest)base.GetWebRequest(address);
webRequest.AllowAutoRedirect = false;

webRequest.Method = "HEAD";

return webRequest;
}
}
}
}
24 changes: 14 additions & 10 deletions Core/Net/NetAsyncDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,14 @@ private class NetAsyncDownloaderDownloadPart
public Exception error;
public int lastProgressUpdateSize;

public event DownloadProgressChangedEventHandler Progress;
/// <summary>
/// Percentage, bytes received, total bytes to receive
/// </summary>
public event Action<int, long, long> Progress;
public event Action<object, AsyncCompletedEventArgs, string> Done;

private string mimeType;
private WebClient agent;
private ResumingWebClient agent;

public NetAsyncDownloaderDownloadPart(Net.DownloadTarget target)
{
Expand All @@ -51,7 +54,7 @@ public NetAsyncDownloaderDownloadPart(Net.DownloadTarget target)
public void Download(Uri url, string path)
{
ResetAgent();
agent.DownloadFileAsync(url, path);
agent.DownloadFileAsyncWithResume(url, path);
}

public void Abort()
Expand All @@ -61,7 +64,7 @@ public void Abort()

private void ResetAgent()
{
agent = new WebClient();
agent = new ResumingWebClient();

agent.Headers.Add("User-Agent", Net.UserAgentString);

Expand All @@ -86,7 +89,11 @@ private void ResetAgent()
// Forward progress and completion events to our listeners
agent.DownloadProgressChanged += (sender, args) =>
{
Progress?.Invoke(sender, args);
Progress?.Invoke(args.ProgressPercentage, args.BytesReceived, args.TotalBytesToReceive);
};
agent.DownloadProgress += (percent, bytesReceived, totalBytesToReceive) =>
{
Progress?.Invoke(percent, bytesReceived, totalBytesToReceive);
};
agent.DownloadFileCompleted += (sender, args) =>
{
Expand Down Expand Up @@ -163,11 +170,8 @@ private void DownloadModule(Net.DownloadTarget target)
dl.target.url.ToString().Replace(" ", "%20"));

// Schedule for us to get back progress reports.
dl.Progress += (sender, args) =>
FileProgressReport(index,
args.ProgressPercentage,
args.BytesReceived,
args.TotalBytesToReceive);
dl.Progress += (ProgressPercentage, BytesReceived, TotalBytesToReceive) =>
FileProgressReport(index, ProgressPercentage, BytesReceived, TotalBytesToReceive);

// And schedule a notification if we're done (or if something goes wrong)
dl.Done += (sender, args, etag) =>
Expand Down
29 changes: 14 additions & 15 deletions Core/Net/NetAsyncModulesDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
using System.IO;
using System.Linq;

using ChinhDo.Transactions.FileManager;
using log4net;

namespace CKAN
Expand Down Expand Up @@ -62,33 +61,28 @@ public void DownloadModules(IEnumerable<CkanModule> modules)
.Where(group => !currentlyActive.Contains(group.Key))
.ToDictionary(group => group.Key, group => group.First());

// Make sure we have enough space to download this stuff
var downloadSize = unique_downloads.Values.Select(m => m.download_size).Sum();
CKANPathUtils.CheckFreeSpace(new DirectoryInfo(new TxFileManager().GetTempDirectory()),
downloadSize,
Properties.Resources.NotEnoughSpaceToDownload);
// Make sure we have enough space to cache this stuff
cache.CheckFreeSpace(downloadSize);
// Make sure we have enough space to download and cache
cache.CheckFreeSpace(unique_downloads.Values
.Select(m => m.download_size)
.Sum());

this.modules.AddRange(unique_downloads.Values);

try
{
// Start the downloads!
downloader.DownloadAndWait(
unique_downloads.Select(item => new Net.DownloadTarget(
downloader.DownloadAndWait(unique_downloads
.Select(item => new Net.DownloadTarget(
item.Key,
item.Value.InternetArchiveDownload,
// Use a temp file name
null,
cache.GetInProgressFileName(item.Value),
item.Value.download_size,
// Send the MIME type to use for the Accept header
// The GitHub API requires this to include application/octet-stream
string.IsNullOrEmpty(item.Value.download_content_type)
? defaultMimeType
: $"{item.Value.download_content_type};q=1.0,{defaultMimeType};q=0.9"
)).ToList()
);
: $"{item.Value.download_content_type};q=1.0,{defaultMimeType};q=0.9"))
.ToList());
this.modules.Clear();
AllComplete?.Invoke();
}
Expand All @@ -104,8 +98,11 @@ public void DownloadModules(IEnumerable<CkanModule> modules)

private void ModuleDownloadComplete(Uri url, string filename, Exception error, string etag)
{
log.DebugFormat("Received download completion: {0}, {1}, {2}",
url, filename, error?.Message);
if (error != null)
{
// If there was an error in DOWNLOADING, keep the file so we can retry it later
log.Info(error.ToString());
}
else
Expand All @@ -120,6 +117,8 @@ private void ModuleDownloadComplete(Uri url, string filename, Exception error, s
catch (InvalidModuleFileKraken kraken)
{
User.RaiseError(kraken.ToString());
// If there was an error in STORING, delete the file so we can try it from scratch later
File.Delete(filename);
}
catch (FileNotFoundException e)
{
Expand Down
Loading

0 comments on commit 140ab77

Please sign in to comment.