From b160fe7d360ddaad1b2c63f8214b18a96ae9b390 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Tue, 13 Feb 2018 17:58:28 -0600 Subject: [PATCH] Fallback to archive.org URLs for failed downloads of FOSS packages --- Core/Net/AutoUpdate.cs | 4 +-- Core/Net/Net.cs | 22 ++++++------ Core/Net/NetAsyncDownloader.cs | 25 ++++++++++--- Core/Net/NetAsyncModulesDownloader.cs | 1 + Core/Types/CkanModule.cs | 16 +++++++++ Core/Types/License.cs | 52 +++++++++++++++++++++++++-- 6 files changed, 102 insertions(+), 18 deletions(-) diff --git a/Core/Net/AutoUpdate.cs b/Core/Net/AutoUpdate.cs index 270710f3cf..6ce0722a23 100644 --- a/Core/Net/AutoUpdate.cs +++ b/Core/Net/AutoUpdate.cs @@ -152,8 +152,8 @@ public void StartUpdateProcess(bool launchCKANAfterUpdate, IUser user = null) Net.DownloadWithProgress( new[] { - new Net.DownloadTarget(fetchedUpdaterUrl.Item1, updaterFilename, fetchedUpdaterUrl.Item2), - new Net.DownloadTarget(fetchedCkanUrl.Item1, ckanFilename, fetchedCkanUrl.Item2), + new Net.DownloadTarget(fetchedUpdaterUrl.Item1, null, updaterFilename, fetchedUpdaterUrl.Item2), + new Net.DownloadTarget(fetchedCkanUrl.Item1, null, ckanFilename, fetchedCkanUrl.Item2), }, user ); diff --git a/Core/Net/Net.cs b/Core/Net/Net.cs index 3c1172e4a1..9f453d9eda 100644 --- a/Core/Net/Net.cs +++ b/Core/Net/Net.cs @@ -121,19 +121,21 @@ public static string Download(string url, string filename = null, IUser user = n public class DownloadTarget { - public Uri url { get; private set; } - public string filename { get; private set; } - public long size { get; private set; } - public string mimeType { get; private set; } + public Uri url { get; private set; } + public Uri fallbackUrl { get; private set; } + public string filename { get; private set; } + public long size { get; private set; } + public string mimeType { get; private set; } - public DownloadTarget(Uri url, string filename = null, long size = 0, string mimeType = "") + public DownloadTarget(Uri url, Uri fallback = null, string filename = null, long size = 0, string mimeType = "") { - this.url = url; - this.filename = string.IsNullOrEmpty(filename) + this.url = url; + this.fallbackUrl = fallback; + this.filename = string.IsNullOrEmpty(filename) ? FileTransaction.GetTempFileName() : filename; - this.size = size; - this.mimeType = mimeType; + this.size = size; + this.mimeType = mimeType; } } @@ -144,7 +146,7 @@ public static string DownloadWithProgress(string url, string filename = null, IU public static string DownloadWithProgress(Uri url, string filename = null, IUser user = null) { - var targets = new[] {new DownloadTarget(url, filename)}; + var targets = new[] {new DownloadTarget(url, null, filename)}; DownloadWithProgress(targets, user); return targets.First().filename; } diff --git a/Core/Net/NetAsyncDownloader.cs b/Core/Net/NetAsyncDownloader.cs index b1b402ecfa..63e82dc7d8 100644 --- a/Core/Net/NetAsyncDownloader.cs +++ b/Core/Net/NetAsyncDownloader.cs @@ -22,18 +22,22 @@ public class NetAsyncDownloader private class NetAsyncDownloaderDownloadPart { public Uri url; + public Uri fallbackUrl; public WebClient agent = new WebClient(); public DateTime lastProgressUpdateTime; public string path; public long bytesLeft; public long size; public int bytesPerSecond; + public bool triedFallback; public Exception error; public int lastProgressUpdateSize; public NetAsyncDownloaderDownloadPart(Net.DownloadTarget target, string path = null) { this.url = target.url; + this.fallbackUrl = target.fallbackUrl; + this.triedFallback = false; this.path = path ?? Path.GetTempFileName(); size = bytesLeft = target.size; lastProgressUpdateTime = DateTime.Now; @@ -426,16 +430,29 @@ private void FileDownloadComplete(int index, Exception error) if (error != null) { log.InfoFormat("Error downloading {0}: {1}", downloads[index].url, error); + + // Check whether we were already downloading the fallback url + if (!downloads[index].triedFallback && downloads[index].fallbackUrl != null) + { + log.InfoFormat("Trying fallback URL: {0}", downloads[index].fallbackUrl); + // Try the fallbackUrl + downloads[index].triedFallback = true; + downloads[index].agent.DownloadFileAsync(downloads[index].fallbackUrl, downloads[index].path); + // Short circuit the completion process so the fallback can run + return; + } + else + { + // If there was an error, remember it, but we won't raise it until + // all downloads are finished or cancelled. + downloads[index].error = error; + } } else { log.InfoFormat("Finished downloading {0}", downloads[index].url); } - // If there was an error, remember it, but we won't raise it until - // all downloads are finished or cancelled. - downloads[index].error = error; - if (++completed_downloads == downloads.Count) { FinalizeDownloads(); diff --git a/Core/Net/NetAsyncModulesDownloader.cs b/Core/Net/NetAsyncModulesDownloader.cs index eed0fdab3a..d1b15e67b7 100644 --- a/Core/Net/NetAsyncModulesDownloader.cs +++ b/Core/Net/NetAsyncModulesDownloader.cs @@ -57,6 +57,7 @@ public void DownloadModules(NetModuleCache cache, IEnumerable module downloader.DownloadAndWait( unique_downloads.Select(item => new Net.DownloadTarget( item.Key, + item.Value.InternetArchiveDownload, // Use a temp file name null, item.Value.download_size, diff --git a/Core/Types/CkanModule.cs b/Core/Types/CkanModule.cs index 2671d6046b..066f6d2141 100644 --- a/Core/Types/CkanModule.cs +++ b/Core/Types/CkanModule.cs @@ -623,6 +623,22 @@ public string DescribeInstallStanzas() return string.Join(", ", descriptions); } + /// + /// Return an archive.org URL for this download, or null if it's not there. + /// The filenames look a lot like the filenames in Net.Cache, but don't be fooled! + /// Here it's the first 8 characters of the SHA1 of the DOWNLOADED FILE, not the URL! + /// + public Uri InternetArchiveDownload + { + get + { + return license.All(l => l.Redistributable) + ? new Uri( + $"https://archive.org/download/{identifier}-{version}/{download_hash.sha1.Substring(0, 8)}-{identifier}-{version}.zip") + : null; + } + } + /// /// Format a byte count into readable file size /// diff --git a/Core/Types/License.cs b/Core/Types/License.cs index df567d55a4..c9f57ed58c 100644 --- a/Core/Types/License.cs +++ b/Core/Types/License.cs @@ -13,7 +13,8 @@ public class License public static License UnknownLicense => _unknownLicense ?? (_unknownLicense = new License("unknown")); // TODO: It would be lovely for our build system to write these for us. - private static readonly HashSet valid_licenses = new HashSet { + private static readonly HashSet valid_licenses = new HashSet() + { "public-domain", "AFL-3.0", "AGPL-3.0", @@ -49,6 +50,39 @@ public class License "open-source", "restricted", "unrestricted", "unknown" }; + private static readonly HashSet redistributable_licenses = new HashSet() + { + "public-domain", + "Apache", "Apache-1.0", "Apache-2.0", + "Artistic", "Artistic-1.0", "Artistic-2.0", + "BSD-2-clause", "BSD-3-clause", "BSD-4-clause", + "ISC", + "CC-BY", "CC-BY-1.0", "CC-BY-2.0", "CC-BY-2.5", "CC-BY-3.0", "CC-BY-4.0", + "CC-BY-SA", "CC-BY-SA-1.0", "CC-BY-SA-2.0", "CC-BY-SA-2.5", "CC-BY-SA-3.0", "CC-BY-SA-4.0", + "CC-BY-NC", "CC-BY-NC-1.0", "CC-BY-NC-2.0", "CC-BY-NC-2.5", "CC-BY-NC-3.0", "CC-BY-NC-4.0", + "CC-BY-NC-SA", "CC-BY-NC-SA-1.0", "CC-BY-NC-SA-2.0", "CC-BY-NC-SA-2.5", "CC-BY-NC-SA-3.0", "CC-BY-NC-SA-4.0", + "CC-BY-NC-ND", "CC-BY-NC-ND-1.0", "CC-BY-NC-ND-2.0", "CC-BY-NC-ND-2.5", "CC-BY-NC-ND-3.0", "CC-BY-NC-ND-4.0", + "CC0", + "CDDL", "CPL", + "EFL-1.0", "EFL-2.0", + "Expat", "MIT", + "GPL-1.0", "GPL-2.0", "GPL-3.0", + "LGPL-2.0", "LGPL-2.1", "LGPL-3.0", + "GFDL-1.0", "GFDL-1.1", "GFDL-1.2", "GFDL-1.3", + "GFDL-NIV-1.0", "GFDL-NIV-1.1", "GFDL-NIV-1.2", "GFDL-NIV-1.3", + "LPPL-1.0", "LPPL-1.1", "LPPL-1.2", "LPPL-1.3c", + "MPL-1.1", + "Perl", + "Python-2.0", + "QPL-1.0", + "W3C", + "Zlib", + "Zope", + "WTFPL", + "Unlicense", + "open-source", "unrestricted" + }; + private string license; /// @@ -69,6 +103,21 @@ public License(string license) this.license = license; } + /// + /// Return whether this license permits CKAN and others to redistribute the module. + /// We automatically upload such mods to https://archive.org/details/kspckanmods + /// + /// + /// True if redistributable, false otherwise. + /// + public bool Redistributable + { + get + { + return redistributable_licenses.Contains(license); + } + } + /// /// Returns the license as a string. /// @@ -78,4 +127,3 @@ public override string ToString() } } } -