Fallback to archive.org URLs for failed downloads of FOSS packages

Core/Net/AutoUpdate.cs

@@ -152,8 +152,8 @@ public void StartUpdateProcess(bool launchCKANAfterUpdate, IUser user = null)
             Net.DownloadWithProgress(
                 new[]
                 {
-                    new Net.DownloadTarget(fetchedUpdaterUrl.Item1, updaterFilename, fetchedUpdaterUrl.Item2),
-                    new Net.DownloadTarget(fetchedCkanUrl.Item1,    ckanFilename,    fetchedCkanUrl.Item2),
+                    new Net.DownloadTarget(fetchedUpdaterUrl.Item1, null, updaterFilename, fetchedUpdaterUrl.Item2),
+                    new Net.DownloadTarget(fetchedCkanUrl.Item1,    null, ckanFilename,    fetchedCkanUrl.Item2),
                 },
                 user
             );

Core/Net/Net.cs

@@ -121,19 +121,21 @@ public static string Download(string url, string filename = null, IUser user = n
 
         public class DownloadTarget
         {
-            public Uri    url      { get; private set; }
-            public string filename { get; private set; }
-            public long   size     { get; private set; }
-            public string mimeType { get; private set; }
+            public Uri    url         { get; private set; }
+            public Uri    fallbackUrl { get; private set; }
+            public string filename    { get; private set; }
+            public long   size        { get; private set; }
+            public string mimeType    { get; private set; }
 
-            public DownloadTarget(Uri url, string filename = null, long size = 0, string mimeType = "")
+            public DownloadTarget(Uri url, Uri fallback = null, string filename = null, long size = 0, string mimeType = "")
             {
-                this.url      = url;
-                this.filename = string.IsNullOrEmpty(filename)
+                this.url         = url;
+                this.fallbackUrl = fallback;
+                this.filename    = string.IsNullOrEmpty(filename)
                     ? FileTransaction.GetTempFileName()
                     : filename;
-                this.size     = size;
-                this.mimeType = mimeType;
+                this.size        = size;
+                this.mimeType    = mimeType;
             }
         }
 
@@ -144,7 +146,7 @@ public static string DownloadWithProgress(string url, string filename = null, IU
 
         public static string DownloadWithProgress(Uri url, string filename = null, IUser user = null)
         {
-            var targets = new[] {new DownloadTarget(url, filename)};
+            var targets = new[] {new DownloadTarget(url, null, filename)};
             DownloadWithProgress(targets, user);
             return targets.First().filename;
         }

Core/Net/NetAsyncDownloader.cs

@@ -22,18 +22,22 @@ public class NetAsyncDownloader
         private class NetAsyncDownloaderDownloadPart
         {
             public Uri url;
+            public Uri fallbackUrl;
             public WebClient agent = new WebClient();
             public DateTime lastProgressUpdateTime;
             public string path;
             public long bytesLeft;
             public long size;
             public int bytesPerSecond;
+            public bool triedFallback;
             public Exception error;
             public int lastProgressUpdateSize;
 
             public NetAsyncDownloaderDownloadPart(Net.DownloadTarget target, string path = null)
             {
                 this.url  = target.url;
+                this.fallbackUrl = target.fallbackUrl;
+                this.triedFallback = false;
                 this.path = path ?? Path.GetTempFileName();
                 size = bytesLeft = target.size;
                 lastProgressUpdateTime = DateTime.Now;
@@ -426,16 +430,29 @@ private void FileDownloadComplete(int index, Exception error)
             if (error != null)
             {
                 log.InfoFormat("Error downloading {0}: {1}", downloads[index].url, error);
+
+                // Check whether we were already downloading the fallback url
+                if (!downloads[index].triedFallback && downloads[index].fallbackUrl != null)
+                {
+                    log.InfoFormat("Trying fallback URL: {0}", downloads[index].fallbackUrl);
+                    // Try the fallbackUrl
+                    downloads[index].triedFallback = true;
+                    downloads[index].agent.DownloadFileAsync(downloads[index].fallbackUrl, downloads[index].path);
+                    // Short circuit the completion process so the fallback can run
+                    return;
+                }
+                else
+                {
+                    // If there was an error, remember it, but we won't raise it until
+                    // all downloads are finished or cancelled.
+                    downloads[index].error = error;
+                }
             }
             else
             {
                 log.InfoFormat("Finished downloading {0}", downloads[index].url);
             }
 
-            // If there was an error, remember it, but we won't raise it until
-            // all downloads are finished or cancelled.
-            downloads[index].error = error;
-
             if (++completed_downloads == downloads.Count)
             {
                 FinalizeDownloads();

Core/Net/NetAsyncModulesDownloader.cs

@@ -57,6 +57,7 @@ public void DownloadModules(NetModuleCache cache, IEnumerable<CkanModule> module
                 downloader.DownloadAndWait(
                     unique_downloads.Select(item => new Net.DownloadTarget(
                         item.Key,
+                        item.Value.InternetArchiveDownload,
                         // Use a temp file name
                         null,
                         item.Value.download_size,

Core/Types/CkanModule.cs

@@ -623,6 +623,22 @@ public string DescribeInstallStanzas()
             return string.Join(", ", descriptions);
         }
 
+        /// <summary>
+        /// Return an archive.org URL for this download, or null if it's not there.
+        /// The filenames look a lot like the filenames in Net.Cache, but don't be fooled!
+        /// Here it's the first 8 characters of the SHA1 of the DOWNLOADED FILE, not the URL!
+        /// </summary>
+        public Uri InternetArchiveDownload
+        {
+            get
+            {
+                return license.All(l => l.Redistributable)
+                    ? new Uri(
+                        $"https://archive.org/download/{identifier}-{version}/{download_hash.sha1.Substring(0, 8)}-{identifier}-{version}.zip")
+                    : null;
+            }
+        }
+
         /// <summary>
         /// Format a byte count into readable file size
         /// </summary>

Core/Types/License.cs

@@ -13,7 +13,8 @@ public class License
         public static License UnknownLicense => _unknownLicense ?? (_unknownLicense = new License("unknown"));
 
         // TODO: It would be lovely for our build system to write these for us.
-        private static readonly HashSet<string> valid_licenses = new HashSet<string> {
+        private static readonly HashSet<string> valid_licenses = new HashSet<string>()
+        {
             "public-domain",
             "AFL-3.0",
             "AGPL-3.0",
@@ -49,6 +50,39 @@ public class License
             "open-source", "restricted", "unrestricted", "unknown"
         };
 
+        private static readonly HashSet<string> redistributable_licenses = new HashSet<string>()
+        {
+            "public-domain",
+            "Apache", "Apache-1.0", "Apache-2.0",
+            "Artistic", "Artistic-1.0", "Artistic-2.0",
+            "BSD-2-clause", "BSD-3-clause", "BSD-4-clause",
+            "ISC",
+            "CC-BY", "CC-BY-1.0", "CC-BY-2.0", "CC-BY-2.5", "CC-BY-3.0", "CC-BY-4.0",
+            "CC-BY-SA", "CC-BY-SA-1.0", "CC-BY-SA-2.0", "CC-BY-SA-2.5", "CC-BY-SA-3.0", "CC-BY-SA-4.0",
+            "CC-BY-NC", "CC-BY-NC-1.0", "CC-BY-NC-2.0", "CC-BY-NC-2.5", "CC-BY-NC-3.0", "CC-BY-NC-4.0",
+            "CC-BY-NC-SA", "CC-BY-NC-SA-1.0", "CC-BY-NC-SA-2.0", "CC-BY-NC-SA-2.5", "CC-BY-NC-SA-3.0", "CC-BY-NC-SA-4.0",
+            "CC-BY-NC-ND", "CC-BY-NC-ND-1.0", "CC-BY-NC-ND-2.0", "CC-BY-NC-ND-2.5", "CC-BY-NC-ND-3.0", "CC-BY-NC-ND-4.0",
+            "CC0",
+            "CDDL", "CPL",
+            "EFL-1.0", "EFL-2.0",
+            "Expat", "MIT",
+            "GPL-1.0", "GPL-2.0", "GPL-3.0",
+            "LGPL-2.0", "LGPL-2.1", "LGPL-3.0",
+            "GFDL-1.0", "GFDL-1.1", "GFDL-1.2", "GFDL-1.3",
+            "GFDL-NIV-1.0", "GFDL-NIV-1.1", "GFDL-NIV-1.2", "GFDL-NIV-1.3",
+            "LPPL-1.0", "LPPL-1.1", "LPPL-1.2", "LPPL-1.3c",
+            "MPL-1.1",
+            "Perl",
+            "Python-2.0",
+            "QPL-1.0",
+            "W3C",
+            "Zlib",
+            "Zope",
+            "WTFPL",
+            "Unlicense",
+            "open-source", "unrestricted"
+        };
+
         private string license;
 
         /// <summary>
@@ -69,6 +103,21 @@ public License(string license)
             this.license = license;
         }
 
+        /// <summary>
+        /// Return whether this license permits CKAN and others to redistribute the module.
+        /// We automatically upload such mods to https://archive.org/details/kspckanmods
+        /// </summary>
+        /// <returns>
+        /// True if redistributable, false otherwise.
+        /// </returns>
+        public bool Redistributable
+        {
+            get
+            {
+                return redistributable_licenses.Contains(license);
+            }
+        }
+
         /// <summary>
         /// Returns the license as a string.
         /// </summary>
@@ -78,4 +127,3 @@ public override string ToString()
         }
     }
 }
-