retry download without s3 cache flag if cache miss (#1104)

* retry download without s3 cache flag if cache miss
* changing abort on exception when fetching from internal cache
---------

Signed-off-by: Justin Gruber <justin.gruber@progress.com>

lib/omnibus/cli.rb

@@ -85,7 +85,13 @@ def build(name)
       project = Project.load(name, manifest)
       say("Building #{project.name} #{project.build_version}...")
       Omnibus::S3Cache.populate if @options[:populate_s3_cache] && !Omnibus::S3Cache.fetch_missing.empty?
-      project.download
+      begin
+        project.download
+      rescue
+        Config.use_s3_caching(false) if Config.use_s3_caching
+        project = Project.load(name, nil)
+        project.download
+      end
       project.build
 
       if @options[:output_manifest]

lib/omnibus/project.rb

@@ -1062,7 +1062,10 @@ def built_manifest
     end
 
     def download
-      ThreadPool.new(Config.workers) do |pool|
+      # Setting abort_on_exception to false because it was causing
+      # errors by shutting down the main thread when encountering a cache miss
+      # in S3 and falling back to the internal source repo
+      ThreadPool.new(Config.workers, false) do |pool|
         softwares.each do |software|
           pool.schedule { software.fetch }
         end

lib/omnibus/thread_pool.rb

@@ -46,14 +46,16 @@ class ThreadPool
     #
     # @param [Integer] size
     #   the number of items to put in the thread pool
+    # @param [Boolean] abort_on_exception
+    #   if the thread should abort the main thread also on a failure
     #
-    def initialize(size)
+    def initialize(size, abort_on_exception = true)
       @size = size
       @jobs = Queue.new
 
       @pool = Array.new(@size) do |i|
         Thread.new do
-          Thread.abort_on_exception = true
+          Thread.abort_on_exception = abort_on_exception
           Thread.current[:id] = i
 
           catch(:exit) do