Merge pull request #393 from playpasshq/feature/threaded-uploads

Use threads for uploads

CHANGELOG.md

@@ -7,7 +7,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
-- Nothing
+- Add option `concurrent_uploads` to improve speed of uploading 
+  (https://github.com/AssetSync/asset_sync/pull/393)
 
 ### Changed
 

README.md

@@ -242,6 +242,9 @@ AssetSync.configure do |config|
   # Upload the manifest file also.
   # config.include_manifest = false
   #
+  # Upload files concurrently
+  # config.concurrent_uploads = false
+  #
   # Fail silently.  Useful for environments such as Heroku
   # config.fail_silently = true
   #
@@ -338,6 +341,7 @@ AssetSync.config.gzip_compression == ENV['ASSET_SYNC_GZIP_COMPRESSION']
 * **gzip\_compression**: (`true, false`) when enabled, will automatically replace files that have a gzip compressed equivalent with the compressed version. **default:** `'false'`
 * **manifest**: (`true, false`) when enabled, will use the `manifest.yml` generated by Rails to get the list of local files to upload. **experimental**. **default:** `'false'`
 * **include_manifest**: (`true, false`) when enabled, will upload the `manifest.yml` generated by Rails. **default:** `'false'`
+* **concurrent_uploads**: (`true, false`) when enabled, will upload the files in different Threads, this greatly improves the upload speed. **default:** `'false'`
 * **enabled**: (`true, false`) when false, will disable asset sync. **default:** `'true'` (enabled)
 * **ignored\_files**: an array of files to ignore e.g. `['ignore_me.js', %r(ignore_some/\d{32}\.css)]` Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy **default**: `[]`
 * **cache\_asset\_regexps**: an array of files to add cache headers e.g. `['cache_me.js', %r(cache_some\.\d{8}\.css)]` Useful if there are some files that are added to sprockets assets list and need to be set as 'Cacheable' on uploaded server.  Only rails compiled regexp is matched internally **default**: `[]`

lib/asset_sync/config.rb

@@ -26,6 +26,7 @@ class Invalid < StandardError; end
     attr_accessor :cdn_distribution_id
     attr_accessor :cache_asset_regexps
     attr_accessor :include_manifest
+    attr_accessor :concurrent_uploads
     attr_writer :public_path
 
     # FOG configuration
@@ -66,6 +67,7 @@ class Invalid < StandardError; end
     validates :google_storage_access_key_id,      :presence => true, :if => :google_interop?
     validates :google_json_key_location,          :presence => true, :if => :google_service_account?
     validates :google_project,                    :presence => true, :if => :google_service_account?
+    validates :concurrent_uploads,    :inclusion => { :in => [true, false] }
 
     def initialize
       self.fog_region = nil
@@ -84,6 +86,7 @@ def initialize
       self.invalidate = []
       self.cache_asset_regexps = []
       self.include_manifest = false
+      self.concurrent_uploads = false
       @additional_local_file_paths_procs = []
 
       load_yml! if defined?(::Rails) && yml_exists?
@@ -206,6 +209,7 @@ def load_yml!
       self.cdn_distribution_id    = yml['cdn_distribution_id'] if yml.has_key?("cdn_distribution_id")
       self.cache_asset_regexps    = yml['cache_asset_regexps'] if yml.has_key?("cache_asset_regexps")
       self.include_manifest       = yml['include_manifest'] if yml.has_key?("include_manifest")
+      self.concurrent_uploads     = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads')
 
       self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name")
       self.azure_storage_access_key   = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key")

lib/asset_sync/engine.rb

@@ -42,6 +42,7 @@ class Engine < Rails::Engine
           config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION')
           config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
           config.include_manifest = (ENV['ASSET_SYNC_INCLUDE_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_INCLUDE_MANIFEST')
+          config.concurrent_uploads = (ENV['ASSET_SYNC_CONCURRENT_UPLOADS'] == 'true') if ENV.has_key?('ASSET_SYNC_CONCURRENT_UPLOADS')
         end
 
         config.prefix = ENV['ASSET_SYNC_PREFIX'] if ENV.has_key?('ASSET_SYNC_PREFIX')

lib/asset_sync/storage.rb

@@ -249,10 +249,20 @@ def upload_files
       local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files
       local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq
 
-      # Upload new files
-      local_files_to_upload.each do |f|
-        next unless File.file? "#{path}/#{f}" # Only files.
-        upload_file f
+      if self.config.concurrent_uploads
+        threads = ThreadGroup.new
+        # Upload new files
+        local_files_to_upload.each do |f|
+          next unless File.file? "#{path}/#{f}" # Only files.
+          threads.add(Thread.new { upload_file f })
+        end
+        sleep 1 while threads.list.any? # wait for threads to finish uploading
+      else
+        # Upload new files
+        local_files_to_upload.each do |f|
+          next unless File.file? "#{path}/#{f}" # Only files.
+          upload_file f
+        end
       end
 
       if self.config.cdn_distribution_id && files_to_invalidate.any?

lib/generators/asset_sync/templates/asset_sync.rb

@@ -63,6 +63,9 @@
     # Upload the manifest file also.
     # config.include_manifest = false
     #
+    # Upload files concurrently
+    # config.concurrent_uploads = false
+    #
     # Fail silently.  Useful for environments such as Heroku
     # config.fail_silently = true
     #

spec/unit/storage_spec.rb

@@ -54,6 +54,22 @@
       storage.upload_files
     end
 
+    it 'should upload files concurrently if enabled' do
+      @config.concurrent_uploads = true
+      storage = AssetSync::Storage.new(@config)
+
+      allow(storage).to receive(:get_local_files).and_return(@local_files)
+      allow(storage).to receive(:get_remote_files).and_return(@remote_files)
+      allow(File).to receive(:file?).and_return(true) # Pretend they all exist
+
+      expect(Thread).to receive(:new).exactly(3).times.and_call_original
+      (@local_files - @remote_files + storage.always_upload_files).each do |file|
+        expect(storage).to receive(:upload_file).with(file)
+      end
+
+      storage.upload_files
+    end
+
     it 'should upload updated non-fingerprinted files' do
       @local_files = [
         'public/image.png',
@@ -125,7 +141,7 @@
       end
     end
 
-    it 'should upload additonal  files' do
+    it 'should upload additonal files' do
       @local_files = [
         'public/image.png',
         'public/image-82389298328.png',