From df8c4eb333e6163c64bd84f42ed72a3c1c283b44 Mon Sep 17 00:00:00 2001 From: Jan Stevens Date: Thu, 26 Dec 2019 15:25:53 +0100 Subject: [PATCH 1/4] Use threads for uploads --- lib/asset_sync/config.rb | 3 +++ lib/asset_sync/storage.rb | 18 ++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/asset_sync/config.rb b/lib/asset_sync/config.rb index f0a5384b..bf0daaeb 100644 --- a/lib/asset_sync/config.rb +++ b/lib/asset_sync/config.rb @@ -52,6 +52,7 @@ class Invalid < StandardError; end # Azure Blob with Fog::AzureRM attr_accessor :azure_storage_account_name attr_accessor :azure_storage_access_key + attr_accessor :concurrent_uploads validates :existing_remote_files, :inclusion => { :in => %w(keep delete ignore) } @@ -66,6 +67,7 @@ class Invalid < StandardError; end validates :google_storage_access_key_id, :presence => true, :if => :google_interop? validates :google_json_key_location, :presence => true, :if => :google_service_account? validates :google_project, :presence => true, :if => :google_service_account? + validates :concurrent_uploads, :inclusion => { :in => [true, false] } def initialize self.fog_region = nil @@ -84,6 +86,7 @@ def initialize self.invalidate = [] self.cache_asset_regexps = [] self.include_manifest = false + self.concurrent_uploads = false @additional_local_file_paths_procs = [] load_yml! if defined?(::Rails) && yml_exists? diff --git a/lib/asset_sync/storage.rb b/lib/asset_sync/storage.rb index b5925665..7e6a702c 100644 --- a/lib/asset_sync/storage.rb +++ b/lib/asset_sync/storage.rb @@ -249,10 +249,20 @@ def upload_files local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq - # Upload new files - local_files_to_upload.each do |f| - next unless File.file? "#{path}/#{f}" # Only files. - upload_file f + if self.config.concurrent_uploads + threads = ThreadGroup.new + # Upload new files + local_files_to_upload.each do |f| + next unless File.file? "#{path}/#{f}" # Only files. + threads.add(Thread.new { upload_file f }) + end + sleep 1 while threads.list.any? # wait for threads to finish uploading + else + # Upload new files + local_files_to_upload.each do |f| + next unless File.file? "#{path}/#{f}" # Only files. + upload_file f + end end if self.config.cdn_distribution_id && files_to_invalidate.any? From 0906f15c9823dd52cb446b84b6cf7cfeeb8add9a Mon Sep 17 00:00:00 2001 From: Jan Stevens Date: Fri, 3 Jan 2020 12:13:04 +0100 Subject: [PATCH 2/4] Add concurrent uploads config in the same way as include_manifest --- CHANGELOG.md | 3 ++- README.md | 4 ++++ lib/asset_sync/config.rb | 2 ++ lib/asset_sync/engine.rb | 1 + lib/generators/asset_sync/templates/asset_sync.rb | 3 +++ 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d01b7bf..baf1fc38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Added -- Nothing +- Add option `concurrent_uploads` to improve speed of uploading + (https://github.com/AssetSync/asset_sync/pull/393) ### Changed diff --git a/README.md b/README.md index af9f8f8c..d6c00328 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,9 @@ AssetSync.configure do |config| # Upload the manifest file also. # config.include_manifest = false # + # Upload files concurrently + # config.concurrent_uploads = false + # # Fail silently. Useful for environments such as Heroku # config.fail_silently = true # @@ -338,6 +341,7 @@ AssetSync.config.gzip_compression == ENV['ASSET_SYNC_GZIP_COMPRESSION'] * **gzip\_compression**: (`true, false`) when enabled, will automatically replace files that have a gzip compressed equivalent with the compressed version. **default:** `'false'` * **manifest**: (`true, false`) when enabled, will use the `manifest.yml` generated by Rails to get the list of local files to upload. **experimental**. **default:** `'false'` * **include_manifest**: (`true, false`) when enabled, will upload the `manifest.yml` generated by Rails. **default:** `'false'` +* **concurrent_uploads**: (`true, false`) when enabled, will upload the files in different Threads, this greatly improves the upload speed. **default:** `'false'` * **enabled**: (`true, false`) when false, will disable asset sync. **default:** `'true'` (enabled) * **ignored\_files**: an array of files to ignore e.g. `['ignore_me.js', %r(ignore_some/\d{32}\.css)]` Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy **default**: `[]` * **cache\_asset\_regexps**: an array of files to add cache headers e.g. `['cache_me.js', %r(cache_some\.\d{8}\.css)]` Useful if there are some files that are added to sprockets assets list and need to be set as 'Cacheable' on uploaded server. Only rails compiled regexp is matched internally **default**: `[]` diff --git a/lib/asset_sync/config.rb b/lib/asset_sync/config.rb index bf0daaeb..dbc25295 100644 --- a/lib/asset_sync/config.rb +++ b/lib/asset_sync/config.rb @@ -26,6 +26,7 @@ class Invalid < StandardError; end attr_accessor :cdn_distribution_id attr_accessor :cache_asset_regexps attr_accessor :include_manifest + attr_accessor :concurrent_uploads attr_writer :public_path # FOG configuration @@ -209,6 +210,7 @@ def load_yml! self.cdn_distribution_id = yml['cdn_distribution_id'] if yml.has_key?("cdn_distribution_id") self.cache_asset_regexps = yml['cache_asset_regexps'] if yml.has_key?("cache_asset_regexps") self.include_manifest = yml['include_manifest'] if yml.has_key?("include_manifest") + self.concurrent_uploads = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads') self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name") self.azure_storage_access_key = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key") diff --git a/lib/asset_sync/engine.rb b/lib/asset_sync/engine.rb index 069fb0d5..2d34ec34 100644 --- a/lib/asset_sync/engine.rb +++ b/lib/asset_sync/engine.rb @@ -42,6 +42,7 @@ class Engine < Rails::Engine config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION') config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST') config.include_manifest = (ENV['ASSET_SYNC_INCLUDE_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_INCLUDE_MANIFEST') + config.concurrent_uploads = (ENV['ASSET_SYNC_CONCURRENT_UPLOADS'] == 'true') if ENV.has_key?('ASSET_SYNC_CONCURRENT_UPLOADS') end config.prefix = ENV['ASSET_SYNC_PREFIX'] if ENV.has_key?('ASSET_SYNC_PREFIX') diff --git a/lib/generators/asset_sync/templates/asset_sync.rb b/lib/generators/asset_sync/templates/asset_sync.rb index 38981091..5f92dc3a 100644 --- a/lib/generators/asset_sync/templates/asset_sync.rb +++ b/lib/generators/asset_sync/templates/asset_sync.rb @@ -63,6 +63,9 @@ # Upload the manifest file also. # config.include_manifest = false # + # Upload files concurrently + # config.concurrent_uploads = false + # # Fail silently. Useful for environments such as Heroku # config.fail_silently = true # From 175e84e21fd448273ca3a004af6667caad6eb0f6 Mon Sep 17 00:00:00 2001 From: Jan Stevens Date: Fri, 3 Jan 2020 12:19:53 +0100 Subject: [PATCH 3/4] Add unit test for threaded upload --- spec/unit/storage_spec.rb | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/spec/unit/storage_spec.rb b/spec/unit/storage_spec.rb index b54f7469..3cc5d7f6 100644 --- a/spec/unit/storage_spec.rb +++ b/spec/unit/storage_spec.rb @@ -54,6 +54,22 @@ storage.upload_files end + it 'should upload files concurrently if enabled' do + @config.concurrent_uploads = true + storage = AssetSync::Storage.new(@config) + + allow(storage).to receive(:get_local_files).and_return(@local_files) + allow(storage).to receive(:get_remote_files).and_return(@remote_files) + allow(File).to receive(:file?).and_return(true) # Pretend they all exist + + expect(Thread).to receive(:new).exactly(3).times.and_call_original + (@local_files - @remote_files + storage.always_upload_files).each do |file| + expect(storage).to receive(:upload_file).with(file) + end + + storage.upload_files + end + it 'should upload updated non-fingerprinted files' do @local_files = [ 'public/image.png', @@ -125,7 +141,7 @@ end end - it 'should upload additonal files' do + it 'should upload additonal files' do @local_files = [ 'public/image.png', 'public/image-82389298328.png', From 55a8c6190f49d6c7fc8253c73666b15b6159b338 Mon Sep 17 00:00:00 2001 From: Jan Stevens Date: Fri, 3 Jan 2020 12:20:59 +0100 Subject: [PATCH 4/4] Remove duplicated attr accessor --- lib/asset_sync/config.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/asset_sync/config.rb b/lib/asset_sync/config.rb index dbc25295..cba7c019 100644 --- a/lib/asset_sync/config.rb +++ b/lib/asset_sync/config.rb @@ -53,7 +53,6 @@ class Invalid < StandardError; end # Azure Blob with Fog::AzureRM attr_accessor :azure_storage_account_name attr_accessor :azure_storage_access_key - attr_accessor :concurrent_uploads validates :existing_remote_files, :inclusion => { :in => %w(keep delete ignore) }