Merge branch 'seek-1.14' into fairdomhub

Gemfile

@@ -178,10 +178,10 @@ group :development do
   gem 'web-console', '>= 4.1.0'
   gem 'rack-mini-profiler', '~> 2.0'
   gem 'listen', '~> 3.3'
+  gem 'ruby-prof'
 end
 
 group :test do
-  gem 'ruby-prof'
   gem 'test-prof'
   gem 'rails-perftest'
   gem 'minitest', '~> 5.14'

app/assets/javascripts/single_page/dynamic_table.js.erb

@@ -213,7 +213,7 @@ const handleSelect = (e) => {
               const missingColsCount = colsCount - defaultCols.length - splitted.length;
               splitted = _defaultCols.concat(splitted);
 
-              // Preserve empty placeholder for columns of type of 'Registered Sample (multiple)'
+              // Preserve empty placeholder for columns of type of 'Registered Sample List'
               sampleLinkIndexes.forEach((x) => splitted.splice(x, 0, []));
 
               return missingColsCount < 0 ?

app/models/auth_lookup.rb

@@ -7,11 +7,12 @@ class AuthLookup < ActiveRecord::Base
 
   def self.prepare
     c = count
-    if c != (User.count + 1)  # 1 entry for each user + anonymous
-      delete_all unless c.zero?
+    user_c = User.where.not(person_id: nil).count
+    if c != (user_c + 1)  # 1 entry for each user + anonymous
+      in_batches(of: 100000, order: :desc) { |r| r.delete_all } unless c.zero?
 
       # Only need to specify user ID on insert, since all permission fields are `false` by default.
-      import [:user_id], ([0] + User.pluck(:id)).map { |i| [i] },
+      import [:user_id], ([0] + User.where.not(person_id: nil).pluck(:id)).map { |i| [i] },
              validate: false,
              batch_size: Seek::Util.bulk_insert_batch_size
     else
@@ -29,7 +30,7 @@ def self.batch_update(permission, overwrite = true)
       updates["can_#{a}"] = permission[index] if overwrite || permission[index]
     end
 
-    update_all(updates) unless updates.empty?
+    in_batches(of: 100000, order: :desc) { |r| r.update_all(updates) } unless updates.empty?
   end
 
   def as_array

app/views/content_blobs/examine_url/_override.html.erb

@@ -0,0 +1,19 @@
+<div class="alert alert-danger" role="alert">
+  <span class="error_icon"></span>
+  Processing the URL responded with a response code <%= @info[:code] %>. <%= @error_msg %>.
+</div>
+
+<div class="alert alert-warning" role="alert">
+  <span class="warning_icon"></span>
+  <b>Warning</b>: You are attempting to register a remote URL that may not be valid. Our system is designed to validate URLs to ensure security and functionality. Please double-check the URL for accuracy.
+  If you are certain that the URL is correct and want to proceed despite the validation warning, please check the box below to override the URL check:
+
+ <%  title = "I understand the risks and want to override URL validation." %>
+  <%=
+  content_tag(:div, class: 'checkbox') do
+    content_tag(:label, class: 'override_url_check') do
+        check_box_tag("content_blobs[][override_url_check]",'yes', false) + title.html_safe
+        end
+  end
+%>
+</div>

lib/seek/isa_templates/template_attributes_schema.json

@@ -58,10 +58,9 @@
 				"String",
 				"Boolean",
 				"Registered Sample",
-				"Registered Sample (multiple)",
+				"Registered Sample List",
 				"Controlled Vocabulary",
 				"URI",
-				"Ontology",
 				"Registered Data file"
 			]
 		},

lib/seek/isa_templates/template_extractor.rb

@@ -158,13 +158,19 @@ def self.valid_isa_json?(json)
       end
 
       def self.get_sample_attribute_type(title)
-        SampleAttributeType.where(title: title).first.id
+        sa = SampleAttributeType.find_by(title: title)
+        raise "Could not find a Sample Attribute named '#{title}'" if sa.nil?
+
+        sa.id
       end
 
       def self.get_isa_tag_id(title)
         return nil if title.blank?
 
-        IsaTag.where(title: title).first.id
+        it = IsaTag.find_by(title: title)
+        raise "Could not find an ISA Tag named '#{title}'" if it.nil?
+
+        it.id
       end
 
       def self.seed_isa_tags

lib/seek/json_metadata/attribute.rb

@@ -77,8 +77,8 @@ def linked_sample_type_and_attribute_type_consistency
         end
         if seek_sample? && linked_sample_type.nil?
           errors.add(:seek_sample, 'Linked Sample Type must be set if attribute type is Registered Sample')
-        elsif seek_sample_multi? && linked_sample_type.nil? 
-          errors.add(:seek_sample_multi, 'Linked Sample Type must be set if attribute type is Registered Sample (multiple)')
+        elsif seek_sample_multi? && linked_sample_type.nil?
+          errors.add(:seek_sample_multi, 'Linked Sample Type must be set if attribute type is Registered Sample List')
         end
       end
     end

lib/seek/upload_handling/data_upload.rb

@@ -144,13 +144,17 @@ def process_from_url(blob_params)
         when 'http', 'https'
           handler = Seek::DownloadHandling::HTTPHandler.new(@data_url)
           info = handler.info
-          if info[:code] == 490
-            flash.now[:error] = 'The given URL is inaccessible.'
-            return false
-          end
-          unless [200, 401, 403].include?(info[:code])
-            flash.now[:error] = "Processing the URL responded with a response code (#{info[:code]}), indicating the URL is inaccessible."
-            return false
+          if (info[:code] == 400 || 404) && blob_params[:override_url_check].present?
+            flash.now[:notice] = 'The given URL is inaccessible but you can override the url validation.'
+          else
+            if info[:code] == 490
+              flash.now[:error] = 'The given URL is inaccessible.'
+              return false
+            end
+            unless [200, 401, 403].include?(info[:code])
+              flash.now[:error] = "Processing the URL responded with a response code (#{info[:code]}), indicating the URL is inaccessible."
+              return false
+            end
           end
         when 'ftp'
           handler = Seek::DownloadHandling::FTPHandler.new(@data_url)

lib/seek/upload_handling/examine_url.rb

@@ -30,7 +30,7 @@ def examine_url
         end
 
         respond_to do |format|
-          format.html { render partial: 'content_blobs/examine_url_result', status: @type == 'error' ? 400 : 200 }
+          format.html { render partial: 'content_blobs/examine_url_result', status: ( @type == 'error'|| @type == 'override') ? 400 : 200 }
         end
       end
 
@@ -74,8 +74,10 @@ def handle_bad_http_response(code)
         when 405
           @error_msg = "We can't find out information about this URL - Method not allowed response."
         when 404
+          @type = 'override'
           @error_msg = 'Nothing can be found at that URL. Please check the address and try again'
         when 400
+          @type = 'override'
           @error_msg = 'The URL appears to be invalid'
         when 490
           @error_msg = 'That URL is inaccessible. Please check the address and try again'

lib/tasks/seek_dev.rake

@@ -4,6 +4,7 @@ require 'rubygems'
 require 'rake'
 require 'active_record/fixtures'
 require 'benchmark'
+#require 'ruby-prof'
 
 include SysMODB::SpreadsheetExtractor
 
@@ -36,6 +37,21 @@ namespace :seek_dev do
     puts output.read
   end
 
+  task(:profile_command, [:command] => :environment) do |_t, args|
+    unless args[:command].present?
+      puts "command not found"
+      puts
+      puts "Usage: bundle exec rake seek_dev:profile_command['the command']"
+      exit -1
+    end
+    result = RubyProf.profile do
+      eval(args[:command])
+    end
+    printer = RubyProf::GraphHtmlPrinter.new(result)
+    printer.print(STDOUT, {})
+  end
+
+
   task(:dump_controlled_vocab, [:id] => :environment) do |_t, args|
     vocab = SampleControlledVocab.find(args.id)
     json = { title: vocab.title, description: vocab.description, ols_root_term_uri: vocab.ols_root_term_uri,

test/functional/content_blobs_controller_test.rb

@@ -169,7 +169,8 @@ def setup
     assert_response 400
     assert_equal 404, assigns(:info)[:code]
     assert @response.body.include?('Nothing can be found at that URL')
-    assert_equal 'error', assigns(:type)
+    assert @response.body.include?('I understand the risks and want to override URL validation')
+    assert_equal 'override', assigns(:type)
     assert assigns(:error_msg)
   end
 
@@ -192,7 +193,8 @@ def setup
     assert_response 400
     assert_equal 404, assigns(:info)[:code]
     assert @response.body.include?('Nothing can be found at that URL')
-    assert_equal 'error', assigns(:type)
+    assert @response.body.include?('I understand the risks and want to override URL validation')
+    assert_equal 'override', assigns(:type)
     assert assigns(:error_msg)
   end
 
@@ -201,7 +203,8 @@ def setup
     get :examine_url, xhr: true, params: { data_url: 'this is not a uri' }
     assert_response 400
     assert @response.body.include?('The URL appears to be invalid')
-    assert_equal 'error', assigns(:type)
+    assert @response.body.include?('I understand the risks and want to override URL validation')
+    assert_equal 'override', assigns(:type)
     assert assigns(:error_msg)
   end
 
@@ -222,6 +225,7 @@ def setup
         assert_response 400
         assert_equal 490, assigns(:info)[:code]
         assert @response.body.include?('URL is inaccessible')
+        refute @response.body.include?('I understand the risks and want to override URL validation')
         assert_equal 'error', assigns(:type)
         assert assigns(:error_msg)
       end

test/functional/data_files_controller_test.rb

@@ -2081,6 +2081,70 @@ def test_show_item_attributed_to_jerm_file
     assert_equal 'http://mockedlocation.com/txt_test.txt', assigns(:data_file).content_blob.url
   end
 
+
+  test 'users should be able to override URL validation when the URL examination returns a 404 or 400 status code.' do
+    mock_http
+
+    params = { data_file: {
+      title: 'Remote File',
+      project_ids: [projects(:sysmo_project).id]
+    },
+               content_blobs: [{
+                                 data_url: 'http://mocked404.com',
+                                 make_local_copy: '1'
+                               }],
+               policy_attributes: valid_sharing }
+
+    assert_no_difference('DataFile.count') do
+      assert_no_difference('ContentBlob.count') do
+        post :create, params: params
+      end
+    end
+
+    params = { data_file: {
+      title: 'Remote File',
+      project_ids: [projects(:sysmo_project).id]
+    },
+               content_blobs: [{
+                                 data_url: 'http://mocked404.com',
+                                 make_local_copy: '1',
+                                 override_url_check: 'yes'
+                               }],
+               policy_attributes: valid_sharing }
+
+    assert_difference('DataFile.count') do
+      assert_difference('ContentBlob.count') do
+        post :create, params: params
+      end
+    end
+
+    assert_redirected_to data_file_path(assigns(:data_file))
+    assert_equal 'http://mocked404.com', assigns(:data_file).content_blob.url
+
+
+    params = { data_file: {
+      title: 'Remote File',
+      project_ids: [projects(:sysmo_project).id]
+    },
+               content_blobs: [{
+                                 data_url: 'http://mocked400.com',
+                                 make_local_copy: '1',
+                                 override_url_check: 'yes'
+                               }],
+               policy_attributes: valid_sharing }
+
+    assert_difference('DataFile.count') do
+      assert_difference('ContentBlob.count') do
+        post :create, params: params
+      end
+    end
+
+    assert_redirected_to data_file_path(assigns(:data_file))
+    assert_equal 'http://mocked400.com', assigns(:data_file).content_blob.url
+
+
+  end
+
   test 'should display null license text' do
     df = FactoryBot.create :data_file, policy: FactoryBot.create(:public_policy)
 
@@ -3648,6 +3712,7 @@ def mock_http
     stub_request(:any, 'http://mocked302.com').to_return(status: 302, headers: { location: 'http://redirectlocation.com' })
     stub_request(:any, 'http://mocked401.com/file.txt').to_return(status: 401)
     stub_request(:any, 'http://mocked403.com/file.txt').to_return(status: 403)
+    stub_request(:any, 'http://mocked400.com').to_return(status: 400)
     stub_request(:any, 'http://mocked404.com').to_return(status: 404)
 
     stub_request(:get, 'http://mockedlocation.com/small.txt').to_return(body: 'bananafish' * 10, status: 200, headers: { content_type: 'text/plain; charset=UTF-8', content_length: 100 })