Merge pull request #991 from ElixirTeSS/check-url-speedup

Misc fixes

app/controllers/application_controller.rb

@@ -47,9 +47,10 @@ def handle_error(status_code = 500, message = nil)
     status_code = status_code.to_i
     @message = message
     respond_to do |format|
-      format.html  { render 'static/error', status: status_code}
+      format.html  { render 'static/error', status: status_code }
       format.json { render json: { error: { message: message, code: status_code } }, status: status_code }
       format.json_api { render json: { error: { message: message, code: status_code } }, status: status_code }
+      format.any { head status_code }
     end
   end
 

app/models/collection.rb

@@ -83,10 +83,6 @@ def self.visible_by(user)
   end
 
   # implement methods to allow processing as resource
-  def last_scraped
-    nil
-  end
-
   def content_provider
     nil
   end

app/models/learning_path_topic.rb

@@ -55,10 +55,6 @@ def self.facet_fields
   end
 
   # implement methods to allow processing as resource
-  def last_scraped
-    nil
-  end
-
   def content_provider
     nil
   end

config/locales/en.yml

@@ -611,6 +611,7 @@ en:
     503: "Apologies, TeSS is temporarily down or unavailable due to maintenance (503 Service Unavailable)."
     422: "The request you sent was well-formed but the change you wanted was rejected (422 Unprocessable Entity)."
     404: "The requested page could not be found - you may have mistyped the address or the page may have moved (404 Not Found)."
+    406: "The requested format is not available (406 Not Acceptable)."
   warnings:
     link_broken: TeSS has been unable to access this %{resource_type}'s URL since %{fail_date} - the page may have been moved.
     archived: This %{resource_type} has been archived, its content may no longer be relevant.

config/routes.rb

@@ -141,7 +141,7 @@
   get 'job_status' => 'application#job_status'
 
   # error pages
-  %w( 404 422 500 503 ).each do |code|
+  %w( 404 406 422 500 503 ).each do |code|
     get code, to: 'application#handle_error', status_code: code
   end
 

lib/tasks/check_urls.rake

@@ -77,11 +77,14 @@ end
 # to store as "code" which might be tracked back to a particular problem.
 def get_bad_response(url)
   begin
-    sleep(rand(10))
+    host = URI.parse(url).host rescue nil
+    if @prev_host == host
+      n = rand(4) + 1
+      sleep(n)
+    end
+    @prev_host = host
     response = HTTParty.head(url, verify: false)
-    #puts "#{response.code}, #{url}"
-    return nil if response.code.to_s =~ /2[0-9]{2}/  # Success!
-    return nil if response.code.to_s =~ /3[0-9]{2}/  # Redirection
+    return nil if response.code >= 200 && response.code < 400 # Success or redirects are OK
     return response.code
   rescue EOFError => e
     puts "  #{e.class.name}: #{e}"

test/integration/error_handler_test.rb

@@ -3,6 +3,7 @@
 class ErrorHandlerTest < ActionDispatch::IntegrationTest
   ERRORS = {
     404 => /could not be found/,
+    406 => /format is not available/,
     422 => /change you wanted was rejected/,
     500 => /TeSS encountered an error/,
     503 => /TeSS is temporarily down/
@@ -29,5 +30,12 @@ class ErrorHandlerTest < ActionDispatch::IntegrationTest
       assert_response code
       assert_match message_matcher, JSON.parse(response.body).dig('error', 'message')
     end
+
+    test "should get blank response for #{code} error as unknown format" do
+      get "/#{code}", headers: { 'Accept': 'application/banana.protocol' }
+
+      assert_response code
+      assert_empty response.body
+    end
   end
 end