Merge branch 'develop' into secondary-menu-redesign

Gemfile

@@ -99,6 +99,10 @@ gem 'jc-validates_timeliness', '~> 3.1.1'
 
 gem 'enumerize', '~> 2.0'
 gem 'uuidtools', '~> 2.1.5'
+
+# Note: if other modifications are made to the default_scope
+# there are manually constructed queries that need to be updated to match
+# (search for ':deleted_at' to find the relevant places)
 gem 'acts_as_paranoid', git: 'https://github.com/ActsAsParanoid/acts_as_paranoid.git', branch: :master, ref: 'c2db19554ddaedcac0a2b8d6a0563dea83c972c5'
 
 

app/controllers/application_controller.rb

@@ -67,6 +67,9 @@ class ApplicationController < ActionController::Base
                     (session[:last_seen_at].blank? || Time.zone.at(session[:last_seen_at].to_i) < 10.minutes.ago)
                 }
 
+  # We've had headers misbehave. Validating them here means we can email someone about the problem!
+  after_action :validate_headers
+
   # A dummy method to get rid of all the Rubymine errors.
   # @return [User]
   def current_user
@@ -93,7 +96,8 @@ def possessive_determiner
   # @return [void]
   def add_archived_at_header(model)
     if model.respond_to?(:deleted_at) && !model.deleted_at.blank?
-      response.headers['X-Archived-At'] = model.deleted_at.httpdate # must be a string, can't just pass a Date or Time
+      # response header must be a string, can't just pass a Date or Time
+      response.headers['X-Archived-At'] = model.deleted_at.httpdate
     end
   end
 
@@ -571,5 +575,13 @@ def should_check_authorization?
     !is_devise_controller && !is_admin_controller
   end
 
+  def validate_headers
+    if response.headers.has_key?('Content-Length') && response.headers['Content-Length'].to_i < 0
+      raise CustomErrors::BadHeaderError
+    end
+    if response.headers.has_key?(:content_length) && response.headers[:content_length].to_i < 0
+      raise CustomErrors::BadHeaderError
+    end
+  end
 
 end

app/models/audio_event.rb

@@ -114,8 +114,24 @@ def self.filter_settings
 
   # Project audio events to the format for CSV download
   # @return  [Arel::Nodes::Node] audio event csv query
+  # @param [User] user
+  # @param [Project] project
+  # @param [Site] site
+  # @param [AudioRecording] audio_recording
+  # @param [Float] start_offset
+  # @param [Float] end_offset
+  # @param [String] timezone_name
+  # @return [Arel:SelectManager]
   def self.csv_query(user, project, site, audio_recording, start_offset, end_offset, timezone_name)
 
+    # Note: if other modifications are made to the default_scope (like acts_as_paranoid does),
+    # manually constructed queries like this need to be updated to match
+    # (search for ':deleted_at' to find the relevant places)
+
+    # Note: tried using Arel from ActiveRecord
+    # e.g. AudioEvent.all.ast.cores[0].wheres
+    # but was more trouble to use than directly constructing Arel
+
     audio_events = AudioEvent.arel_table
     users = User.arel_table
     audio_recordings = AudioRecording.arel_table
@@ -158,6 +174,7 @@ def self.csv_query(user, project, site, audio_recording, start_offset, end_offse
     projects_aggregate =
         projects_sites
             .join(projects).on(projects[:id].eq(projects_sites[:project_id]))
+            .where(projects[:deleted_at].eq(nil))
             .where(projects_sites[:site_id].eq(sites[:id]))
             .project(projects_agg)
 
@@ -190,9 +207,12 @@ def self.csv_query(user, project, site, audio_recording, start_offset, end_offse
 
     query =
         audio_events
+            .where(audio_events[:deleted_at].eq(nil))
             .join(users).on(users[:id].eq(audio_events[:creator_id]))
             .join(audio_recordings).on(audio_recordings[:id].eq(audio_events[:audio_recording_id]))
+            .where(audio_recordings[:deleted_at].eq(nil))
             .join(sites).on(sites[:id].eq(audio_recordings[:site_id]))
+            .where(sites[:deleted_at].eq(nil))
             .order(audio_events[:id].desc)
             .project(
                 audio_events[:id].as('audio_event_id'),
@@ -246,15 +266,25 @@ def self.csv_query(user, project, site, audio_recording, start_offset, end_offse
                     .as('library_url'),
             )
 
+    # ensure deleted projects are not included
+    site_ids_for_live_project_ids = projects
+                      .where(projects[:deleted_at].eq(nil))
+                      .join(projects_sites).on(projects[:id].eq(projects_sites[:project_id]))
+                      .where(sites[:id].eq(projects_sites[:site_id]))
+                      .project(sites[:id]).distinct
+
+    query = query.where(sites[:id].in(site_ids_for_live_project_ids))
+
+
     if user
       query = query.where(users[:id].eq(user.id))
     end
 
     if project
-
       site_ids = sites
                      .join(projects_sites).on(sites[:id].eq(projects_sites[:site_id]))
                      .join(projects).on(projects[:id].eq(projects_sites[:project_id]))
+                     .where(projects[:deleted_at].eq(nil))
                      .where(projects[:id].eq(project.id))
                      .project(sites[:id]).distinct
 

app/models/range_request.rb

@@ -29,6 +29,7 @@ module Constants
     MULTIPART_HEADER_LENGTH = 49
     MULTIPART_DASH_LINE_BREAK_LENGTH = 8
     CONVERT_INDEX_TO_LENGTH = 1
+    CONVERT_LENGTH_TO_INDEX = -1
     REQUIRED_PARAMETERS = [:start_offset, :end_offset, :recorded_date, :site_id, :site_name, :ext, :file_path, :media_type]
 
 
@@ -303,7 +304,7 @@ def response_info(options, rails_request)
         range_end_bytes: [],
 
         # the largest possible value for range_end_bytes
-        range_end_bytes_max: file_size - CONVERT_INDEX_TO_LENGTH,
+        range_end_bytes_max: file_size + CONVERT_LENGTH_TO_INDEX,
 
         range_length_max: @max_range_size,
         write_buffer_size: @write_buffer_size,
@@ -359,6 +360,15 @@ def response_range(info)
     #
     # Can also have multiple ranges delimited by commas, as in:
     #      Range: bytes=0-500,600-1000 * Get bytes 0-500 (the first 501 bytes), inclusive plus bytes 600-1000 (401 bytes) inclusive
+    #
+    # https://tools.ietf.org/html/rfc7233#page-4
+    # Byte offsets (start and end) should be inclusive
+    # Byte offsets start at 0
+    # e.g.
+    # The first 500 bytes (byte offsets 0-499, inclusive):
+    #    bytes=0-499
+    # The second 500 bytes (byte offsets 500-999, inclusive):
+    #   bytes=500-999
 
     return_value[:range_start_bytes] = []
     return_value[:range_end_bytes] = []
@@ -368,31 +378,46 @@ def response_range(info)
       start_range = current_range[start_index]
       end_range = current_range[end_index]
 
+      # e.g. "-", ""
+      # NB: these technically aren't legal forms (as far as I can tell)
       if start_range.blank? && end_range.blank?
         # default to 0 - @max_range_size (or whatever is available) of file
         start_range = info[:range_start_bytes_min]
-        end_range = [@max_range_size, info[:range_end_bytes_max]].min
+        end_range = [@max_range_size + CONVERT_LENGTH_TO_INDEX, info[:range_end_bytes_max]].min
 
+      # e.g. "0-1", "0-500", "400-1000"
       elsif !start_range.blank? && !end_range.blank?
         # both supplied
         start_range = start_range.to_i
         end_range = end_range.to_i
 
+      # e.g. "500-", "0-"
       elsif !start_range.blank? && end_range.blank?
         # given a start but no end, get the smallest of remaining length and @max_range_size
         start_range = start_range.to_i
-        end_range = info[:range_end_bytes_max] - start_range.to_i
+        end_range = [start_range + @max_range_size + CONVERT_LENGTH_TO_INDEX, info[:range_end_bytes_max]].min
 
+      # https://tools.ietf.org/html/rfc7233#page-5
+      # assuming a representation of length 10000:
+      # The final 500 bytes (byte offsets 9500-9999, inclusive):
+      #   bytes=-500
+      # e.g. "-200"
       elsif start_range.blank? && !end_range.blank?
         # No beginning specified, get last n bytes of file
-        start_range = info[:range_end_bytes_max] - [end_range.to_i, @max_range_size].min
+        start_range = info[:range_end_bytes_max] + CONVERT_INDEX_TO_LENGTH - [end_range.to_i, @max_range_size].min
         end_range = info[:range_end_bytes_max]
 
       end
 
       start_range = info[:range_start_bytes_min] if start_range < info[:range_start_bytes_min]
       end_range = info[:range_end_bytes_max] if end_range > info[:range_end_bytes_max]
-      end_range = start_range + @max_range_size if (end_range - start_range) > @max_range_size
+      # e.g. bytes=0-499, max_range_size=500 => 499 - 0 + 1 = 500 > 500
+      if (end_range - start_range + CONVERT_INDEX_TO_LENGTH) > @max_range_size
+        fail CustomErrors::BadRequestError, 'The requested range exceeded the maximum allowed.'
+      end
+      if start_range > end_range
+        fail CustomErrors::BadRequestError, 'The requested range specified a first byte that was greater than the last byte.'
+      end
 
       return_value[:range_start_bytes].push(start_range)
       return_value[:range_end_bytes].push(end_range)

app/views/sites/_harvest.yml.haml

@@ -24,7 +24,7 @@ utc_offset: 'INTENTIONALLY_INVALID'
 \# structured metadata to add to each recording.
 \# use this to record information about sensors, etc...
 metadata:
-\#   sensor_type: SM2
-\#   notes:
-\#     - stripped left channel due to bad mic
-\#     - appears to have electronic interference from solar panel
+\#  sensor_type: SM2
+\#  notes:
+\#    - stripped left channel due to bad mic
+\#    - appears to have electronic interference from solar panel

lib/modules/access/by_permission.rb

@@ -146,15 +146,15 @@ def analysis_jobs_items(analysis_job, user, system_mode = false, levels = Access
 
         if system_mode
           query = AnalysisJobsItem.system_query
-              .joins(audio_recording: :site)
-              .order(audio_recording_id: :asc)
+                      .joins(audio_recording: :site)
+                      .order(audio_recording_id: :asc)
         else
           analysis_job = Access::Core.validate_analysis_job(analysis_job)
           query = AnalysisJobsItem
-              .joins(audio_recording: :site)
-              .order(audio_recording_id: :asc)
-              .joins(:analysis_job) # this join ensures only non-deleted results are returned
-              .where(analysis_jobs: {id: analysis_job.id})
+                      .joins(audio_recording: :site)
+                      .order(audio_recording_id: :asc)
+                      .joins(:analysis_job) # this join ensures only non-deleted results are returned
+                      .where(analysis_jobs: {id: analysis_job.id})
         end
 
         permission_sites(user, levels, query)
@@ -237,7 +237,6 @@ def permission_projects(user, levels)
       def permission_sites(user, levels, query, project_ids = nil)
 
         is_admin, query = permission_admin(user, levels, query)
-        return query if is_admin
 
 =begin
   EXISTS
@@ -252,14 +251,11 @@ def permission_sites(user, levels, query, project_ids = nil)
         pt = Project.arel_table
         ps = Arel::Table.new(:projects_sites)
         st = Site.arel_table
+
         # levels_modified is not used because permission_projects also
         # calls calculate_levels, which would end up with the inverse results
         levels_modified, exists = calculate_levels(levels)
 
-        # project permission will never be nil, which is the way it should work
-        # when being used as a part of a subquery rather than the whole subquery
-        permissions = permission_projects(user, levels)
-
         permissions_by_site =
             ps
                 .join(pt).on(ps[:project_id].eq(pt[:id]))
@@ -272,11 +268,15 @@ def permission_sites(user, levels, query, project_ids = nil)
                   .where(pt[:id].in(project_ids))
         end
 
-        permissions_by_site =
-            permissions_by_site
-                .where(permissions)
-                .project(1)
-                .exists
+        unless is_admin
+          # project permission will never be nil, which is the way it should work
+          # when being used as a part of a subquery rather than the whole subquery
+          permissions = permission_projects(user, levels)
+          permissions_by_site = permissions_by_site.where(permissions)
+        end
+
+        permissions_by_site = permissions_by_site.project(1).exists
+
 =begin
   EXISTS
     (SELECT 1

lib/modules/custom_errors.rb

@@ -5,6 +5,7 @@ class ItemNotFoundError < StandardError; end
   class TooManyItemsFoundError < StandardError; end
   class AnalysisJobStartError < StandardError; end
   class OrphanedSiteError < StandardError; end
+  class BadHeaderError < StandardError; end
   class RequestedMediaTypeError < StandardError
     attr_reader :available_formats_info
     def initialize(message = nil, available_formats_info = nil)

lib/modules/filter/comparison.rb

@@ -26,6 +26,7 @@ def compose_eq(table, column_name, allowed, value)
     # @return [Arel::Nodes::Node] condition
     def compose_eq_node(node, value)
       validate_node_or_attribute(node)
+      validate_basic_class(node, value)
       node.eq(value)
     end
 
@@ -46,6 +47,7 @@ def compose_not_eq(table, column_name, allowed, value)
     # @return [Arel::Nodes::Node] condition
     def compose_not_eq_node(node, value)
       validate_node_or_attribute(node)
+      validate_basic_class(node, value)
       node.not_eq(value)
     end
 
@@ -66,6 +68,7 @@ def compose_lt(table, column_name, allowed, value)
     # @return [Arel::Nodes::Node] condition
     def compose_lt_node(node, value)
       validate_node_or_attribute(node)
+      validate_basic_class(node, value)
       node.lt(value)
     end
 
@@ -104,6 +107,7 @@ def compose_gt(table, column_name, allowed, value)
     # @return [Arel::Nodes::Node] condition
     def compose_gt_node(node, value)
       validate_node_or_attribute(node)
+      validate_basic_class(node, value)
       node.gt(value)
     end
 
@@ -142,6 +146,7 @@ def compose_lteq(table, column_name, allowed, value)
     # @return [Arel::Nodes::Node] condition
     def compose_lteq_node(node, value)
       validate_node_or_attribute(node)
+      validate_basic_class(node, value)
       node.lteq(value)
     end
 
@@ -180,6 +185,7 @@ def compose_gteq(table, column_name, allowed, value)
     # @return [Arel::Nodes::Node] condition
     def compose_gteq_node(node, value)
       validate_node_or_attribute(node)
+      validate_basic_class(node, value)
       node.gteq(value)
     end
 

lib/modules/filter/subset.rb

@@ -16,7 +16,7 @@ module Subset
     # @return [Arel::Nodes::Node] condition
     def compose_contains(table, column_name, allowed, value)
       validate_table_column(table, column_name, allowed)
-      compose_contains_node(table[column_name],value)
+      compose_contains_node(table[column_name], value)
     end
 
     # Create contains condition.
@@ -25,6 +25,7 @@ def compose_contains(table, column_name, allowed, value)
     # @return [Arel::Nodes::Node] condition
     def compose_contains_node(node, value)
       validate_node_or_attribute(node)
+      validate_basic_class(node, value)
       sanitized_value = sanitize_like_value(value)
       contains_value = "%#{sanitized_value}%"
       node.matches(contains_value)
@@ -65,6 +66,7 @@ def compose_starts_with(table, column_name, allowed, value)
     # @return [Arel::Nodes::Node] condition
     def compose_starts_with_node(node, value)
       validate_node_or_attribute(node)
+      validate_basic_class(node, value)
       sanitized_value = sanitize_like_value(value)
       contains_value = "#{sanitized_value}%"
       node.matches(contains_value)
@@ -105,6 +107,7 @@ def compose_ends_with(table, column_name, allowed, value)
     # @return [Arel::Nodes::Node] condition
     def compose_ends_with_node(node, value)
       validate_node_or_attribute(node)
+      validate_basic_class(node, value)
       sanitized_value = sanitize_like_value(value)
       contains_value = "%#{sanitized_value}"
       node.matches(contains_value)
@@ -307,6 +310,8 @@ def compose_range(table, column_name, allowed, from, to)
     # @return [Arel::Nodes::Node] condition
     def compose_range_node(node, from, to)
       validate_node_or_attribute(node)
+      validate_basic_class(node, from)
+      validate_basic_class(node, to)
       range = Range.new(from, to, true)
       node.in(range)
     end
@@ -348,6 +353,7 @@ def compose_regex(table, column_name, allowed, value)
     # @return [Arel::Nodes::Node] condition
     def compose_regex_node(node, value)
       validate_node_or_attribute(node)
+      validate_string(value)
       Arel::Nodes::Regexp.new(node, Arel::Nodes.build_quoted(value))
     end
 
@@ -370,6 +376,7 @@ def compose_not_regex(table, column_name, allowed, value)
     # @return [Arel::Nodes::Node] condition
     def compose_not_regex_node(node, value)
       validate_node_or_attribute(node)
+      validate_string(value)
       Arel::Nodes::NotRegexp.new(node, Arel::Nodes.build_quoted(value))
     end