Merge pull request #3679 from benwbrum/3644-search-attempt-tracking

WIP: Added search attempt tracking for #3644

app/assets/stylesheets/application.scss

@@ -111,6 +111,7 @@ $gapSize:                   20px;
 @import "sections/user";
 @import "sections/ia-import";
 @import "sections/iiif-import";
+@import "sections/search_attempt";
 
 @import "plugins/select2/core";
 

app/assets/stylesheets/sections/search_attempt.scss

@@ -0,0 +1,8 @@
+// ---------------------------
+// SEARCH RESULTS
+// ---------------------------
+
+.search-heading {
+    margin: 0;
+    padding: 0 0 1em 0;
+}

app/controllers/admin_controller.rb

@@ -278,6 +278,45 @@ def moderation
     @collections = Collection.where(messageboards_enabled:true)
   end
 
+  def searches
+    if params[:filter] == 'nonowner' # Only transcriber searches
+      searches = SearchAttempt.where(owner: false)
+    elsif params[:filter] == 'findaproject'
+      searches = SearchAttempt.where(search_type: 'findaproject')
+    elsif params[:filter] == 'collectionwork'
+      searches = SearchAttempt.where.not(search_type: 'findaproject')
+    elsif params[:filter] == 'collection'
+      searches = SearchAttempt.where(search_type: 'collection')
+    elsif params[:filter] == 'collection-title'
+      searches = SearchAttempt.where(search_type: 'collection-title')
+    elsif params[:filter] == 'work'
+      searches = SearchAttempt.where(search_type: 'work')
+    else
+      searches = SearchAttempt.all
+    end
+    @searches = searches.order('id DESC').paginate :page => params[:page], :per_page => PAGES_PER_SCREEN
+
+    this_week = SearchAttempt.where('created_at > ?', 1.week.ago)
+    unless this_week.empty?
+      by_visit = this_week.joins(:visit).group('visits.id')
+      @find_a_project_searches_per_day = (this_week.where(search_type: 'findaproject').count / 7.0).round(2)
+      @collection_work_searches_per_day = (this_week.where.not(search_type: 'findaproject').count / 7.0).round(2)
+      @find_a_project_average_hits = this_week.where(search_type: 'findaproject').average(:hits).round(2)
+      @collection_work_average_hits = this_week.where.not(search_type: 'findaproject').average(:hits).round(2)
+      @clickthrough_rate = ((this_week.where('clicks > 0').count.to_f / this_week.count) * 100).round(1)
+      @clickthrough_rate_visit = ((by_visit.sum(:clicks).values.count{|c|c>0}.to_f / by_visit.length) * 100).round(1)
+      @contribution_rate = ((this_week.where('contributions > 0').count.to_f / this_week.count) * 100).round(1) 
+      @contribution_rate_visit = ((by_visit.sum(:contributions).values.count{|c|c>0}.to_f / by_visit.length) * 100).round(1)
+    end
+
+    start_d = params[:start_date]
+    end_d = params[:end_date]
+    max_date = 1.day.ago.end_of_day
+    @start_date = start_d&.to_datetime&.beginning_of_day || 1.week.ago.beginning_of_day
+    @end_date = end_d&.to_datetime&.end_of_day || max_date
+    @end_date = max_date if max_date < @end_date
+  end
+
   private
 
   def user_params

app/controllers/application_controller.rb

@@ -399,6 +399,22 @@ def set_api_user
     end
   end
 
+  def update_search_attempt_contributions
+    if session[:search_attempt_id]
+      search_attempt = SearchAttempt.find(session[:search_attempt_id])
+      search_attempt.increment!(:contributions)
+    end
+  end
+
+  def update_search_attempt_user(user, session_var)
+    if session_var[:search_attempt_id]
+      search_attempt = SearchAttempt.find(session_var[:search_attempt_id])
+      search_attempt.user = user
+      search_attempt.owner = user.owner
+      search_attempt.save
+    end
+  end
+
 private
   def store_current_location
     store_location_for(:user, request.url)

app/controllers/article_controller.rb

@@ -276,6 +276,7 @@ def record_deed
     deed.collection = @article.collection
     deed.user = current_user
     deed.save!
+    update_search_attempt_contributions
   end
 
   def combine_articles(from_article, to_article)

app/controllers/bulk_export_controller.rb

@@ -124,6 +124,7 @@ def bulk_export_params
         :work_id,
         :include_metadata, 
         :include_contributors,
+        :admin_searches,
         :collection_notes,
         :report_arguments => [
           :start_date, 

app/controllers/collection_controller.rb

@@ -186,8 +186,13 @@ def show
           end
         end
 
-        if params[:work_search]
-          @works = @collection.search_works(params[:work_search]).includes(:work_statistic).paginate(page: params[:page], per_page: 10)
+        # Coming from work title/metadata search
+        if params[:search_attempt_id]
+          @search_attempt = SearchAttempt.find_by(id: params[:search_attempt_id])
+          if session[:search_attempt_id] != @search_attempt.id
+            session[:search_attempt_id] = @search_attempt.id
+          end
+          @works = @search_attempt.results.paginate(page: params[:page], per_page: 10)
         elsif (params[:works] == 'untranscribed')
           ids = @collection.works.includes(:work_statistic).where.not(work_statistics: {complete: 100}).pluck(:id)
           @works = @collection.works.order_by_incomplete.where(id: ids).paginate(page: params[:page], per_page: 10)
@@ -247,8 +252,12 @@ def show
             end
           end
 
-          if params[:work_search]
-            @works = @collection.search_works(params[:work_search]).includes(:work_statistic).paginate(page: params[:page], per_page: 10)
+          if params[:search_attempt_id]
+            @search_attempt = SearchAttempt.find_by(id: params[:search_attempt_id])
+            if session[:search_attempt_id] != @search_attempt.id
+              session[:search_attempt_id] = @search_attempt.id
+            end
+            @works = @search_attempt.results.paginate(page: params[:page], per_page: 10)
           elsif (params[:works] == 'untranscribed')
             ids = @collection.works.includes(:work_statistic).where.not(work_statistics: {complete: 100}).pluck(:id)
             @works = @collection.works.order_by_incomplete.where(id: ids).paginate(page: params[:page], per_page: 10)
@@ -690,6 +699,6 @@ def set_collection_for_work(collection, work)
   end
 
   def collection_params
-    params.require(:collection).permit(:title, :slug, :intro_block, :transcription_conventions, :help, :link_help, :subjects_disabled, :subjects_enabled, :review_type, :hide_completed, :text_language, :default_orientation, :voice_recognition, :picture, :user_download, :enable_spellcheck)
+    params.require(:collection).permit(:title, :slug, :intro_block, :transcription_conventions, :help, :link_help, :subjects_disabled, :subjects_enabled, :review_type, :hide_completed, :text_language, :default_orientation, :voice_recognition, :picture, :user_download, :enable_spellcheck, :search_attempt_id)
   end
 end

app/controllers/concerns/admin_exporter.rb

@@ -0,0 +1,38 @@
+module AdminExporter
+
+  def admin_searches_csv(start_date, end_date)
+    rows = []
+    header = ['Query', 'Date', 'Search Type', 'Collection Title', 'Collection ID', 'Work Title', 'Work ID', 'Hits', 'Clicks', 'Contributions', 'Visit ID', 'User ID', 'Owner']
+
+    start_date = start_date&.beginning_of_day || 1.week.ago.beginning_of_day
+    end_date = end_date&.end_of_day || 1.day.ago.end_of_day
+    searches = SearchAttempt.where('created_at BETWEEN ? AND ?', start_date, end_date).order(:created_at)
+    searches.each do |search|
+      row = []
+      row << search.query
+      row << search.created_at
+      row << search.search_type
+      row << search.collection&.title || ""
+      row << search.collection_id || ""
+      row << search.work&.title || ""
+      row << search.work&.id || ""
+      row << search.hits
+      row << search.clicks
+      row << search.contributions
+      row << search.visit_id
+      row << search.user_id
+      row << search.owner
+      rows << row
+    end
+
+    csv_string = CSV.generate(headers: true) do |csv|
+      csv << header
+      rows.each do |row|
+        csv << row
+      end
+    end
+
+    csv_string
+  end
+
+end

app/controllers/concerns/export_service.rb

@@ -2,6 +2,7 @@ module ExportService
   include AbstractXmlHelper
   include StaticSiteExporter
   include OwnerExporter
+  include AdminExporter
   include ContributorHelper
   require 'subject_exporter'
   require 'subject_details_exporter'
@@ -105,6 +106,12 @@ def export_owner_detailed_activity_csv(out:, owner:, report_arguments:)
     out.write(detailed_activity_csv(owner, report_arguments["start_date"].to_datetime, report_arguments["end_date"].to_datetime))
   end
 
+  def export_admin_searches_csv(out:, report_arguments:)
+    path = "admin_searches.csv"
+    out.put_next_entry(path)
+    out.write(admin_searches_csv(report_arguments["start_date"].to_datetime, report_arguments["end_date"].to_datetime))
+  end
+
   def export_collection_activity_csv(out:, collection:, report_arguments:)
     path = "collection_detailed_activity.csv"
     out.put_next_entry(path)

app/controllers/concerns/owner_exporter.rb

@@ -86,7 +86,4 @@ def owner_mailing_list_csv(owner)
     csv_string
   end
 
-
-
-
 end

app/controllers/dashboard_controller.rb

@@ -137,24 +137,13 @@ def guest
   end
 
   def landing_page
-    if params[:search]
-      # Get matching Collections and Docsets
-      @search_results = Collection.search(params[:search]).unrestricted + DocumentSet.search(params[:search]).unrestricted
+    # Get random Collections and DocSets from paying users
+    @owners = User.findaproject_owners.order(:display_name).joins(:collections).left_outer_joins(:document_sets).includes(:collections)
 
-      # Get user_ids from the resulting search
-      search_user_ids = User.search(params[:search]).pluck(:id) + @search_results.map(&:owner_user_id)
-
-      # Get matching users and users from Collections and DocSets search
-      @owners = User.where(id: search_user_ids).where.not(account_type: nil)
-    else
-      # Get random Collections and DocSets from paying users
-      @owners = User.findaproject_owners.order(:display_name).joins(:collections).left_outer_joins(:document_sets).includes(:collections)
-
-      # Sampled Randomly down to 8 items for Carousel
-      docsets = DocumentSet.carousel.includes(:owner).where(owner_user_id: @owners.ids.uniq).sample(5)
-      colls = Collection.carousel.includes(:owner).where(owner_user_id: @owners.ids.uniq).sample(5)
-      @collections = (docsets + colls).sample(8)
-    end
+    # Sampled Randomly down to 8 items for Carousel
+    docsets = DocumentSet.carousel.includes(:owner).where(owner_user_id: @owners.ids.uniq).sample(5)
+    colls = Collection.carousel.includes(:owner).where(owner_user_id: @owners.ids.uniq).sample(5)
+    @collections = (docsets + colls).sample(8)
   end
 
   private

app/controllers/display_controller.rb

@@ -71,10 +71,6 @@ def read_all_works
     session[:col_id] = @collection.slug
   end
 
-  def search
-    redirect_to paged_search_path(request.params)
-  end
-
   def paged_search
     if @article
       render plain: "This functionality has been disabled.  Please contact support@frothepage.com if you need it."
@@ -112,52 +108,16 @@ def paged_search
       end
       @pages = Page.order('work_id, position').joins(:work).where(work_id: @collection.works.ids).where(conditions).paginate(page: params[:page])
     else
-      # restrict to pages that include that subject
-      if params[:work_id]
-        @search_work_string = sanitize_and_format_search_string(params[:search_work_string])
-        pages = search_pages_by_work(params[:work_id], @search_work_string)
-      else
-        @search_string = sanitize_and_format_search_string(params[:search_string])
-        pages = search_pages_in_collection(@search_string)
+      @search_attempt = SearchAttempt.find_by(slug: params[:id])
+      if session[:search_attempt_id] != @search_attempt.id
+        session[:search_attempt_id] = @search_attempt.id
       end
+      # restrict to pages that include that subject
+      @collection = @search_attempt.collection
+      @work = @search_attempt&.work
+      pages = @search_attempt.results
       @pages = pages.paginate(page: params[:page])
     end
     logger.debug "DEBUG #{@search_string}"
   end
-
-  private
-
-  def sanitize_and_format_search_string(search_string)
-    return CGI::escapeHTML(search_string) if search_string.present?
-
-    ''
-  end
-
-  def search_pages_by_work(work_id, search_work_string)
-    return Page.none unless work_id.present? && search_work_string.present?
-
-    formatted_search_string = format_search_string(search_work_string)
-    Page.order('work_id, position')
-        .joins(:work)
-        .where(work_id: work_id)
-        .where("MATCH(search_text) AGAINST(? IN BOOLEAN MODE)", formatted_search_string)
-  end
-
-  def search_pages_in_collection(search_string)
-    return Page.none unless search_string.present?
-
-    formatted_search_string = format_search_string(search_string)
-    Page.order('work_id, position')
-        .joins(:work)
-        .where(work_id: @collection.works.ids)
-        .where("MATCH(search_text) AGAINST(? IN BOOLEAN MODE)", formatted_search_string)
-  end
-
-  def format_search_string(search_string)
-    # convert 'natural' search strings unless they're precise
-    return search_string if search_string.match(/["+-]/)
-
-    search_string.gsub!(/\s+/, ' ')
-    "+\"#{search_string}\""
-  end
 end

app/controllers/notes_controller.rb

@@ -70,6 +70,7 @@ def record_deed
     deed.user = current_user
 
     deed.save!
+    update_search_attempt_contributions
   end
 
   def discussions