Merge pull request #3665 from rubyforgood/historical_trends_caching

Historical Trend Speed up

Gemfile

@@ -99,6 +99,8 @@ gem 'icalendar', require: false
 gem "jwt"
 # Use Newrelic for logs and APM
 gem "newrelic_rpm"
+# Scheduling
+gem 'rufus-scheduler'
 # Used to manage periodic cron-like jobs
 gem "clockwork"
 

Gemfile.lock

@@ -162,6 +162,8 @@ GEM
       dotenv (= 2.8.1)
       railties (>= 3.2)
     erubi (1.12.0)
+    et-orbi (1.2.7)
+      tzinfo
     execjs (2.8.1)
     factory_bot (6.2.0)
       activesupport (>= 5.0.0)
@@ -210,6 +212,9 @@ GEM
       sanitize (< 7)
     foreman (0.87.2)
     formatador (0.3.0)
+    fugit (1.8.1)
+      et-orbi (~> 1, >= 1.2.7)
+      raabro (~> 1.4)
     geocoder (1.8.2)
     globalid (1.1.0)
       activesupport (>= 5.0)
@@ -403,6 +408,7 @@ GEM
     public_suffix (5.0.1)
     puma (6.3.0)
       nio4r (~> 2.0)
+    raabro (1.4.0)
     racc (1.7.1)
     rack (2.2.7)
     rack-protection (2.1.0)
@@ -514,6 +520,8 @@ GEM
       ffi (~> 1.12)
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
+    rufus-scheduler (3.9.1)
+      fugit (~> 1.1, >= 1.1.6)
     sanitize (6.0.2)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -692,6 +700,7 @@ DEPENDENCIES
   rubocop
   rubocop-performance
   rubocop-rails (~> 2.20.1)
+  rufus-scheduler
   sass-rails
   selenium-webdriver
   shoulda-matchers (~> 5.3)

app/controllers/historical_trends/base_controller.rb

@@ -1,32 +1,5 @@
 class HistoricalTrends::BaseController < ApplicationController
-  def series(type)
-    items = []
-
-    current_organization.items.sort.each do |item|
-      next if item.line_items.where(itemizable_type: type, item: item).blank?
-
-      dates = Hash.new
-
-      (1..Time.zone.today.month).each do |month|
-        dates[month] = 0
-      end
-
-      total_items(item.line_items, type).each do |line_item|
-        month = line_item.dig(0).to_date.month
-        dates[month] = line_item.dig(1)
-      end
-
-      items << { name: item.name, data: dates.values }
-    end
-
-    items.sort_by { |hsh| hsh[:name] }
-  end
-
-  private
-
-  def total_items(line_items, type)
-    line_items.where(itemizable_type: type)
-              .group_by_month(:created_at)
-              .sum(:quantity)
+  def cached_series(type)
+    Rails.cache.fetch("#{current_organization.short_name}-historical-#{type}-data") { HistoricalTrendService.new(current_organization.id, type).series }
   end
 end

app/controllers/historical_trends/distributions_controller.rb

@@ -1,6 +1,6 @@
 class HistoricalTrends::DistributionsController < HistoricalTrends::BaseController
   def index
-    @series = series('Distribution')
+    @series = cached_series('Distribution')
     @title = 'Monthly Distributions'
   end
 end

app/controllers/historical_trends/donations_controller.rb

@@ -1,6 +1,6 @@
 class HistoricalTrends::DonationsController < HistoricalTrends::BaseController
   def index
-    @series = series('Donation')
+    @series = cached_series('Donation')
     @title = 'Monthly Donations'
   end
 end

app/controllers/historical_trends/purchases_controller.rb

@@ -1,6 +1,6 @@
 class HistoricalTrends::PurchasesController < HistoricalTrends::BaseController
   def index
-    @series = series('Purchase')
+    @series = cached_series('Purchase')
     @title = "Monthly Purchases"
   end
 end

app/helpers/historical_trends_helper.rb

@@ -0,0 +1,21 @@
+module HistoricalTrendsHelper
+  MONTHS = [
+    "Jan",
+    "Feb",
+    "Mar",
+    "Apr",
+    "May",
+    "Jun",
+    "Jul",
+    "Aug",
+    "Sep",
+    "Oct",
+    "Nov",
+    "Dec"
+  ]
+
+  def last_12_months
+    current_month = Time.zone.now.month
+    MONTHS.rotate(current_month)
+  end
+end

app/jobs/historical_data_cache_job.rb

@@ -0,0 +1,7 @@
+class HistoricalDataCacheJob < ApplicationJob
+  def perform(org_id:, type:)
+    organization = Organization.find_by(id: org_id)
+
+    Rails.cache.write("#{organization.short_name}-historical-#{type}-data", HistoricalTrendService.new(organization.id, type).series)
+  end
+end

app/models/organization.rb

@@ -137,6 +137,12 @@ def upcoming
   scope :alphabetized, -> { order(:name) }
   scope :search_name, ->(query) { where('name ilike ?', "%#{query}%") }
 
+  scope :is_active, -> {
+    joins(:users)
+      .where('users.last_sign_in_at > ?', 4.months.ago)
+      .distinct
+  }
+
   def assign_attributes_from_account_request(account_request)
     assign_attributes(
       name: account_request.organization_name,

app/services/historical_trend_service.rb

@@ -0,0 +1,36 @@
+class HistoricalTrendService
+  def initialize(organization_id, type)
+    @organization = Organization.find(organization_id)
+    @type = type
+  end
+
+  def series
+    items = []
+
+    @organization.items.active.sort.each do |item|
+      next if item.line_items.where(itemizable_type: @type, item: item).blank?
+
+      month_offset = [*1..12].rotate(Time.zone.today.month)
+
+      dates = (1..12).index_with { |i| 0 }
+
+      total_items(item.line_items, @type).each do |line_item|
+        month = line_item.dig(0).to_date.month
+        dates[(month_offset.index(month) + 1)] = line_item.dig(1)
+      end
+
+      items << {name: item.name, data: dates.values, visible: false}
+    end
+
+    items.sort_by { |hsh| hsh[:name] }
+  end
+
+  private
+
+  def total_items(line_items, type)
+    line_items.where(created_at: 1.year.ago.beginning_of_month..Time.current)
+      .where(itemizable_type: type)
+      .group_by_month(:created_at)
+      .sum(:quantity)
+  end
+end

app/views/historical_trends/distributions/index.html.erb

@@ -8,23 +8,10 @@
         text: @title
       },
       subtitle: {
-        text: "Source: humanessentials.app"
+        text: "Cached Data, may be up to 24 hours old"
       },
       xAxis: {
-        categories: [
-          'Jan',
-          'Feb',
-          'Mar',
-          'Apr',
-          'May',
-          'Jun',
-          'Jul',
-          'Aug',
-          'Sep',
-          'Oct',
-          'Nov',
-          'Dec'
-        ],
+        categories: last_12_months,
         crosshair: true
       },
       yAxis: {

app/views/historical_trends/donations/index.html.erb

@@ -8,23 +8,10 @@
         text: @title
       },
       subtitle: {
-        text: "Source: humanessentials.app"
+        text: "Cached Data, may be up to 24 hours old"
       },
       xAxis: {
-        categories: [
-          'Jan',
-          'Feb',
-          'Mar',
-          'Apr',
-          'May',
-          'Jun',
-          'Jul',
-          'Aug',
-          'Sep',
-          'Oct',
-          'Nov',
-          'Dec'
-        ],
+        categories: last_12_months,
         crosshair: true
       },
       yAxis: {

app/views/historical_trends/purchases/index.html.erb

@@ -8,23 +8,10 @@
         text: @title
       },
       subtitle: {
-        text: "Source: humanessentials.app"
+        text: "Cached Data, may be up to 24 hours old"
       },
       xAxis: {
-        categories: [
-          'Jan',
-          'Feb',
-          'Mar',
-          'Apr',
-          'May',
-          'Jun',
-          'Jul',
-          'Aug',
-          'Sep',
-          'Oct',
-          'Nov',
-          'Dec'
-        ],
+        categories: last_12_months,
         crosshair: true
       },
       yAxis: {

app/views/shared/_highcharts.html.erb

@@ -1,7 +1,7 @@
 <div data-controller="highchart" data-highchart-config-value="<%= config %>">
   <div data-highchart-target="chart"></div>
-  <div class='flex flex-row justify-center w-full space-x-2 py-3'>
-    <button type="button" class="inline-flex items-center rounded border border-transparent bg-indigo-600 px-2.5 py-1.5 text-xs font-medium text-white shadow-sm hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-indigo-500 focus:ring-offset-2" data-action="click->highchart#selectAll">Select All</button>
-    <button type="button" class="inline-flex items-center rounded border border-transparent bg-gray-600 px-2.5 py-1.5 text-xs font-medium text-white shadow-sm hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-indigo-500 focus:ring-offset-2" data-action="click->highchart#deselectAll">Deselect All</button>
+  <div>
+    <button type="button" class="btn btn-sm btn-secondary" data-action="click->highchart#selectAll">Select All</button>
+    <button type="button" class="btn btn-sm btn-info" data-action="click->highchart#deselectAll">Deselect All</button>
   </div>
 </div>

config/environments/production.rb

@@ -61,7 +61,7 @@
   config.log_level = :info
 
   # Use a different cache store in production.
-  # config.cache_store = :mem_cache_store
+  config.cache_store = :redis_cache_store, { url: ENV['REDIS_URL'] }
 
   # Use a real queuing backend for Active Job (and separate queues per environment)
   # config.active_job.queue_adapter     = :resque

config/initializers/task_scheduler.rb

@@ -0,0 +1,7 @@
+require 'rufus-scheduler'
+
+scheduler = Rufus::Scheduler.singleton
+
+scheduler.cron '0 3 * * *' do
+  system('bundle exec rake cache_historical_data')
+end

lib/tasks/cache_historical_data.rake

@@ -0,0 +1,16 @@
+desc "This task is run by a scheduling tool nightly to cache the processor intensive queries"
+task :cache_historical_data => :environment do
+  Rails.logger.info("Caching historical data")
+  DATA_TYPES = ['Distribution', 'Purchase', 'Donation']
+
+  orgs = Organization.is_active
+
+  orgs.each do |org|
+    DATA_TYPES.each do |type|
+      puts "Queuing up #{type} cache data for #{org.name}"
+      HistoricalDataCacheJob.perform_later(org_id: org.id, type: type)
+    end
+  end
+
+  Rails.logger.info("Done!")
+end

spec/helpers/historical_trends_helper_spec.rb

@@ -0,0 +1,23 @@
+require "rspec"
+
+RSpec.describe HistoricalTrendsHelper do
+  describe "#last_12_months" do
+    it "returns the last 12 months starting from July when the current month is June" do
+      allow_any_instance_of(Time).to receive(:month).and_return(6)
+
+      expect(last_12_months).to eq(["Jul", "Aug", "Sep", "Oct", "Nov", "Dec", "Jan", "Feb", "Mar", "Apr", "May", "Jun"])
+    end
+
+    it "returns the last 12 months starting from Feb when the current month is Jan" do
+      allow_any_instance_of(Time).to receive(:month).and_return(1)
+
+      expect(last_12_months).to eq(["Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", "Jan"])
+    end
+
+    it "returns the last 12 months starting from Jan when the current month is Dec" do
+      allow_any_instance_of(Time).to receive(:month).and_return(12)
+
+      expect(last_12_months).to eq(["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"])
+    end
+  end
+end

spec/jobs/historical_data_cache_job_spec.rb

@@ -0,0 +1,25 @@
+require "rails_helper"
+
+RSpec.describe HistoricalDataCacheJob, type: :job do
+  include ActiveJob::TestHelper
+
+  let(:organization) { create(:organization) }
+  let(:type) { "Donation" }
+  let(:job) { described_class.perform_later(org_id: organization.id, type: type) }
+
+  it "queues the job" do
+    expect { job }.to have_enqueued_job(described_class)
+      .with(org_id: organization.id, type: type)
+      .on_queue("default")
+  end
+
+  it "caches the historical data" do
+    expected_data = {name: "Item 2", data: [0, 0, 0, 0, 0, 60, 0, 0, 30, 0, 0, 0], visible: false}
+    allow_any_instance_of(HistoricalTrendService).to receive(:series).and_return(expected_data)
+
+    perform_enqueued_jobs { job }
+
+    cached_data = Rails.cache.read("#{organization.short_name}-historical-#{type}-data")
+    expect(cached_data).to eq(expected_data)
+  end
+end

spec/models/organization_spec.rb

@@ -317,6 +317,17 @@
     end
   end
 
+  describe 'is_active' do
+    let!(:active_organization) { create(:organization) }
+    let!(:inactive_organization) { create(:organization) }
+    let!(:active_user) { create(:user, organization: active_organization, last_sign_in_at: 1.month.ago) }
+    let!(:inactive_user) { create(:user, organization: inactive_organization, last_sign_in_at: 6.months.ago) }
+
+    it 'returns active organizations' do
+      expect(Organization.is_active).to contain_exactly(active_organization)
+    end
+  end
+
   describe "total_inventory" do
     it "returns a sum total of all inventory at all storage locations" do
       item = create(:item)