diff --git a/Gemfile b/Gemfile index e8703efcf8..9cddd3a505 100644 --- a/Gemfile +++ b/Gemfile @@ -99,6 +99,8 @@ gem 'icalendar', require: false gem "jwt" # Use Newrelic for logs and APM gem "newrelic_rpm" +# Scheduling +gem 'rufus-scheduler' # Used to manage periodic cron-like jobs gem "clockwork" diff --git a/Gemfile.lock b/Gemfile.lock index c240e23379..296482f84f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -163,6 +163,8 @@ GEM dotenv (= 2.8.1) railties (>= 3.2) erubi (1.12.0) + et-orbi (1.2.7) + tzinfo execjs (2.8.1) factory_bot (6.2.0) activesupport (>= 5.0.0) @@ -211,6 +213,9 @@ GEM sanitize (< 7) foreman (0.87.2) formatador (0.3.0) + fugit (1.8.1) + et-orbi (~> 1, >= 1.2.7) + raabro (~> 1.4) geocoder (1.8.2) globalid (1.1.0) activesupport (>= 5.0) @@ -404,6 +409,7 @@ GEM public_suffix (5.0.1) puma (6.3.0) nio4r (~> 2.0) + raabro (1.4.0) racc (1.7.1) rack (2.2.7) rack-protection (2.1.0) @@ -514,6 +520,8 @@ GEM ffi (~> 1.12) ruby2_keywords (0.0.5) rubyzip (2.3.2) + rufus-scheduler (3.9.1) + fugit (~> 1.1, >= 1.1.6) sanitize (6.0.2) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -696,6 +704,7 @@ DEPENDENCIES rubocop rubocop-performance rubocop-rails (~> 2.20.1) + rufus-scheduler sass-rails shoulda-matchers (~> 5.3) simple_form diff --git a/app/controllers/historical_trends/base_controller.rb b/app/controllers/historical_trends/base_controller.rb index a88844a906..d460ba686c 100644 --- a/app/controllers/historical_trends/base_controller.rb +++ b/app/controllers/historical_trends/base_controller.rb @@ -1,32 +1,5 @@ class HistoricalTrends::BaseController < ApplicationController - def series(type) - items = [] - - current_organization.items.sort.each do |item| - next if item.line_items.where(itemizable_type: type, item: item).blank? - - dates = Hash.new - - (1..Time.zone.today.month).each do |month| - dates[month] = 0 - end - - total_items(item.line_items, type).each do |line_item| - month = line_item.dig(0).to_date.month - dates[month] = line_item.dig(1) - end - - items << { name: item.name, data: dates.values } - end - - items.sort_by { |hsh| hsh[:name] } - end - - private - - def total_items(line_items, type) - line_items.where(itemizable_type: type) - .group_by_month(:created_at) - .sum(:quantity) + def cached_series(type) + Rails.cache.fetch("#{current_organization.short_name}-historical-#{type}-data") { HistoricalTrendService.new(current_organization.id, type).series } end end diff --git a/app/controllers/historical_trends/distributions_controller.rb b/app/controllers/historical_trends/distributions_controller.rb index d626092328..e13a6ede62 100644 --- a/app/controllers/historical_trends/distributions_controller.rb +++ b/app/controllers/historical_trends/distributions_controller.rb @@ -1,6 +1,6 @@ class HistoricalTrends::DistributionsController < HistoricalTrends::BaseController def index - @series = series('Distribution') + @series = cached_series('Distribution') @title = 'Monthly Distributions' end end diff --git a/app/controllers/historical_trends/donations_controller.rb b/app/controllers/historical_trends/donations_controller.rb index 2d3608b95a..92a3f5422e 100644 --- a/app/controllers/historical_trends/donations_controller.rb +++ b/app/controllers/historical_trends/donations_controller.rb @@ -1,6 +1,6 @@ class HistoricalTrends::DonationsController < HistoricalTrends::BaseController def index - @series = series('Donation') + @series = cached_series('Donation') @title = 'Monthly Donations' end end diff --git a/app/controllers/historical_trends/purchases_controller.rb b/app/controllers/historical_trends/purchases_controller.rb index ee5031fe56..1f5859b2af 100644 --- a/app/controllers/historical_trends/purchases_controller.rb +++ b/app/controllers/historical_trends/purchases_controller.rb @@ -1,6 +1,6 @@ class HistoricalTrends::PurchasesController < HistoricalTrends::BaseController def index - @series = series('Purchase') + @series = cached_series('Purchase') @title = "Monthly Purchases" end end diff --git a/app/helpers/historical_trends_helper.rb b/app/helpers/historical_trends_helper.rb new file mode 100644 index 0000000000..29091194c5 --- /dev/null +++ b/app/helpers/historical_trends_helper.rb @@ -0,0 +1,21 @@ +module HistoricalTrendsHelper + MONTHS = [ + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec" + ] + + def last_12_months + current_month = Time.zone.now.month + MONTHS.rotate(current_month) + end +end diff --git a/app/jobs/historical_data_cache_job.rb b/app/jobs/historical_data_cache_job.rb new file mode 100644 index 0000000000..5c1f71f4f0 --- /dev/null +++ b/app/jobs/historical_data_cache_job.rb @@ -0,0 +1,7 @@ +class HistoricalDataCacheJob < ApplicationJob + def perform(org_id:, type:) + organization = Organization.find_by(id: org_id) + + Rails.cache.write("#{organization.short_name}-historical-#{type}-data", HistoricalTrendService.new(organization.id, type).series) + end +end diff --git a/app/models/organization.rb b/app/models/organization.rb index b26b0337cf..8e91cffa52 100644 --- a/app/models/organization.rb +++ b/app/models/organization.rb @@ -137,6 +137,12 @@ def upcoming scope :alphabetized, -> { order(:name) } scope :search_name, ->(query) { where('name ilike ?', "%#{query}%") } + scope :is_active, -> { + joins(:users) + .where('users.last_sign_in_at > ?', 4.months.ago) + .distinct + } + def assign_attributes_from_account_request(account_request) assign_attributes( name: account_request.organization_name, diff --git a/app/services/historical_trend_service.rb b/app/services/historical_trend_service.rb new file mode 100644 index 0000000000..b44f912c9d --- /dev/null +++ b/app/services/historical_trend_service.rb @@ -0,0 +1,36 @@ +class HistoricalTrendService + def initialize(organization_id, type) + @organization = Organization.find(organization_id) + @type = type + end + + def series + items = [] + + @organization.items.active.sort.each do |item| + next if item.line_items.where(itemizable_type: @type, item: item).blank? + + month_offset = [*1..12].rotate(Time.zone.today.month) + + dates = (1..12).index_with { |i| 0 } + + total_items(item.line_items, @type).each do |line_item| + month = line_item.dig(0).to_date.month + dates[(month_offset.index(month) + 1)] = line_item.dig(1) + end + + items << {name: item.name, data: dates.values, visible: false} + end + + items.sort_by { |hsh| hsh[:name] } + end + + private + + def total_items(line_items, type) + line_items.where(created_at: 1.year.ago.beginning_of_month..Time.current) + .where(itemizable_type: type) + .group_by_month(:created_at) + .sum(:quantity) + end +end diff --git a/app/views/historical_trends/distributions/index.html.erb b/app/views/historical_trends/distributions/index.html.erb index da97f42480..50cb5ce5fc 100644 --- a/app/views/historical_trends/distributions/index.html.erb +++ b/app/views/historical_trends/distributions/index.html.erb @@ -8,23 +8,10 @@ text: @title }, subtitle: { - text: "Source: humanessentials.app" + text: "Cached Data, may be up to 24 hours old" }, xAxis: { - categories: [ - 'Jan', - 'Feb', - 'Mar', - 'Apr', - 'May', - 'Jun', - 'Jul', - 'Aug', - 'Sep', - 'Oct', - 'Nov', - 'Dec' - ], + categories: last_12_months, crosshair: true }, yAxis: { diff --git a/app/views/historical_trends/donations/index.html.erb b/app/views/historical_trends/donations/index.html.erb index 285ef96852..0fda67877a 100644 --- a/app/views/historical_trends/donations/index.html.erb +++ b/app/views/historical_trends/donations/index.html.erb @@ -8,23 +8,10 @@ text: @title }, subtitle: { - text: "Source: humanessentials.app" + text: "Cached Data, may be up to 24 hours old" }, xAxis: { - categories: [ - 'Jan', - 'Feb', - 'Mar', - 'Apr', - 'May', - 'Jun', - 'Jul', - 'Aug', - 'Sep', - 'Oct', - 'Nov', - 'Dec' - ], + categories: last_12_months, crosshair: true }, yAxis: { diff --git a/app/views/historical_trends/purchases/index.html.erb b/app/views/historical_trends/purchases/index.html.erb index 1cafad158c..879334e09b 100644 --- a/app/views/historical_trends/purchases/index.html.erb +++ b/app/views/historical_trends/purchases/index.html.erb @@ -8,23 +8,10 @@ text: @title }, subtitle: { - text: "Source: humanessentials.app" + text: "Cached Data, may be up to 24 hours old" }, xAxis: { - categories: [ - 'Jan', - 'Feb', - 'Mar', - 'Apr', - 'May', - 'Jun', - 'Jul', - 'Aug', - 'Sep', - 'Oct', - 'Nov', - 'Dec' - ], + categories: last_12_months, crosshair: true }, yAxis: { diff --git a/app/views/shared/_highcharts.html.erb b/app/views/shared/_highcharts.html.erb index 92989c0f79..fa93289bb6 100644 --- a/app/views/shared/_highcharts.html.erb +++ b/app/views/shared/_highcharts.html.erb @@ -1,7 +1,7 @@
-
- - +
+ +
diff --git a/config/environments/production.rb b/config/environments/production.rb index 5700a4bd8c..5b94bd9610 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -61,7 +61,7 @@ config.log_level = :info # Use a different cache store in production. - # config.cache_store = :mem_cache_store + config.cache_store = :redis_cache_store, { url: ENV['REDIS_URL'] } # Use a real queuing backend for Active Job (and separate queues per environment) # config.active_job.queue_adapter = :resque diff --git a/config/initializers/task_scheduler.rb b/config/initializers/task_scheduler.rb new file mode 100644 index 0000000000..67642ba5a2 --- /dev/null +++ b/config/initializers/task_scheduler.rb @@ -0,0 +1,7 @@ +require 'rufus-scheduler' + +scheduler = Rufus::Scheduler.singleton + +scheduler.cron '0 3 * * *' do + system('bundle exec rake cache_historical_data') +end diff --git a/lib/tasks/cache_historical_data.rake b/lib/tasks/cache_historical_data.rake new file mode 100644 index 0000000000..88431f4856 --- /dev/null +++ b/lib/tasks/cache_historical_data.rake @@ -0,0 +1,16 @@ +desc "This task is run by a scheduling tool nightly to cache the processor intensive queries" +task :cache_historical_data => :environment do + Rails.logger.info("Caching historical data") + DATA_TYPES = ['Distribution', 'Purchase', 'Donation'] + + orgs = Organization.is_active + + orgs.each do |org| + DATA_TYPES.each do |type| + puts "Queuing up #{type} cache data for #{org.name}" + HistoricalDataCacheJob.perform_later(org_id: org.id, type: type) + end + end + + Rails.logger.info("Done!") +end diff --git a/spec/helpers/historical_trends_helper_spec.rb b/spec/helpers/historical_trends_helper_spec.rb new file mode 100644 index 0000000000..d22b7d93f3 --- /dev/null +++ b/spec/helpers/historical_trends_helper_spec.rb @@ -0,0 +1,23 @@ +require "rspec" + +RSpec.describe HistoricalTrendsHelper do + describe "#last_12_months" do + it "returns the last 12 months starting from July when the current month is June" do + allow_any_instance_of(Time).to receive(:month).and_return(6) + + expect(last_12_months).to eq(["Jul", "Aug", "Sep", "Oct", "Nov", "Dec", "Jan", "Feb", "Mar", "Apr", "May", "Jun"]) + end + + it "returns the last 12 months starting from Feb when the current month is Jan" do + allow_any_instance_of(Time).to receive(:month).and_return(1) + + expect(last_12_months).to eq(["Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", "Jan"]) + end + + it "returns the last 12 months starting from Jan when the current month is Dec" do + allow_any_instance_of(Time).to receive(:month).and_return(12) + + expect(last_12_months).to eq(["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]) + end + end +end diff --git a/spec/jobs/historical_data_cache_job_spec.rb b/spec/jobs/historical_data_cache_job_spec.rb new file mode 100644 index 0000000000..22ecbea318 --- /dev/null +++ b/spec/jobs/historical_data_cache_job_spec.rb @@ -0,0 +1,25 @@ +require "rails_helper" + +RSpec.describe HistoricalDataCacheJob, type: :job do + include ActiveJob::TestHelper + + let(:organization) { create(:organization) } + let(:type) { "Donation" } + let(:job) { described_class.perform_later(org_id: organization.id, type: type) } + + it "queues the job" do + expect { job }.to have_enqueued_job(described_class) + .with(org_id: organization.id, type: type) + .on_queue("default") + end + + it "caches the historical data" do + expected_data = {name: "Item 2", data: [0, 0, 0, 0, 0, 60, 0, 0, 30, 0, 0, 0], visible: false} + allow_any_instance_of(HistoricalTrendService).to receive(:series).and_return(expected_data) + + perform_enqueued_jobs { job } + + cached_data = Rails.cache.read("#{organization.short_name}-historical-#{type}-data") + expect(cached_data).to eq(expected_data) + end +end diff --git a/spec/models/organization_spec.rb b/spec/models/organization_spec.rb index 2614703dd1..1da46ccb4d 100644 --- a/spec/models/organization_spec.rb +++ b/spec/models/organization_spec.rb @@ -317,6 +317,17 @@ end end + describe 'is_active' do + let!(:active_organization) { create(:organization) } + let!(:inactive_organization) { create(:organization) } + let!(:active_user) { create(:user, organization: active_organization, last_sign_in_at: 1.month.ago) } + let!(:inactive_user) { create(:user, organization: inactive_organization, last_sign_in_at: 6.months.ago) } + + it 'returns active organizations' do + expect(Organization.is_active).to contain_exactly(active_organization) + end + end + describe "total_inventory" do it "returns a sum total of all inventory at all storage locations" do item = create(:item) diff --git a/spec/services/historical_trend_service_spec.rb b/spec/services/historical_trend_service_spec.rb new file mode 100644 index 0000000000..cfff9f9cef --- /dev/null +++ b/spec/services/historical_trend_service_spec.rb @@ -0,0 +1,27 @@ +require "rails_helper" + +RSpec.describe HistoricalTrendService, type: :service do + let(:organization) { create(:organization) } + let(:type) { "Donation" } + let(:service) { described_class.new(organization.id, type) } + + describe "#series" do + let!(:item1) { create(:item, organization: organization, name: "Item 1") } + let!(:item2) { create(:item, organization: organization, name: "Item 2") } + let!(:line_items) do + (0..11).map do |n| + create(:line_item, item: item1, itemizable_type: type, quantity: 10 * (n + 1), created_at: n.months.ago) + end + end + let!(:line_item2) { create(:line_item, item: item2, itemizable_type: type, quantity: 60, created_at: 6.months.ago) } + let!(:line_item3) { create(:line_item, item: item2, itemizable_type: type, quantity: 30, created_at: 3.months.ago) } + + it "returns an array of items with their monthly data" do + expected_result = [ + {name: "Item 1", data: [120, 110, 100, 90, 80, 70, 60, 50, 40, 30, 20, 10], visible: false}, + {name: "Item 2", data: [0, 0, 0, 0, 0, 60, 0, 0, 30, 0, 0, 0], visible: false} + ] + expect(service.series).to eq(expected_result) + end + end +end