Add cron_graceful_restart_period to avoid missing recurring jobs th…

…at occurred during deployment downtime (#1488)

README.md

@@ -257,6 +257,7 @@ Rails.application.configure do
   config.good_job.shutdown_timeout = 25 # seconds
   config.good_job.enable_cron = true
   config.good_job.cron = { example: { cron: '0 * * * *', class: 'ExampleJob'  } }
+  config.good_job.cron_graceful_restart_period = 5.minutes
   config.good_job.dashboard_default_locale = :en
 
   # ...or all at once.
@@ -298,6 +299,7 @@ Available configuration options are:
 - `max_cache` (integer) sets the maximum number of scheduled jobs that will be stored in memory to reduce execution latency when also polling for scheduled jobs. Caching 10,000 scheduled jobs uses approximately 20MB of memory. You can also set this with the environment variable `GOOD_JOB_MAX_CACHE`.
 - `shutdown_timeout` (integer) number of seconds to wait for jobs to finish when shutting down before stopping the thread. Defaults to forever: `-1`. You can also set this with the environment variable `GOOD_JOB_SHUTDOWN_TIMEOUT`.
 - `enable_cron` (boolean) whether to run cron process. Defaults to `false`. You can also set this with the environment variable `GOOD_JOB_ENABLE_CRON`.
+- `cron_graceful_restart_period` (integer) when restarting cron, attempt to re-enqueue jobs that would have been enqueued by cron within this time period (e.g. `1.minute`). This should match the expected downtime during deploys.
 - `enable_listen_notify` (boolean) whether to enqueue and read jobs with Postgres LISTEN/NOTIFY. Defaults to `true`. You can also set this with the environment variable `GOOD_JOB_ENABLE_LISTEN_NOTIFY`.
 - `cron` (hash) cron configuration. Defaults to `{}`. You can also set this as a JSON string with the environment variable `GOOD_JOB_CRON`
 - `cleanup_discarded_jobs` (boolean) whether to destroy discarded jobs when cleaning up preserved jobs using the `$ good_job cleanup_preserved_jobs` CLI command or calling `GoodJob.cleanup_preserved_jobs`. Defaults to `true`. Can also be set with  the environment variable `GOOD_JOB_CLEANUP_DISCARDED_JOBS`. _This configuration is only used when {GoodJob.preserve_job_records} is `true`._
@@ -625,6 +627,9 @@ If you use the [Dashboard](#dashboard) the scheduled tasks can be viewed in the
 # Enable cron enqueuing in this process
 config.good_job.enable_cron = true
 
+# Without zero-downtime deploys, re-attempt previous schedules after a deploy
+config.good_job.cron_graceful_restart_period = 1.minute
+
 # Configure cron with a hash that has a unique key for each recurring job
 config.good_job.cron = {
   # Every 15 minutes, enqueue `ExampleJob.set(priority: -10).perform_later(42, "life", name: "Alice")`

app/models/good_job/cron_entry.rb

@@ -71,6 +71,19 @@ def next_at(previously_at: nil)
       end
     end
 
+    def within(period, previously_at: nil)
+      if cron_proc?
+        result = Rails.application.executor.wrap { cron.call(previously_at || last_job_at) }
+        if result.is_a?(String)
+          Fugit.parse(result).within(period).map(&:to_t)
+        else
+          result
+        end
+      else
+        fugit.within(period).map(&:to_t)
+      end
+    end
+
     def enabled?
       GoodJob::Setting.cron_key_enabled?(key, default: enabled_by_default?)
     end

lib/good_job/configuration.rb

@@ -219,6 +219,12 @@ def cron_entries
       cron.map { |cron_key, params| GoodJob::CronEntry.new(params.merge(key: cron_key)) }
     end
 
+    def cron_graceful_restart_period
+      options[:cron_graceful_restart_period] ||
+        rails_config[:cron_graceful_restart_period] ||
+        env['GOOD_JOB_CRON_GRACEFUL_RESTART_PERIOD']
+    end
+
     # The number of queued jobs to select when polling for a job to run.
     # This limit is intended to avoid locking a large number of rows when selecting eligible jobs
     # from the queue. This value should be higher than the total number of threads across all good_job

lib/good_job/cron_manager.rb

@@ -31,11 +31,12 @@ def self.task_observer(time, output, thread_error) # rubocop:disable Lint/Unused
 
     # @param cron_entries [Array<CronEntry>]
     # @param start_on_initialize [Boolean]
-    def initialize(cron_entries = [], start_on_initialize: false, executor: Concurrent.global_io_executor)
+    def initialize(cron_entries = [], start_on_initialize: false, graceful_restart_period: nil, executor: Concurrent.global_io_executor)
       @executor = executor
       @running = false
       @cron_entries = cron_entries
       @tasks = Concurrent::Hash.new
+      @graceful_restart_period = graceful_restart_period
 
       start if start_on_initialize
       self.class.instances << self
@@ -47,6 +48,7 @@ def start
         @running = true
         cron_entries.each do |cron_entry|
           create_task(cron_entry)
+          create_graceful_tasks(cron_entry) if @graceful_restart_period
         end
       end
     end
@@ -97,5 +99,24 @@ def create_task(cron_entry, previously_at: nil)
       future.add_observer(self.class, :task_observer)
       future.execute
     end
+
+    # Uses the graceful restart period to re-enqueue jobs that were scheduled to run during the period.
+    # The existing uniqueness logic should ensure this does not create duplicate jobs.
+    # @param cron_entry [CronEntry] the CronEntry object to schedule
+    def create_graceful_tasks(cron_entry)
+      return unless @graceful_restart_period
+
+      time_period = @graceful_restart_period.ago..Time.current
+      cron_entry.within(time_period).each do |cron_at|
+        future = Concurrent::Future.new(args: [self, cron_entry, cron_at], executor: @executor) do |_thr_scheduler, thr_cron_entry, thr_cron_at|
+          Rails.application.executor.wrap do
+            cron_entry.enqueue(thr_cron_at) if thr_cron_entry.enabled?
+          end
+        end
+
+        future.add_observer(self.class, :task_observer)
+        future.execute
+      end
+    end
   end
 end

spec/app/models/good_job/cron_entry_spec.rb

@@ -70,6 +70,12 @@ def perform(meaning, name:)
     end
   end
 
+  describe '#within' do
+    it 'returns an array of timestamps for the time period' do
+      expect(entry.within(2.minutes.ago..Time.current)).to eq([Time.current.at_beginning_of_minute - 1.minute, Time.current.at_beginning_of_minute])
+    end
+  end
+
   describe '#enabled' do
     it 'is enabled by default' do
       expect(entry).to be_enabled

spec/lib/good_job/configuration_spec.rb

@@ -282,6 +282,11 @@
 
       expect(configuration.cron).to eq({})
     end
+
+    it 'has a graceful restart period' do
+      allow(Rails.application.config).to receive(:good_job).and_return({ cron_graceful_restart_period: 5.minutes })
+      expect(described_class.new({}).cron_graceful_restart_period).to eq 5.minutes
+    end
   end
 
   describe '#enable_listen_notify' do

spec/lib/good_job/cron_manager_spec.rb

@@ -108,4 +108,35 @@
       end
     end
   end
+
+  describe 'graceful restarts' do
+    let(:cron_entries) do
+      [
+        GoodJob::CronEntry.new(
+          key: 'example',
+          cron: "0 * * * * *",
+          class: "TestJob"
+        ),
+      ]
+    end
+
+    before do
+      stub_const 'TestJob', (Class.new(ActiveJob::Base) do
+        def perform
+        end
+      end)
+
+      ActiveJob::Base.queue_adapter = GoodJob::Adapter.new(execution_mode: :external)
+    end
+
+    it "reenqueues jobs scheduled for the previous period" do
+      cron_manager = described_class.new(cron_entries, start_on_initialize: false, graceful_restart_period: 5.minutes)
+      cron_manager.start
+      cron_manager.shutdown
+
+      wait_until(max: 5) do
+        expect(GoodJob::Job.count).to eq 5
+      end
+    end
+  end
 end