chore(scalability): Add dedicated clock worker (#2971)

- Clock job should not impact other queues - Adding a dedicated clock worker
getlago · Dec 18, 2024 · 70858cd · 70858cd
1 parent 0b8b0a0
commit 70858cd
Show file tree

Hide file tree

Showing 22 changed files with 154 additions and 20 deletions.
diff --git a/app/jobs/clock/activate_subscriptions_job.rb b/app/jobs/clock/activate_subscriptions_job.rb
@@ -4,7 +4,13 @@ module Clock
   class ActivateSubscriptionsJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     unique :until_executed, on_conflict: :log
 

diff --git a/app/jobs/clock/api_keys/track_usage_job.rb b/app/jobs/clock/api_keys/track_usage_job.rb
@@ -5,7 +5,13 @@ module ApiKeys
     class TrackUsageJob < ApplicationJob
       include SentryCronConcern
 
-      queue_as 'clock'
+      queue_as do
+        if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+          :clock_worker
+        else
+          :clock
+        end
+      end
 
       def perform
         ::ApiKeys::TrackUsageService.call

diff --git a/app/jobs/clock/compute_all_daily_usages_job.rb b/app/jobs/clock/compute_all_daily_usages_job.rb
@@ -4,7 +4,13 @@ module Clock
   class ComputeAllDailyUsagesJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     def perform
       DailyUsages::ComputeAllService.call

diff --git a/app/jobs/clock/create_interval_wallet_transactions_job.rb b/app/jobs/clock/create_interval_wallet_transactions_job.rb
@@ -4,7 +4,13 @@ module Clock
   class CreateIntervalWalletTransactionsJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     def perform
       Wallets::CreateIntervalWalletTransactionsService.call

diff --git a/app/jobs/clock/events_validation_job.rb b/app/jobs/clock/events_validation_job.rb
@@ -4,7 +4,13 @@ module Clock
   class EventsValidationJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     unique :until_executed
 

diff --git a/app/jobs/clock/finalize_invoices_job.rb b/app/jobs/clock/finalize_invoices_job.rb
@@ -4,7 +4,13 @@ module Clock
   class FinalizeInvoicesJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     unique :until_executed, on_conflict: :log
 

diff --git a/app/jobs/clock/free_trial_subscriptions_biller_job.rb b/app/jobs/clock/free_trial_subscriptions_biller_job.rb
@@ -4,7 +4,13 @@ module Clock
   class FreeTrialSubscriptionsBillerJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     def perform
       Subscriptions::FreeTrialBillingService.call

diff --git a/app/jobs/clock/mark_invoices_as_payment_overdue_job.rb b/app/jobs/clock/mark_invoices_as_payment_overdue_job.rb
@@ -4,7 +4,13 @@ module Clock
   class MarkInvoicesAsPaymentOverdueJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     def perform
       Invoice

diff --git a/app/jobs/clock/process_dunning_campaigns_job.rb b/app/jobs/clock/process_dunning_campaigns_job.rb
@@ -4,7 +4,13 @@ module Clock
   class ProcessDunningCampaignsJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     unique :until_executed, on_conflict: :log
 

diff --git a/app/jobs/clock/refresh_draft_invoices_job.rb b/app/jobs/clock/refresh_draft_invoices_job.rb
@@ -4,7 +4,13 @@ module Clock
   class RefreshDraftInvoicesJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     unique :until_executed, on_conflict: :log
 

diff --git a/app/jobs/clock/refresh_lifetime_usages_job.rb b/app/jobs/clock/refresh_lifetime_usages_job.rb
@@ -4,7 +4,14 @@ module Clock
   class RefreshLifetimeUsagesJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
+
     unique :until_executed, on_conflict: :log
 
     def perform

diff --git a/app/jobs/clock/refresh_wallets_ongoing_balance_job.rb b/app/jobs/clock/refresh_wallets_ongoing_balance_job.rb
@@ -4,7 +4,13 @@ module Clock
   class RefreshWalletsOngoingBalanceJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     unique :until_executed, on_conflict: :log
 

diff --git a/app/jobs/clock/retry_failed_invoices_job.rb b/app/jobs/clock/retry_failed_invoices_job.rb
@@ -4,7 +4,13 @@ module Clock
   class RetryFailedInvoicesJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     def perform
       Invoice

diff --git a/app/jobs/clock/retry_generating_subscription_invoices_job.rb b/app/jobs/clock/retry_generating_subscription_invoices_job.rb
@@ -4,7 +4,13 @@ module Clock
   class RetryGeneratingSubscriptionInvoicesJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     THRESHOLD = -> { 1.day.ago }
 

diff --git a/app/jobs/clock/subscriptions_biller_job.rb b/app/jobs/clock/subscriptions_biller_job.rb
@@ -4,7 +4,13 @@ module Clock
   class SubscriptionsBillerJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     def perform
       Subscriptions::BillingService.call

diff --git a/app/jobs/clock/subscriptions_to_be_terminated_job.rb b/app/jobs/clock/subscriptions_to_be_terminated_job.rb
@@ -4,7 +4,13 @@ module Clock
   class SubscriptionsToBeTerminatedJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     def perform
       Subscription

diff --git a/app/jobs/clock/terminate_coupons_job.rb b/app/jobs/clock/terminate_coupons_job.rb
@@ -4,7 +4,13 @@ module Clock
   class TerminateCouponsJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as "clock"
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     def perform
       Coupons::TerminateService.terminate_all_expired

diff --git a/app/jobs/clock/terminate_ended_subscriptions_job.rb b/app/jobs/clock/terminate_ended_subscriptions_job.rb
@@ -4,7 +4,13 @@ module Clock
   class TerminateEndedSubscriptionsJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     def perform
       Subscription

diff --git a/app/jobs/clock/terminate_wallets_job.rb b/app/jobs/clock/terminate_wallets_job.rb
@@ -4,7 +4,13 @@ module Clock
   class TerminateWalletsJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     def perform
       Wallet.active.expired.find_each do |wallet|

diff --git a/app/jobs/clock/webhooks_cleanup_job.rb b/app/jobs/clock/webhooks_cleanup_job.rb
@@ -4,7 +4,13 @@ module Clock
   class WebhooksCleanupJob < ApplicationJob
     include SentryCronConcern
 
-    queue_as 'clock'
+    queue_as do
+      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_CLOCK'])
+        :clock_worker
+      else
+        :clock
+      end
+    end
 
     def perform
       Webhook.where('updated_at < ?', 90.days.ago).destroy_all

diff --git a/config/sidekiq/sidekiq_clock.yml b/config/sidekiq/sidekiq_clock.yml
@@ -0,0 +1,10 @@
+concurrency: 10
+timeout: 25
+retry: 1
+queues:
+  - clock_worker
+
+production:
+  concurrency: <%= ENV.fetch('SIDEKIQ_CONCURRENCY', 5) %>
+staging:
+  concurrency: 10
diff --git a/scripts/start.clock.worker.sh b/scripts/start.clock.worker.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec bundle exec sidekiq -C config/sidekiq/sidekiq_clock.yml
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		#!/bin/bash

		exec bundle exec sidekiq -C config/sidekiq/sidekiq_clock.yml