Skip to content

Commit

Permalink
Replace sidekiq scheduled jobs with periodic jobs (#12825)
Browse files Browse the repository at this point in the history
  • Loading branch information
@RachalCassity
RachalCassity authored Jun 6, 2023
1 parent 0576a96 commit 6a88fa6
Show file tree
Hide file tree
Showing 6 changed files with 147 additions and 230 deletions.
64 changes: 5 additions & 59 deletions config/initializers/sidekiq.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,70 +40,16 @@
# history is captured every 30 seconds by default
config.retain_history(30)
end

if defined?(Sidekiq::Enterprise)
require 'periodic_jobs'
config.periodic(&PERIODIC_JOBS)
end
end

config.client_middleware do |chain|
chain.add SidekiqStatsInstrumentation::ClientMiddleware
end

if defined?(Sidekiq::Enterprise)
config.periodic do |mgr|
mgr.register('0 5 * * 1', 'AppealsApi::WeeklyErrorReport')

mgr.register('5 * * * *', 'AppealsApi::HigherLevelReviewUploadStatusBatch')
# Update HigherLevelReview statuses with their Central Mail status
mgr.register('10 * * * *', 'AppealsApi::NoticeOfDisagreementUploadStatusBatch')
# Update NoticeOfDisagreement statuses with their Central Mail status
mgr.register('15 * * * *', 'AppealsApi::SupplementalClaimUploadStatusBatch')
# Update SupplementalClaim statuses with their Central Mail status
mgr.register('0 2,9,16 * * MON-FRI', 'AppealsApi::FlipperStatusAlert')
# Checks status of Flipper features expected to be enabled and alerts to Slack if any are not enabled"

mgr.register('35 * * * *', 'AppsApi::FetchConnections')
# "Fetches and handles notifications for recent application connections and disconnections

mgr.register('0 2,9,16 * * MON-FRI', 'VBADocuments::FlipperStatusAlert')
mgr.register('0 2,9,16 * * MON-FRI', 'VBADocuments::SlackNotifier')
# Notifies slack channel if certain benefits states get stuck
mgr.register('5 */2 * * *', 'VBADocuments::RunUnsuccessfulSubmissions')
# Run VBADocuments::UploadProcessor for submissions that are stuck in uploaded status
mgr.register('0 0 * * MON-FRI', 'VBADocuments::ReportUnsuccessfulSubmissions')
# Daily report of unsuccessful benefits intake submissions
mgr.register('*/2 * * * *', 'VBADocuments::UploadRemover')
# Clean up submitted documents from S3
mgr.register('*/2 * * * *', 'VBADocuments::UploadScanner')
# Poll upload bucket for unprocessed uploads
mgr.register('45 * * * *', 'VBADocuments::UploadStatusBatch')
# Request updated statuses for benefits intake submissions

mgr.register('0 2,9,16 * * MON-FRI', 'VAForms::FlipperStatusAlert')
mgr.register('0 2 * * *', 'VAForms::FetchLatest')

mgr.register('0 16 * * *', 'VANotify::InProgressForms')
mgr.register('0 1 * * *', 'VANotify::ClearStaleInProgressRemindersSent')
mgr.register('0 * * * *', 'VANotify::InProgress1880Form')

mgr.register('0 * * * *', 'PagerDuty::CacheGlobalDowntime')
mgr.register('*/3 * * * *', 'PagerDuty::PollMaintenanceWindows')

mgr.register('0 2 * * *', 'InProgressFormCleaner')
mgr.register('0 */4 * * *', 'MHV::AccountStatisticsJob')
mgr.register('0 3 * * *', 'Form1095::New1095BsJob')
mgr.register('0 2 * * *', 'Veteran::VSOReloader')
mgr.register('30 2 * * *', 'Identity::UserAcceptableVerifiedCredentialTotalsJob')
mgr.register('* 7 * * *', 'SignIn::DeleteExpiredSessionsJob')
mgr.register('15 2 * * *', 'Preneeds::DeleteOldUploads')

mgr.register('* * * * *', 'SidekiqStatsJob')
mgr.register('* * * * *', 'ExternalServicesStatusJob')
mgr.register('* * * * *', 'ExportBreakerStatus')

# Disable FeatureCleanerJob. https://github.com/department-of-veterans-affairs/va.gov-team/issues/53538
# mgr.register('0 0 * * *', 'FeatureCleanerJob')
mgr.register('0 0 * * *', 'Form1010cg::DeleteOldUploadsJob')
mgr.register('0 1 * * *', 'TransactionalEmailAnalyticsJob')
end
end
end

Sidekiq.configure_client do |config|
Expand Down
179 changes: 11 additions & 168 deletions config/sidekiq_scheduler.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,64 +2,22 @@
# over all workers. The `every` option will run the job once per worker instance
# at the specified interval.

AccountLoginStatisticsJob:
cron: "0 6 * * * America/New_York"
description: "Gather account login statistics for statsd"

AppealsApi::DailyErrorReport:
cron: "0 23 * * MON-FRI America/New_York"
class: AppealsApi::DailyErrorReport
description: "Daily report of appeals errors"
###########################
# PERIODIC JOBS ARE REPLACING SIDEKIQ SCHEDULER

AppealsApi::DailyStuckRecordsReport:
cron: '0 8 * * MON-FRI America/New_York'
class: AppealsApi::DailyStuckRecordsReport
description: "Daily report of all stuck appeals submissions"
# Please add new periodic jobs in lib/periodic_jobs.rb

AppealsApi::DecisionReviewReportDaily:
cron: "0 23 * * MON-FRI America/New_York"
class: AppealsApi::DecisionReviewReportDaily
description: "Daily report of appeals submissions"

AppealsApi::DecisionReviewReportWeekly:
cron: "0 23 * * SUN America/New_York"
class: AppealsApi::DecisionReviewReportWeekly
description: "Weekly report of appeals submissions"

AppealsApi::HigherLevelReviewCleanUpWeekOldPii:
every: ['24h', {first_in: '45m'}]
class: AppealsApi::HigherLevelReviewCleanUpWeekOldPii
description: "Remove PII of HigherLevelReviews that have 1) reached one of the 'completed' statuses and 2) are a week old"

AppealsApi::MonthlyStatsReport:
cron: "0 0 1 * *"
class: AppealsApi::MonthlyStatsReport
description: "Email a decision reviews stats report for the past month to configured recipients"

AppealsApi::NoticeOfDisagreementCleanUpWeekOldPii:
every: ['24h', {first_in: '45m'}]
class: AppealsApi::NoticeOfDisagreementCleanUpWeekOldPii
description: "Remove PII of NoticeOfDisagreements that have 1) reached one of the 'completed' statuses and 2) are a week old"

AppealsApi::SupplementalClaimCleanUpPii:
every: ['24h', {first_in: '45m'}]
class: AppealsApi::SupplementalClaimCleanUpPii
description: "Remove PII of SupplementalClaims that have 1) reached one of the 'completed' statuses and 2) are a week old"

CentralMail::DeleteOldClaims:
cron: "30 2 * * * America/New_York"
class: CentralMail::DeleteOldClaims
description: "Clear out central mail claims older than 2 months"

ClaimsApi::ClaimAuditor:
cron: "0 11 * * * America/New_York"
class: ClaimsApi::ClaimAuditor
description: "Daily alert of pending claims longer than acceptable threshold"

ClaimsApi::ReportUnsuccessfulSubmissions:
cron: "0 23 * * * America/New_York"
class: ClaimsApi::ReportUnsuccessfulSubmissions
description: "Weekly report of unsuccessful claims submissions"
CreateDailySpoolFiles:
cron: "0 3 * * MON-FRI America/New_York"
class: EducationForm::CreateDailySpoolFiles
description: "Generate a daily set of spool files and transmit them over SFTP to a backend system"

CovidVaccine::ExpandedScheduledSubmissionJob:
every: '15m'
Expand All @@ -74,134 +32,19 @@ CovidVaccine::ExpandedSubmissionStateJob:
CovidVaccine::ScheduledBatchJob:
cron: "*/15 * * * * America/New_York"
class: CovidVaccine::ScheduledBatchJob
description: Batches received vaccine records and schedules upload to enrollment service.

CreateDailySpoolFiles:
cron: "0 3 * * MON-FRI America/New_York"
class: EducationForm::CreateDailySpoolFiles
description: "Generate a daily set of spool files and transmit them over SFTP to a backend system"

CypressViewportUpdater::UpdateCypressViewportsJob:
cron: "0 12 3 * * America/New_York"
class: CypressViewportUpdater::UpdateCypressViewportsJob
description: "Updates Cypress files in vets-website with data from Google Analytics."

DeleteOldApplications:
cron: "0 0 * * * America/New_York"
class: EducationForm::DeleteOldApplications
description: "Clear out processed 22-1990 applications that are older than 1 month"

DeleteOldClaims:
cron: "0 2 * * * America/New_York"
class: EVSS::DeleteOldClaims
description: "Clear out EVSS disability claims that have not been updated in 24 hours"

DeleteOldPiiLogsJob:
cron: "20 2 * * * America/New_York"
description: "Clear out old personal information logs"

DeleteOldTransactionsJob:
cron: "0 3 * * * America/New_York" # Daily @ 3am Eastern
description: "Deletes old, completed AsyncTransaction records"

EducationForm::Create10203ApplicantDecisionLetters:
cron: "45 4 * * MON-FRI America/New_York"
description: "Send 10203 applicant decision letters to applicants"
enabled: false

EducationForm::Create10203SpoolSubmissionsReport:
cron: "30 4 * * MON-FRI America/New_York"
description: "Send the daily 10203 report to the call center about spool file submissions"

EducationForm::CreateDailyFiscalYearToDateReport:
cron: "0 4 * * * America/New_York"
description: "Send the daily report to VA stakeholders about Education Benefits submissions"

EducationForm::CreateSpoolSubmissionsReport:
cron: "15 4 * * MON-FRI America/New_York"
description: "Send the daily report to the call center about spool file submissions"

EducationForm::Process10203Submissions:
cron: "0 6-18/6 * * * America/New_York"
class: EducationForm::Process10203Submissions
description: "Go through 22-10203 submissions and determine if application should be processed as part of normal submission process or rejected"

FacilityAccessBulkUpdate:
cron: "30 4 * * * America/New_York"
class: Facilities::AccessDataDownload
description: "Download and cache facility access-to-care metric data"

FacilityDentalServiceBulkUpdate:
cron: "25 4 * * * America/New_York"
class: Facilities::DentalServiceReloadJob
description: "Download and cache facility access-to-care metric data"

FacilityLocationVHADriveTime:
cron: "55 4 * * * America/New_York"
class: Facilities::PSSGDownload
description: "Download and store drive time bands"

FacilityMentalHealthBulkUpdate:
cron: "25 4 * * * America/New_York"
class: Facilities::MentalHealthReloadJob
description: "Download and cache facility mental health phone number data"

FailedClaimsReport:
cron: "30 3 * * MON America/New_York"
class: EVSS::FailedClaimsReport
description: "Notify developers about EVSS claims which could not be uploaded"

GmtThresholdsImport:
cron: "0 0 1 */3 * America/New_York"
class: IncomeLimits::GmtThresholdsImport
description: "Import income limit data CSVs from S3"

TestUserDashboard::DailyMaintenance:
cron: "0 0 * * * America/New_York"
class: TestUserDashboard::DailyMaintenance
description: "Checks in TUD users that weren't properly checked in."


StdCountyImport:
cron: "0 0 1 */3 * America/New_York"
class: IncomeLimits::StdCountyImport
description: "Import income limit data CSVs from S3"

StdIncomeThresholdImport:
cron: "0 0 1 */3 * America/New_York"
class: IncomeLimits::StdIncomeThresholdImport
description: "Import income limit data CSVs from S3"

StdStateImport:
cron: "0 0 1 */3 * America/New_York"
class: IncomeLimits::StdStateImport
description: "Import income limit data CSVs from S3"

StdZipcodeImport:
cron: "0 0 1 */3 * America/New_York"
class: IncomeLimits::StdZipcodeImport
description: "Import income limit data CSVs from S3"

VBADocuments::ReportMonthlySubmissions:
cron: "0 2 1 * * America/New_York"
class: VBADocuments::ReportMonthlySubmissions
description: "Monthly report of benefits intake submissions"
SidekiqAlive::CleanupQueues:
cron: "*/30 * * * * America/New_York"
class: SidekiqAlive::CleanupQueues
description: "Cleanup Sidekiq Alive Zombie Queues"

VRE::CreateCh31SubmissionsReportJob:
cron: "0 0 * * * America/New_York"
include_metadata: true
description: "Send a daily report to the VRE team about Chapter 31 submissions"

Mobile::WeeklyMaintenanceWindowLogger:
cron: "0 13 * * 1 America/New_York"
class: Mobile::V0::WeeklyMaintenanceWindowLogger
description: "Weekly logs of maintenance windows"

Webhooks::SchedulerJob:
at: "<%=Settings.vba_documents.v2_enabled ? 0.seconds.from_now : 100.years.from_now%>"
description: Starts off webhook based notifications

SidekiqAlive::CleanupQueues:
cron: "*/30 * * * * America/New_York"
class: SidekiqAlive::CleanupQueues
description: "Cleanup Sidekiq Alive Zombie Queues"
Loading

0 comments on commit 6a88fa6

Please sign in to comment.