Move limiter logic to child job for copay notifications (#12932)

* Add limiter for batch processing * Adjust limit numbers * Move to concurrent approach * Break out from batch * Decrease concurrent thread count * Revert past decrease * Move throttle to class method * Make static limiter * Lint * Lint pt 2 * Restrict copay notifications to 1000/minute * Move throttle to child job * Limit new statement job * Initialize constants after throttle * Move class throttle method to top of class
department-of-veterans-affairs · Jun 8, 2023 · a8945d1 · a8945d1
1 parent 99e0ba3
commit a8945d1
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 26 deletions.
diff --git a/app/workers/copay_notifications/new_statement_notification_job.rb b/app/workers/copay_notifications/new_statement_notification_job.rb
@@ -17,21 +17,30 @@ class NewStatementNotificationJob
 
     sidekiq_options retry: false
 
+    def self.throttle
+      return Sidekiq::Limiter.unlimited if Rails.env.test?
+
+      Sidekiq::Limiter.bucket('new-copay-statements', 1000, :second, wait_timeout: 60)
+    end
+
+    LIMITER = throttle
     MCP_NOTIFICATION_TEMPLATE = Settings.vanotify.services.dmc.template_id.vha_new_copay_statement_email
     STATSD_KEY_PREFIX = 'api.copay_notifications.new_statement'
 
     def perform(statement)
-      StatsD.increment("#{STATSD_KEY_PREFIX}.total")
-      mpi_response = get_mpi_profile(identifier: statement['veteranIdentifier'],
-                                     identifier_type: statement['identifierType'],
-                                     facility_id: statement['facilityNum'])
-
-      if mpi_response.ok?
-        StatsD.increment("#{STATSD_KEY_PREFIX}.mpi.success")
-        create_notification_email_job(vet360_id: mpi_response.profile.vet360_id, icn: mpi_response.profile.icn)
-      else
-        StatsD.increment("#{STATSD_KEY_PREFIX}.mpi.failure")
-        raise mpi_response.error
+      LIMITER.within_limit do
+        StatsD.increment("#{STATSD_KEY_PREFIX}.total")
+        mpi_response = get_mpi_profile(identifier: statement['veteranIdentifier'],
+                                       identifier_type: statement['identifierType'],
+                                       facility_id: statement['facilityNum'])
+
+        if mpi_response.ok?
+          StatsD.increment("#{STATSD_KEY_PREFIX}.mpi.success")
+          create_notification_email_job(vet360_id: mpi_response.profile.vet360_id, icn: mpi_response.profile.icn)
+        else
+          StatsD.increment("#{STATSD_KEY_PREFIX}.mpi.failure")
+          raise mpi_response.error
+        end
       end
     end
 

diff --git a/app/workers/copay_notifications/parse_new_statements_job.rb b/app/workers/copay_notifications/parse_new_statements_job.rb
@@ -6,28 +6,14 @@ class ParseNewStatementsJob
 
     sidekiq_options retry: false
 
-    def self.throttle
-      return Sidekiq::Limiter.unlimited if Rails.env.test?
-
-      Sidekiq::Limiter.window('new-copay-statements', 1000, 60)
-    end
-
-    LIMITER = throttle
-
     def perform(statements_json_byte)
       StatsD.increment('api.copay_notifications.json_file.total')
       # Decode and parse large json file (~60-90k objects)
       statements_json = Oj.load(Base64.decode64(statements_json_byte))
       unique_statements = statements_json.uniq { |statement| statement['veteranIdentifier'] }
 
-      batch = Sidekiq::Batch.new
-
       unique_statements.each do |statement|
-        LIMITER.within_limit do
-          batch.jobs do
-            CopayNotifications::NewStatementNotificationJob.perform_async(statement)
-          end
-        end
+        CopayNotifications::NewStatementNotificationJob.perform_async(statement)
       end
     end
   end