API-27098: Updates to handling of duplicate UUID response from Centra…

…l Mail (#12906)

lib/central_mail/utilities.rb

@@ -12,7 +12,7 @@ module Utilities
     FILE_NUMBER_REGEX = /^\d{8,9}$/
     INVALID_ZIP_CODE_ERROR_REGEX = /Invalid zipCode/
     MISSING_ZIP_CODE_ERROR_REGEX = /Missing zipCode/
-    NON_FAILING_ERROR_REGEX = /Document already uploaded with uuid/
+    DUPLICATE_UUID_REGEX = /Document already uploaded with uuid/
     INVALID_ZIP_CODE_ERROR_MSG = 'Invalid ZIP Code. ZIP Code must be 5 digits, ' \
                                  'or 9 digits in XXXXX-XXXX format. Specify \'00000\' for non-US addresses.'
     MISSING_ZIP_CODE_ERROR_MSG = 'Missing ZIP Code. ZIP Code must be 5 digits, ' \

modules/appeals_api/app/workers/appeals_api/pdf_submit_job.rb

@@ -30,6 +30,8 @@ class PdfSubmitJob
     # 503 Database Offline || SOLR Service Offline || Intake API is undergoing maintenance
     RETRYABLE_EMMS_RESP_STATUS_CODES = [401, 429, 500, 503].freeze
 
+    STATSD_DUPLICATE_UUID_KEY = 'api.appeals.document_upload.duplicate_uuid'
+
     # Retry for ~7 days
     sidekiq_options retry: 20, unique_for: 7.days
 
@@ -75,13 +77,24 @@ def upload_to_central_mail(appeal, pdf_path)
 
     def process_response(response, appeal, metadata)
       if response.success?
-        appeal.update_status!(status: 'submitted')
-        log_submission(appeal, metadata)
+        handle_successful_submission(appeal, metadata)
+      elsif response.status == 400 && response.body.match?(DUPLICATE_UUID_REGEX)
+        StatsD.increment(STATSD_DUPLICATE_UUID_KEY)
+        Rails.logger.warn("#{appeal.class.to_s.gsub('::', ' ')}: Duplicate UUID submitted to Central Mail",
+                          'uuid' => appeal.id)
+        # Treating these as a 'success' is intentional; we have confirmed that when we receive the 'duplicate UUID'
+        # response from Central Mail, this indicates that there was an earlier submission that was successful
+        handle_successful_submission(appeal, metadata)
       else
         map_error(response.status, response.body, AppealsApi::UploadError)
       end
     end
 
+    def handle_successful_submission(appeal, metadata)
+      appeal.update_status!(status: 'submitted')
+      log_submission(appeal, metadata)
+    end
+
     def log_upload_error(appeal, e)
       Rails.logger.error("#{appeal.class.to_s.gsub('::', ' ')}: Submission failure",
                          'source' => appeal.consumer_name,

modules/appeals_api/spec/workers/pdf_submit_job_spec.rb

@@ -226,6 +226,36 @@
     end
   end
 
+  context 'with a duplicate UUID response from Central Mail' do
+    before do
+      allow(CentralMail::Service).to receive(:new) { client_stub }
+      allow(faraday_response).to receive(:status).and_return(400)
+      allow(faraday_response).to receive(:body)
+        .and_return("Document already uploaded with uuid [uuid: #{higher_level_review.id}]")
+      allow(faraday_response).to receive(:success?).and_return(false)
+      expect(client_stub).to receive(:upload).and_return(faraday_response)
+      allow(StatsD).to receive(:increment)
+      allow(Rails.logger).to receive(:warn)
+    end
+
+    it 'sets the appeal status to submitted' do
+      described_class.new.perform(higher_level_review.id, 'AppealsApi::HigherLevelReview', 'V2')
+      expect(higher_level_review.reload.status).to eq('submitted')
+    end
+
+    it 'increments the StatsD duplicate UUID counter' do
+      described_class.new.perform(higher_level_review.id, 'AppealsApi::HigherLevelReview', 'V2')
+      expect(StatsD).to have_received(:increment).with(described_class::STATSD_DUPLICATE_UUID_KEY)
+    end
+
+    it 'logs a duplicate UUID warning' do
+      described_class.new.perform(higher_level_review.id, 'AppealsApi::HigherLevelReview', 'V2')
+      expect(Rails.logger).to have_received(:warn)
+        .with('AppealsApi HlrPdfSubmitWrapper: Duplicate UUID submitted to Central Mail',
+              'uuid' => higher_level_review.id)
+    end
+  end
+
   context 'an error throws' do
     it 'updates the NOD status to reflect the error' do
       submit_job_worker = described_class.new

modules/vba_documents/app/workers/vba_documents/upload_processor.rb

@@ -13,6 +13,8 @@ class UploadProcessor
     include Sidekiq::Worker
     include VBADocuments::UploadValidations
 
+    STATSD_DUPLICATE_UUID_KEY = 'api.vba.document_upload.duplicate_uuid'
+
     # Ensure that multiple jobs for the same GUID aren't spawned,
     # to avoid race condition when parsing the multipart file
     sidekiq_options unique_for: 30.days
@@ -143,15 +145,25 @@ def submit(metadata, parts)
 
     def process_response(response)
       # record submission attempt, record time and success status to an array
-      if response.success? || response.body.match?(NON_FAILING_ERROR_REGEX) # TODO: GovCIO needs to return this...
-        @upload.update(status: 'received')
-        @upload.track_uploaded_received(:cause, @cause)
+      if response.success?
+        handle_successful_submission
+      elsif response.status == 400 && response.body.match?(DUPLICATE_UUID_REGEX)
+        StatsD.increment(STATSD_DUPLICATE_UUID_KEY)
+        Rails.logger.warn("#{self.class.name}: Duplicate UUID submitted to Central Mail", 'uuid' => @upload.guid)
+        # Treating these as a 'success' is intentional; we have confirmed that when we receive the 'duplicate UUID'
+        # response from Central Mail, this indicates that there was an earlier submission that was successful
+        handle_successful_submission
       elsif response.status == 429 && response.body =~ /UUID already in cache/
         @upload.track_uploaded_received(:uuid_already_in_cache_cause, @cause)
         @upload.track_concurrent_duplicate
       else
         map_error(response.status, response.body, VBADocuments::UploadError)
       end
     end
+
+    def handle_successful_submission
+      @upload.update(status: 'received')
+      @upload.track_uploaded_received(:cause, @cause)
+    end
   end
 end

modules/vba_documents/spec/workers/upload_processor_spec.rb

@@ -258,6 +258,36 @@
       end
     end
 
+    context 'with a duplicate UUID response from Central Mail' do
+      before do
+        allow(VBADocuments::MultipartParser).to receive(:parse) { valid_parts_attachment }
+        allow(CentralMail::Service).to receive(:new) { client_stub }
+        allow(faraday_response).to receive(:status).and_return(400)
+        allow(faraday_response).to receive(:body)
+          .and_return("Document already uploaded with uuid [uuid: #{upload.guid}]")
+        allow(faraday_response).to receive(:success?).and_return(false)
+        allow(client_stub).to receive(:upload).and_return(faraday_response)
+        allow(StatsD).to receive(:increment)
+        allow(Rails.logger).to receive(:warn)
+      end
+
+      it 'sets the upload submission status to received' do
+        described_class.new.perform(upload.guid, test_caller)
+        expect(upload.reload.status).to eq('received')
+      end
+
+      it 'increments the StatsD duplicate UUID counter' do
+        described_class.new.perform(upload.guid, test_caller)
+        expect(StatsD).to have_received(:increment).with(described_class::STATSD_DUPLICATE_UUID_KEY)
+      end
+
+      it 'logs a duplicate UUID warning' do
+        described_class.new.perform(upload.guid, test_caller)
+        expect(Rails.logger).to have_received(:warn)
+          .with("#{described_class}: Duplicate UUID submitted to Central Mail", 'uuid' => upload.guid)
+      end
+    end
+
     it 'sets error status for invalid multipart format' do
       allow(VBADocuments::MultipartParser).to receive(:parse)
         .and_raise(VBADocuments::UploadError.new(code: 'DOC101'))