bug(DataExport) - Decrease batch size for data export & use tempfile (#…

…2786)

## Description

when running sidekiq with high amount of concurrency, the default batch
size could result in large amounts of longer running jobs. These could
make a worker consume too much memory.

This PR aims to fine-tune the batch size by reducing it by a factor 5
(so jobs will be faster and consume less memory individually). It will
also not keep the entire CSV in memory, but rather use a tempfile.

app/services/data_exports/csv/invoice_fees.rb

@@ -21,7 +21,7 @@ def initialize(
       end
 
       def call
-        result.csv_lines = ::CSV.generate(headers: false) do |csv|
+        result.csv_file = with_csv do |csv|
           invoices.each do |invoice|
             serialized_invoice = serializer_klass.new(invoice).serialize
 

app/services/data_exports/csv/invoices.rb

@@ -15,7 +15,7 @@ def initialize(data_export_part:, serializer_klass: V1::InvoiceSerializer)
       end
 
       def call
-        result.csv_lines = ::CSV.generate(headers: false) do |csv|
+        result.csv_file = with_csv do |csv|
           invoices.each do |invoice|
             csv << serialized_invoice(invoice)
           end
@@ -53,6 +53,14 @@ def self.headers
 
       private
 
+      def with_csv
+        tempfile = Tempfile.create([data_export_part.id, ".csv"])
+        yield CSV.new(tempfile, headers: false)
+
+        tempfile.rewind
+        tempfile
+      end
+
       attr_reader :data_export_part, :serializer_klass, :output, :batch_size
 
       def serialized_invoice(invoice)

app/services/data_exports/export_resources_service.rb

@@ -4,7 +4,7 @@ module DataExports
   class ExportResourcesService < BaseService
     EXPIRED_FAILURE_MESSAGE = 'Data Export already expired'
     PROCESSED_FAILURE_MESSAGE = 'Data Export already processed'
-    DEFAULT_BATCH_SIZE = 100
+    DEFAULT_BATCH_SIZE = 20
 
     ResourceTypeNotSupportedError = Class.new(StandardError)
 

app/services/data_exports/process_part_service.rb

@@ -10,10 +10,15 @@ def initialize(data_export_part:)
 
     def call
       result.data_export_part = data_export_part
+      return result if data_export_part.completed
 
       # produce CSV lines into StringIO
       export_result = data_export.export_class.call(data_export_part:).raise_if_error!
-      data_export_part.update!(csv_lines: export_result.csv_lines, completed: true)
+      file = export_result.csv_file
+      data_export_part.update!(csv_lines: file.read, completed: true)
+      # Explicitely close and unlink the file
+      file.close
+      File.unlink(file.path)
 
       # check if we are the last one to finish
       if last_completed

spec/services/data_exports/csv/invoice_fees_spec.rb

@@ -130,7 +130,11 @@
 
       expect(result).to be_success
 
-      generated_csv = result.csv_lines
+      file = result.csv_file
+      generated_csv = file.read
+
+      file.close
+      File.unlink(file.path)
 
       expect(generated_csv).to eq(expected_csv)
     end

spec/services/data_exports/csv/invoices_spec.rb

@@ -90,8 +90,11 @@
       CSV
 
       expect(result).to be_success
-      generated_csv = result.csv_lines
+      file = result.csv_file
+      generated_csv = file.read
 
+      file.close
+      File.unlink(file.path)
       expect(generated_csv).to eq(expected_csv)
     end
   end