Skip to content

Commit

Permalink
feat: implement batch progress reporting
Browse files Browse the repository at this point in the history
  • Loading branch information
gee-forr committed Apr 7, 2021
1 parent 2e66ba6 commit 6b9ffe0
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 1 deletion.
14 changes: 13 additions & 1 deletion lib/activerecord-import/import.rb
Original file line number Diff line number Diff line change
Expand Up @@ -805,17 +805,29 @@ def import_without_validations_or_callbacks( column_names, array_of_attributes,
if supports_import?
# generate the sql
post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
import_size = values_sql.size

batch_size = options[:batch_size] || import_size
run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
progress_proc = options[:batch_progress]
current_batch = 0
batches = (import_size / batch_size.to_f).ceil

batch_size = options[:batch_size] || values_sql.size
values_sql.each_slice(batch_size) do |batch_values|
batch_started_at = Time.now.to_i

# perform the inserts
result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
batch_values,
options,
"#{model_name} Create Many" )

number_inserted += result.num_inserts
ids += result.ids
results += result.results
current_batch += 1

progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
end
else
transaction(requires_new: true) do
Expand Down
9 changes: 9 additions & 0 deletions test/import_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,15 @@
assert_equal 3, result.num_inserts if Topic.supports_import?
end
end

it "should accept and call an optional callable to run after each batch" do
lambda_called = 0

my_proc = ->(_row_count, _batches, _batch, _duration) { lambda_called += 1 }
Topic.import Build(10, :topics), batch_size: 4, batch_progress: my_proc

assert_equal 3, lambda_called
end
end

context "with :synchronize option" do
Expand Down

0 comments on commit 6b9ffe0

Please sign in to comment.