diff --git a/lib/tasks/event.rake b/lib/tasks/event.rake index 65aca0b..5ab7ff5 100644 --- a/lib/tasks/event.rake +++ b/lib/tasks/event.rake @@ -15,14 +15,25 @@ namespace :event do events = Event .where(source_id: ["crossref", "datacite-crossref"]) .where(created_at: start_date...end_date) + .order(:id) puts("Number of events: #{events.count}") - Parallel.each(events, in_threads: 20) do |event| - SqsUtilities.send_events_other_doi_job_message({ - subj_id: event.subj_id, - obj_id: event.obj_id, - }) + batch_count = 0 + + events.in_batches(of: 10_000) do |batch| + batch_count += 1 + puts("Processing batch: #{batch_count}") + batch_events = batch.select(:id, :subj_id, :obj_id).to_a + + Parallel.each(batch_events, in_threads: 20) do |batch_event| + SqsUtilities.send_events_other_doi_job_message({ + subj_id: batch_event.subj_id, + obj_id: batch_event.obj_id, + }) + end end + + puts("Rake task has completed!") end end