From 7a77da2c224db3a7ba27cf2390f8d696016a4fbb Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Wed, 6 Jul 2016 11:47:45 -0400 Subject: [PATCH] 9542: Avoid retrieving huge result sets from postgres. --- services/api/lib/eventbus.rb | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/services/api/lib/eventbus.rb b/services/api/lib/eventbus.rb index aaeebdccf0..e7f2bb1310 100644 --- a/services/api/lib/eventbus.rb +++ b/services/api/lib/eventbus.rb @@ -93,8 +93,8 @@ class EventBus begin # Must have at least one filter set up to receive events if ws.filters.length > 0 - # Start with log rows readable by user, sorted in ascending order - logs = Log.readable_by(ws.user).order("id asc") + # Start with log rows readable by user + logs = Log.readable_by(ws.user) cond_id = nil cond_out = [] @@ -132,11 +132,21 @@ class EventBus logs = logs.where(cond_id, *param_out) end - # Execute query and actually send the matching log rows - logs.each do |l| + # Execute query and actually send the matching log rows. Load + # the full log records only when we're ready to send them, + # though: otherwise, (1) postgres has to build the whole + # result set and return it to us before we can send the first + # event, and (2) we store lots of records in memory while + # waiting to spool them out to the client. Both of these are + # troublesome when log records are large (e.g., a collection + # update contains both old and new manifest_text). + # + # Note: find_each implies order('id asc'), which is what we + # want. + logs.select(:id).find_each do |l| if not ws.sent_ids.include?(l.id) # only send if not a duplicate - ws.send(l.as_api_response.to_json) + ws.send(Log.find(l.id).as_api_response.to_json) end if not ws.last_log_id.nil? # record ids only when sending "catchup" messages, not notifies -- 2.30.2