20223: Use in_batches in used_by to tamp down memory usage

[arvados.git] / services / api / app / controllers / arvados / v1 / collections_controller.rb

diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index 1b0850a5802eab5e7b451cf40ac1beecf392a61b..1d680063666e66eb80523137fa502ce83583bfe8 100644 (file)
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -60,7 +60,7 @@ class Arvados::V1::CollectionsController < ApplicationController
     super
   end
 
-  def find_object_by_uuid
+  def find_object_by_uuid(with_lock: false)
     if loc = Keep::Locator.parse(params[:id])
       loc.strip_hints!
 
@@ -80,21 +80,29 @@ class Arvados::V1::CollectionsController < ApplicationController
       # it will select the Collection object with the longest
       # available lifetime.
 
-      if c = Collection.
+      select_attrs = (@select || ["manifest_text"]) | ["portable_data_hash", "trash_at"]
+      model = Collection
+      if with_lock && Rails.configuration.API.LockBeforeUpdate
+        model = model.lock
+      end
+      if c = model.
                readable_by(*@read_users, opts).
                where({ portable_data_hash: loc.to_s }).
                order("trash_at desc").
-               select(((@select || ["manifest_text"]) | ["portable_data_hash", "trash_at"]).join(", ")).
+               select(select_attrs.join(", ")).
                limit(1).
                first
         @object = {
           uuid: c.portable_data_hash,
           portable_data_hash: c.portable_data_hash,
-          manifest_text: c.manifest_text,
+          trash_at: c.trash_at,
         }
+        if select_attrs.index("manifest_text")
+          @object[:manifest_text] = c.manifest_text
+        end
       end
     else
-      super
+      super(with_lock: with_lock)
     end
   end
 
@@ -175,11 +183,11 @@ class Arvados::V1::CollectionsController < ApplicationController
           end
         end
 
-        Container.readable_by(*@read_users).where(output: loc.to_s).each do |c|
+        Container.readable_by(*@read_users).where(output: loc.to_s).in_batches(of: 15).each do |c|
           search_edges(visited, c.uuid, :search_up)
         end
 
-        Container.readable_by(*@read_users).where(log: loc.to_s).each do |c|
+        Container.readable_by(*@read_users).where(log: loc.to_s).in_batches(of: 15).each do |c|
           search_edges(visited, c.uuid, :search_up)
         end
       elsif direction == :search_down
@@ -199,7 +207,7 @@ class Arvados::V1::CollectionsController < ApplicationController
           end
         end
 
-        Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).each do |c|
+        Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).in_batches(of: 15).each do |c|
           if c.output != loc.to_s && c.log != loc.to_s
             search_edges(visited, c.uuid, :search_down)
           end
@@ -268,11 +276,11 @@ class Arvados::V1::CollectionsController < ApplicationController
               end
             end
 
-            ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).each do |cr|
+            ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).in_batches(of: 15).each do |cr|
               search_edges(visited, cr.uuid, :search_up)
             end
 
-            ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).each do |cr|
+            ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).in_batches(of: 15).each do |cr|
               search_edges(visited, cr.uuid, :search_up)
             end
           elsif direction == :search_down