20223: Use in_batches in used_by to tamp down memory usage

[arvados.git] / services / api / app / controllers / arvados / v1 / collections_controller.rb

diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index 2e7e2f82b07c024168bd34080f000a91eaea55ac..1d680063666e66eb80523137fa502ce83583bfe8 100644 (file)
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -43,6 +43,14 @@ class Arvados::V1::CollectionsController < ApplicationController
     super
   end
 
+  def update
+    # preserve_version should be disabled unless explicitly asked otherwise.
+    if !resource_attrs[:preserve_version]
+      resource_attrs[:preserve_version] = false
+    end
+    super
+  end
+
   def find_objects_for_index
     opts = {
       include_trash: params[:include_trash] || ['destroy', 'trash', 'untrash'].include?(action_name),
@@ -52,7 +60,7 @@ class Arvados::V1::CollectionsController < ApplicationController
     super
   end
 
-  def find_object_by_uuid
+  def find_object_by_uuid(with_lock: false)
     if loc = Keep::Locator.parse(params[:id])
       loc.strip_hints!
 
@@ -61,26 +69,40 @@ class Arvados::V1::CollectionsController < ApplicationController
         include_old_versions: params[:include_old_versions],
       }
 
-      # It matters which Collection object we pick because we use it to get signed_manifest_text,
-      # the value of which is affected by the value of trash_at.
+      # It matters which Collection object we pick because blob
+      # signatures depend on the value of trash_at.
       #
-      # From postgres doc: "By default, null values sort as if larger than any non-null
-      # value; that is, NULLS FIRST is the default for DESC order, and
-      # NULLS LAST otherwise."
+      # From postgres doc: "By default, null values sort as if larger
+      # than any non-null value; that is, NULLS FIRST is the default
+      # for DESC order, and NULLS LAST otherwise."
       #
       # "trash_at desc" sorts null first, then latest to earliest, so
       # it will select the Collection object with the longest
       # available lifetime.
 
-      if c = Collection.readable_by(*@read_users, opts).where({ portable_data_hash: loc.to_s }).order("trash_at desc").limit(1).first
+      select_attrs = (@select || ["manifest_text"]) | ["portable_data_hash", "trash_at"]
+      model = Collection
+      if with_lock && Rails.configuration.API.LockBeforeUpdate
+        model = model.lock
+      end
+      if c = model.
+               readable_by(*@read_users, opts).
+               where({ portable_data_hash: loc.to_s }).
+               order("trash_at desc").
+               select(select_attrs.join(", ")).
+               limit(1).
+               first
         @object = {
           uuid: c.portable_data_hash,
           portable_data_hash: c.portable_data_hash,
-          manifest_text: c.signed_manifest_text,
+          trash_at: c.trash_at,
         }
+        if select_attrs.index("manifest_text")
+          @object[:manifest_text] = c.manifest_text
+        end
       end
     else
-      super
+      super(with_lock: with_lock)
     end
   end
 
@@ -161,11 +183,11 @@ class Arvados::V1::CollectionsController < ApplicationController
           end
         end
 
-        Container.readable_by(*@read_users).where(output: loc.to_s).each do |c|
+        Container.readable_by(*@read_users).where(output: loc.to_s).in_batches(of: 15).each do |c|
           search_edges(visited, c.uuid, :search_up)
         end
 
-        Container.readable_by(*@read_users).where(log: loc.to_s).each do |c|
+        Container.readable_by(*@read_users).where(log: loc.to_s).in_batches(of: 15).each do |c|
           search_edges(visited, c.uuid, :search_up)
         end
       elsif direction == :search_down
@@ -185,7 +207,7 @@ class Arvados::V1::CollectionsController < ApplicationController
           end
         end
 
-        Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).each do |c|
+        Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).in_batches(of: 15).each do |c|
           if c.output != loc.to_s && c.log != loc.to_s
             search_edges(visited, c.uuid, :search_down)
           end
@@ -254,11 +276,11 @@ class Arvados::V1::CollectionsController < ApplicationController
               end
             end
 
-            ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).each do |cr|
+            ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).in_batches(of: 15).each do |cr|
               search_edges(visited, cr.uuid, :search_up)
             end
 
-            ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).each do |cr|
+            ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).in_batches(of: 15).each do |cr|
               search_edges(visited, cr.uuid, :search_up)
             end
           elsif direction == :search_down
@@ -313,7 +335,7 @@ class Arvados::V1::CollectionsController < ApplicationController
 
   protected
 
-  def load_limit_offset_order_params *args
+  def load_select_param *args
     super
     if action_name == 'index'
       # Omit manifest_text and unsigned_manifest_text from index results unless expressly selected.