14484: Adds test for pdh grouping functionality in the container model

Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti@veritasgenetics.com>

diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index f3da800827b16468137b31bf8c4be4b57754fdd1..0f48a750119164bb0790e1ff926e39bf6c3b0976 100644 (file)
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -411,8 +411,7 @@ class Container < ArvadosModel
   #
   # Correctly groups pdhs to use for batch database updates. Helps avoid
   # updating too many database rows in a single transaction.
-  def self.group_pdhs_for_multiple_transactions(distinct_ordered_pdhs, distinct_pdh_count, log_prefix)
-    batch_size_max = 1 << 28 # 256 MiB
+  def self.group_pdhs_for_multiple_transactions(distinct_ordered_pdhs, distinct_pdh_count, batch_size_max, log_prefix)
     batch_size = 0
     batch_pdhs = {}
     last_pdh = '0'

diff --git a/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb b/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb
index 99db8133d47e807323908a5b92a0948e18cba19d..3e87b0c8b76d0ee372b0ba94b8d3676b80a80996 100755 (executable)
--- a/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb
+++ b/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb
@@ -46,7 +46,11 @@ class AddFileInfoToCollection < ActiveRecord::Migration
       end
     }
 
-    Container.group_pdhs_for_multiple_transactions(ordered_pdh_query, distinct_pdh_count, "AddFileInfoToCollection") do |pdhs|
+    batch_size_max = 1 << 28 # 256 MiB
+    Container.group_pdhs_for_multiple_transactions(ordered_pdh_query,
+                                                   distinct_pdh_count,
+                                                   batch_size_max,
+                                                   "AddFileInfoToCollection") do |pdhs|
       do_batch(pdhs)
     end
   end

diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index 2b7fda8d7f1493725c773ecd0ca9051634cdc9fd..783d2a985fa569d8a2cd0275a28441e60636c2c9 100644 (file)
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -962,8 +962,14 @@ class ContainerTest < ActiveSupport::TestCase
   test "pdh_grouping_by_manifest_size" do
     batch_size_max = 200
     pdhs_in = ['x1+30', 'x2+30', 'x3+201', 'x4+100', 'x5+100']
+    pdh_lambda = lambda { |last_pdh, &block|
+      pdhs = pdhs_in.select{|pdh| pdh > last_pdh} 
+      pdhs.each do |p|
+        block.call(p)
+      end
+    }
     batched_pdhs = []
-    Container.group_pdhs_by_manifest_size(pdhs_in, batch_size_max) do |pdhs|
+    Container.group_pdhs_for_multiple_transactions(pdh_lambda, pdhs_in.size, batch_size_max, "") do |pdhs|
       batched_pdhs << pdhs
     end
     expected = [['x1+30', 'x2+30'], ['x3+201'], ['x4+100', 'x5+100']]