From fe08de0ddf1d3e536cb3518dcb9c82ca62197273 Mon Sep 17 00:00:00 2001 From: Eric Biagiotti Date: Wed, 27 Mar 2019 16:56:47 -0400 Subject: [PATCH] 14484: Adds test for pdh grouping functionality in the container model Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti --- services/api/app/models/container.rb | 3 +-- .../migrate/20190322174136_add_file_info_to_collection.rb | 6 +++++- services/api/test/unit/container_test.rb | 8 +++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb index f3da800827..0f48a75011 100644 --- a/services/api/app/models/container.rb +++ b/services/api/app/models/container.rb @@ -411,8 +411,7 @@ class Container < ArvadosModel # # Correctly groups pdhs to use for batch database updates. Helps avoid # updating too many database rows in a single transaction. - def self.group_pdhs_for_multiple_transactions(distinct_ordered_pdhs, distinct_pdh_count, log_prefix) - batch_size_max = 1 << 28 # 256 MiB + def self.group_pdhs_for_multiple_transactions(distinct_ordered_pdhs, distinct_pdh_count, batch_size_max, log_prefix) batch_size = 0 batch_pdhs = {} last_pdh = '0' diff --git a/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb b/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb index 99db8133d4..3e87b0c8b7 100755 --- a/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb +++ b/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb @@ -46,7 +46,11 @@ class AddFileInfoToCollection < ActiveRecord::Migration end } - Container.group_pdhs_for_multiple_transactions(ordered_pdh_query, distinct_pdh_count, "AddFileInfoToCollection") do |pdhs| + batch_size_max = 1 << 28 # 256 MiB + Container.group_pdhs_for_multiple_transactions(ordered_pdh_query, + distinct_pdh_count, + batch_size_max, + "AddFileInfoToCollection") do |pdhs| do_batch(pdhs) end end diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb index 2b7fda8d7f..783d2a985f 100644 --- a/services/api/test/unit/container_test.rb +++ b/services/api/test/unit/container_test.rb @@ -962,8 +962,14 @@ class ContainerTest < ActiveSupport::TestCase test "pdh_grouping_by_manifest_size" do batch_size_max = 200 pdhs_in = ['x1+30', 'x2+30', 'x3+201', 'x4+100', 'x5+100'] + pdh_lambda = lambda { |last_pdh, &block| + pdhs = pdhs_in.select{|pdh| pdh > last_pdh} + pdhs.each do |p| + block.call(p) + end + } batched_pdhs = [] - Container.group_pdhs_by_manifest_size(pdhs_in, batch_size_max) do |pdhs| + Container.group_pdhs_for_multiple_transactions(pdh_lambda, pdhs_in.size, batch_size_max, "") do |pdhs| batched_pdhs << pdhs end expected = [['x1+30', 'x2+30'], ['x3+201'], ['x4+100', 'x5+100']] -- 2.30.2