14484: Adds migration for collection file count and total size

Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti@veritasgenetics.com>

diff --git a/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb b/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb
index 816514e67ab7e1eac02bb7f82ef5a8e29121da11..97bab1e566d83df3924f87b372af0305d4a18950 100755 (executable)
--- a/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb
+++ b/services/api/db/migrate/20190322174136_add_file_info_to_collection.rb
@@ -3,8 +3,51 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 class AddFileInfoToCollection < ActiveRecord::Migration
-  def change
+  def do_batch(pdhs)
+    pdhs_str = ''
+    pdhs.each do |pdh|
+      pdhs_str << "'" << pdh[0] << "'" << ','
+    end
+
+    collections = ActiveRecord::Base.connection.exec_query(
+      'SELECT DISTINCT portable_data_hash, manifest_text FROM collections '\
+      "WHERE portable_data_hash IN (#{pdhs_str[0..-2]}) "
+    )
+
+    collections.rows.each do |row|
+      file_count = 0
+      file_size_total = 0
+      row[1].scan(/\S+/) do |token|
+        is_file = token.match(/^[[:digit:]]+:[[:digit:]]+:([^\000-\040\\]|\\[0-3][0-7][0-7])+$/)
+        if is_file
+          _, filesize, filename = token.split(':', 3)
+
+          # Avoid counting empty dir placeholders
+          break if filename == '.' && filesize.zero?
+
+          file_count += 1
+          file_size_total += filesize.to_i
+        end
+      end
+      ActiveRecord::Base.connection.exec_query('BEGIN')
+      ActiveRecord::Base.connection.exec_query("UPDATE collections SET file_count=#{file_count}, "\
+                                               "file_size_total=#{file_size_total} "\
+                                               "WHERE portable_data_hash='#{row[0]}'")
+      ActiveRecord::Base.connection.exec_query('COMMIT')
+    end
+  end
+
+  def up
     add_column :collections, :file_count, :integer, default: 0, null: false
     add_column :collections, :file_size_total, :integer, default: 0, null: false
+
+    Container.group_pdhs_for_multiple_transactions('AddFileInfoToCollection') do |pdhs|
+      do_batch(pdhs)
+    end
+  end
+
+  def down
+    remove_column :collections, :file_count
+    remove_column :collections, :file_size_total
   end
 end