From: Tom Clegg <tclegg@veritasgenetics.com>
Date: Mon, 6 Aug 2018 21:22:48 +0000 (-0400)
Subject: 13752: Remove column size limit.
X-Git-Tag: 1.2.0~38^2
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/63e4678b027feb29ee7c1fc5e344b6d58be8abfa?hp=2b0c4469c697b3ee752c690bab57b421a70e3853

13752: Remove column size limit.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>
---

diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 4772768c8f..85b12a377b 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -190,22 +190,16 @@ class Collection < ArvadosModel
   end
 
   def manifest_files
+    return '' if !self.manifest_text
+
     names = ''
-    if self.manifest_text
-      self.manifest_text.scan(/ \d+:\d+:(\S+)/) do |name|
-        names << name.first.gsub('\040',' ') + "\n"
-        break if names.length > 2**12
-      end
+    self.manifest_text.scan(/ \d+:\d+:(\S+)/) do |name|
+      names << name.first.gsub('\040',' ') + "\n"
     end
-
-    if self.manifest_text and names.length < 2**12
-      self.manifest_text.scan(/^\.\/(\S+)/m) do |stream_name|
-        names << stream_name.first.gsub('\040',' ') + "\n"
-        break if names.length > 2**12
-      end
+    self.manifest_text.scan(/^\.\/(\S+)/m) do |stream_name|
+      names << stream_name.first.gsub('\040',' ') + "\n"
     end
-
-    names[0,2**12]
+    names
   end
 
   def default_empty_manifest
diff --git a/services/api/db/migrate/20180806133039_index_all_filenames.rb b/services/api/db/migrate/20180806133039_index_all_filenames.rb
new file mode 100644
index 0000000000..79259f91d8
--- /dev/null
+++ b/services/api/db/migrate/20180806133039_index_all_filenames.rb
@@ -0,0 +1,18 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class IndexAllFilenames < ActiveRecord::Migration
+  def up
+    ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN file_names TYPE text'
+    Collection.find_each(batch_size: 20) do |c|
+      ActiveRecord::Base.connection.execute "UPDATE collections
+                    SET file_names = #{ActiveRecord::Base.connection.quote(c.manifest_files)}
+                    WHERE uuid = #{ActiveRecord::Base.connection.quote(c.uuid)}
+                    AND portable_data_hash = #{ActiveRecord::Base.connection.quote(c.portable_data_hash)}"
+    end
+  end
+  def down
+    ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN file_names TYPE varchar(8192)'
+  end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index fef457d733..12158e51b4 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -167,7 +167,7 @@ CREATE TABLE public.collections (
     description character varying(524288),
     properties jsonb,
     delete_at timestamp without time zone,
-    file_names character varying(8192),
+    file_names text,
     trash_at timestamp without time zone,
     is_trashed boolean DEFAULT false NOT NULL,
     storage_classes_desired jsonb DEFAULT '["default"]'::jsonb,
@@ -1623,7 +1623,7 @@ CREATE INDEX collection_index_on_properties ON public.collections USING gin (pro
 -- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
 --
 
-CREATE INDEX collections_full_text_search_idx ON public.collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
+CREATE INDEX collections_full_text_search_idx ON public.collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || COALESCE(file_names, (''::character varying)::text))));
 
 
 --
@@ -3119,3 +3119,5 @@ INSERT INTO schema_migrations (version) VALUES ('20180607175050');
 
 INSERT INTO schema_migrations (version) VALUES ('20180608123145');
 
+INSERT INTO schema_migrations (version) VALUES ('20180806133039');
+