1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: AGPL-3.0
7 class Arvados::V1::CollectionsControllerTest < ActionController::TestCase
10 PERM_TOKEN_RE = /\+A[[:xdigit:]]+@[[:xdigit:]]{8}\b/
12 def permit_unsigned_manifests isok=true
13 # Set security model for the life of a test.
14 Rails.configuration.Collections.BlobSigning = !isok
17 def assert_signed_manifest manifest_text, label='', token: false
18 assert_not_nil manifest_text, "#{label} manifest_text was nil"
19 manifest_text.scan(/ [[:xdigit:]]{32}\S*/) do |tok|
20 assert_match(PERM_TOKEN_RE, tok,
21 "Locator in #{label} manifest_text was not signed")
23 bare = tok.gsub(/\+A[^\+]*/, '').sub(/^ /, '')
24 exp = tok[/\+A[[:xdigit:]]+@([[:xdigit:]]+)/, 1].to_i(16)
25 sig = Blob.sign_locator(
27 key: Rails.configuration.Collections.BlobSigningKey,
29 api_token: token)[/\+A[^\+]*/, 0]
30 assert_includes tok, sig
35 def assert_unsigned_manifest txt, label=''
36 assert_not_nil(txt, "#{label} unsigned_manifest_text was nil")
38 txt.scan(/ [[:xdigit:]]{32}\S*/) do |tok|
40 refute_match(PERM_TOKEN_RE, tok,
41 "Locator in #{label} unsigned_manifest_text was signed: #{tok}")
46 test "should get index" do
47 authorize_with :active
49 assert_response :success
50 assert(assigns(:objects).andand.any?, "no Collections returned in index")
51 refute(json_response["items"].any? { |c| c.has_key?("manifest_text") },
52 "basic Collections index included manifest_text")
53 refute(json_response["items"].any? { |c| c["uuid"] == collections(:collection_owned_by_active_past_version_1).uuid },
54 "basic Collections index included past version")
57 test "get index with include_old_versions" do
58 authorize_with :active
60 include_old_versions: true
62 assert_response :success
63 assert(assigns(:objects).andand.any?, "no Collections returned in index")
64 assert(json_response["items"].any? { |c| c["uuid"] == collections(:collection_owned_by_active_past_version_1).uuid },
65 "past version not included on index")
68 test "collections.get returns unsigned locators, and no unsigned_manifest_text" do
69 permit_unsigned_manifests
70 authorize_with :active
71 get :show, params: {id: collections(:foo_file).uuid}
72 assert_response :success
73 assert_unsigned_manifest json_response["manifest_text"], 'foo_file'
74 refute_includes json_response, 'unsigned_manifest_text'
77 ['v1token', 'v2token'].each do |token_method|
78 test "signatures with #{token_method} are accepted" do
79 token = api_client_authorizations(:active).send(token_method)
80 signed = Blob.sign_locator(
81 'acbd18db4cc2f85cedef654fccc4a4d8+3',
82 key: Rails.configuration.Collections.BlobSigningKey,
84 authorize_with_token token
85 put :update, params: {
86 id: collections(:collection_owned_by_active).uuid,
88 manifest_text: ". #{signed} 0:3:foo.txt\n",
91 assert_response :success
92 assert_unsigned_manifest json_response['manifest_text'], 'updated'
96 test "index with manifest_text selected returns unsigned locators" do
97 columns = %w(uuid owner_uuid manifest_text)
98 authorize_with :active
99 get :index, params: {select: columns}
100 assert_response :success
101 assert(assigns(:objects).andand.any?,
102 "no Collections returned for index with columns selected")
103 json_response["items"].each do |coll|
104 assert_equal(coll.keys - ['kind'], columns,
105 "Collections index did not respect selected columns")
106 assert_unsigned_manifest coll['manifest_text'], coll['uuid']
110 test "index with unsigned_manifest_text selected returns only unsigned locators" do
111 authorize_with :active
112 get :index, params: {select: ['unsigned_manifest_text']}
113 assert_response :success
114 assert_operator json_response["items"].count, :>, 0
116 json_response["items"].each do |coll|
117 assert_equal(coll.keys - ['kind'], ['unsigned_manifest_text'],
118 "Collections index did not respect selected columns")
119 assert_nil coll['manifest_text']
120 locs += assert_unsigned_manifest coll['unsigned_manifest_text'], coll['uuid']
122 assert_operator locs, :>, 0, "no locators found in any manifests"
125 test 'index without select returns everything except manifest' do
126 authorize_with :active
128 assert_response :success
129 assert json_response['items'].any?
130 json_response['items'].each do |coll|
131 assert_includes(coll.keys, 'uuid')
132 assert_includes(coll.keys, 'name')
133 assert_includes(coll.keys, 'created_at')
134 refute_includes(coll.keys, 'manifest_text')
138 ['', nil, false, 'null'].each do |select|
139 test "index with select=#{select.inspect} returns everything except manifest" do
140 authorize_with :active
141 get :index, params: {select: select}
142 assert_response :success
143 assert json_response['items'].any?
144 json_response['items'].each do |coll|
145 assert_includes(coll.keys, 'uuid')
146 assert_includes(coll.keys, 'name')
147 assert_includes(coll.keys, 'created_at')
148 refute_includes(coll.keys, 'manifest_text')
154 ["uuid", "manifest_text"],
156 '["uuid", "manifest_text"]'].each do |select|
157 test "index with select=#{select.inspect} returns no name" do
158 authorize_with :active
159 get :index, params: {select: select}
160 assert_response :success
161 assert json_response['items'].any?
162 json_response['items'].each do |coll|
163 refute_includes(coll.keys, 'name')
168 [0,1,2].each do |limit|
169 test "get index with limit=#{limit}" do
170 authorize_with :active
171 get :index, params: {limit: limit}
172 assert_response :success
173 assert_equal limit, assigns(:objects).count
174 resp = JSON.parse(@response.body)
175 assert_equal limit, resp['limit']
179 test "items.count == items_available" do
180 authorize_with :active
181 get :index, params: {limit: 100000}
182 assert_response :success
183 resp = JSON.parse(@response.body)
184 assert_equal resp['items_available'], assigns(:objects).length
185 assert_equal resp['items_available'], resp['items'].count
186 unique_uuids = resp['items'].collect { |i| i['uuid'] }.compact.uniq
187 assert_equal unique_uuids.count, resp['items'].count
190 test "items.count == items_available with filters" do
191 authorize_with :active
192 get :index, params: {
194 filters: [['uuid','=',collections(:foo_file).uuid]]
196 assert_response :success
197 assert_equal 1, assigns(:objects).length
198 assert_equal 1, json_response['items_available']
199 assert_equal 1, json_response['items'].count
202 test "get index with limit=2 offset=99999" do
203 # Assume there are not that many test fixtures.
204 authorize_with :active
205 get :index, params: {limit: 2, offset: 99999}
206 assert_response :success
207 assert_equal 0, assigns(:objects).count
208 resp = JSON.parse(@response.body)
209 assert_equal 2, resp['limit']
210 assert_equal 99999, resp['offset']
213 def request_capped_index(params={})
214 authorize_with :user1_with_load
215 coll1 = collections(:collection_1_of_201)
216 Rails.configuration.API.MaxIndexDatabaseRead =
217 yield(coll1.manifest_text.size)
218 get :index, params: {
219 select: %w(uuid manifest_text),
220 filters: [["owner_uuid", "=", coll1.owner_uuid]],
225 test "index with manifest_text limited by max_index_database_read returns non-empty" do
226 request_capped_index() { |_| 1 }
227 assert_response :success
228 assert_equal(1, json_response["items"].size)
229 assert_equal(1, json_response["limit"])
230 assert_equal(201, json_response["items_available"])
233 test "max_index_database_read size check follows same order as real query" do
234 authorize_with :user1_with_load
235 txt = '.' + ' d41d8cd98f00b204e9800998ecf8427e+0'*1000 + " 0:0:empty.txt\n"
236 c = Collection.create! manifest_text: txt, name: '0000000000000000000'
237 request_capped_index(select: %w(uuid manifest_text name),
239 filters: [['name','>=',c.name]]) do |_|
242 assert_response :success
243 assert_equal(1, json_response["items"].size)
244 assert_equal(1, json_response["limit"])
245 assert_equal(c.uuid, json_response["items"][0]["uuid"])
246 # The effectiveness of the test depends on >1 item matching the filters.
247 assert_operator(1, :<, json_response["items_available"])
250 test "index with manifest_text limited by max_index_database_read" do
251 request_capped_index() { |size| (size * 3) + 1 }
252 assert_response :success
253 assert_equal(3, json_response["items"].size)
254 assert_equal(3, json_response["limit"])
255 assert_equal(201, json_response["items_available"])
258 test "max_index_database_read does not interfere with limit" do
259 request_capped_index(limit: 5) { |size| size * 20 }
260 assert_response :success
261 assert_equal(5, json_response["items"].size)
262 assert_equal(5, json_response["limit"])
263 assert_equal(201, json_response["items_available"])
266 test "max_index_database_read does not interfere with order" do
267 request_capped_index(select: %w(uuid manifest_text name),
268 order: "name DESC") { |size| (size * 11) + 1 }
269 assert_response :success
270 assert_equal(11, json_response["items"].size)
271 assert_empty(json_response["items"].reject do |coll|
272 coll["name"] =~ /^Collection_9/
274 assert_equal(11, json_response["limit"])
275 assert_equal(201, json_response["items_available"])
278 test "admin can create collection with unsigned manifest" do
279 authorize_with :admin
281 manifest_text: <<-EOS
282 . d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo.txt
283 . acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar.txt
284 . acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar.txt
285 ./baz acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar.txt
288 test_collection[:portable_data_hash] =
289 Digest::MD5.hexdigest(test_collection[:manifest_text]) +
291 test_collection[:manifest_text].length.to_s
293 # post :create will modify test_collection in place, so we save a copy first.
294 # Hash.deep_dup is not sufficient as it preserves references of strings (??!?)
295 post_collection = Marshal.load(Marshal.dump(test_collection))
296 post :create, params: {
297 collection: post_collection
300 assert_response :success
301 assert_nil assigns(:objects)
303 response_collection = assigns(:object)
305 stored_collection = Collection.select([:uuid, :portable_data_hash, :manifest_text]).
306 where(portable_data_hash: response_collection['portable_data_hash']).first
308 assert_equal test_collection[:portable_data_hash], stored_collection['portable_data_hash']
310 # The manifest in the response will have had permission hints added.
311 # Remove any permission hints in the response before comparing it to the source.
312 stripped_manifest = stored_collection['manifest_text'].gsub(/\+A[A-Za-z0-9@_-]+/, '')
313 assert_equal test_collection[:manifest_text], stripped_manifest
315 # TBD: create action should add permission signatures to manifest_text in the response,
316 # and we need to check those permission signatures here.
319 [:admin, :active].each do |user|
320 test "#{user} can get collection using portable data hash" do
323 foo_collection = collections(:foo_file)
325 # Get foo_file using its portable data hash
327 id: foo_collection[:portable_data_hash]
329 assert_response :success
330 assert_not_nil assigns(:object)
331 resp = assigns(:object)
332 assert_equal foo_collection[:portable_data_hash], resp[:portable_data_hash]
333 assert_unsigned_manifest resp[:manifest_text]
335 # The manifest in the response will have had permission hints added.
336 # Remove any permission hints in the response before comparing it to the source.
337 stripped_manifest = resp[:manifest_text].gsub(/\+A[A-Za-z0-9@_-]+/, '')
338 assert_equal foo_collection[:manifest_text], stripped_manifest
342 test "create with owner_uuid set to owned group" do
343 permit_unsigned_manifests
344 authorize_with :active
345 manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
346 post :create, params: {
348 owner_uuid: 'zzzzz-j7d0g-rew6elm53kancon',
349 manifest_text: manifest_text,
350 portable_data_hash: "d30fe8ae534397864cb96c544f4cf102+47"
353 assert_response :success
354 resp = JSON.parse(@response.body)
355 assert_equal 'zzzzz-j7d0g-rew6elm53kancon', resp['owner_uuid']
358 test "create fails with duplicate name" do
359 permit_unsigned_manifests
360 authorize_with :admin
361 manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
362 post :create, params: {
364 owner_uuid: 'zzzzz-tpzed-000000000000000',
365 manifest_text: manifest_text,
366 portable_data_hash: "d30fe8ae534397864cb96c544f4cf102+47",
371 response_errors = json_response['errors']
372 assert_not_nil response_errors, 'Expected error in response'
373 assert(response_errors.first.include?('duplicate key'),
374 "Expected 'duplicate key' error in #{response_errors.first}")
377 [false, true].each do |ensure_unique_name|
378 test "create failure with duplicate name, ensure_unique_name #{ensure_unique_name}" do
379 authorize_with :active
380 post :create, params: {
382 owner_uuid: users(:active).uuid,
384 name: "this...............................................................................................................................................................................................................................................................name is too long"
386 ensure_unique_name: ensure_unique_name
389 # check the real error isn't masked by an
390 # ensure_unique_name-related error (#19698)
391 assert_match /value too long for type/, json_response['errors'][0]
395 [false, true].each do |unsigned|
396 test "create with duplicate name, ensure_unique_name, unsigned=#{unsigned}" do
397 permit_unsigned_manifests unsigned
398 authorize_with :active
399 manifest_text = ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:foo.txt\n"
401 manifest_text = Collection.sign_manifest_only_for_tests manifest_text, api_token(:active)
403 post :create, params: {
405 owner_uuid: users(:active).uuid,
406 manifest_text: manifest_text,
407 name: "owned_by_active"
409 ensure_unique_name: true
411 assert_response :success
412 assert_match /^owned_by_active \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
416 test "create with owner_uuid set to group i can_manage" do
417 permit_unsigned_manifests
418 authorize_with :active
419 manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
420 post :create, params: {
422 owner_uuid: groups(:active_user_has_can_manage).uuid,
423 manifest_text: manifest_text,
424 portable_data_hash: "d30fe8ae534397864cb96c544f4cf102+47"
427 assert_response :success
428 resp = JSON.parse(@response.body)
429 assert_equal groups(:active_user_has_can_manage).uuid, resp['owner_uuid']
432 test "create with owner_uuid fails on group with only can_read permission" do
433 permit_unsigned_manifests
434 authorize_with :active
435 manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
436 post :create, params: {
438 owner_uuid: groups(:all_users).uuid,
439 manifest_text: manifest_text,
440 portable_data_hash: "d30fe8ae534397864cb96c544f4cf102+47"
446 test "create with owner_uuid fails on group with no permission" do
447 permit_unsigned_manifests
448 authorize_with :active
449 manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
450 post :create, params: {
452 owner_uuid: groups(:public).uuid,
453 manifest_text: manifest_text,
454 portable_data_hash: "d30fe8ae534397864cb96c544f4cf102+47"
460 test "admin create with owner_uuid set to group with no permission" do
461 permit_unsigned_manifests
462 authorize_with :admin
463 manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
464 post :create, params: {
466 owner_uuid: 'zzzzz-j7d0g-it30l961gq3t0oi',
467 manifest_text: manifest_text,
468 portable_data_hash: "d30fe8ae534397864cb96c544f4cf102+47"
471 assert_response :success
474 test "should create with collection passed as json" do
475 permit_unsigned_manifests
476 authorize_with :active
477 post :create, params: {
480 "manifest_text":". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n",\
481 "portable_data_hash":"d30fe8ae534397864cb96c544f4cf102+47"\
485 assert_response :success
488 test "should fail to create with checksum mismatch" do
489 permit_unsigned_manifests
490 authorize_with :active
491 post :create, params: {
494 "manifest_text":". d41d8cd98f00b204e9800998ecf8427e 0:0:bar.txt\n",\
495 "portable_data_hash":"d30fe8ae534397864cb96c544f4cf102+47"\
502 test "collection UUID is normalized when created" do
503 permit_unsigned_manifests
504 authorize_with :active
505 post :create, params: {
507 manifest_text: ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n",
508 portable_data_hash: "d30fe8ae534397864cb96c544f4cf102+47+Khint+Xhint+Zhint"
511 assert_response :success
512 assert_not_nil assigns(:object)
513 resp = JSON.parse(@response.body)
514 assert_equal "d30fe8ae534397864cb96c544f4cf102+47", resp['portable_data_hash']
517 test "get full provenance for baz file" do
518 authorize_with :active
519 get :provenance, params: {id: 'ea10d51bcf88862dbcc36eb292017dfd+45'}
520 assert_response :success
521 resp = JSON.parse(@response.body)
522 assert_not_nil resp['ea10d51bcf88862dbcc36eb292017dfd+45'] # baz
523 assert_not_nil resp['fa7aeb5140e2848d39b416daeef4ffc5+45'] # bar
524 assert_not_nil resp['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'] # foo
525 assert_not_nil resp['zzzzz-8i9sb-cjs4pklxxjykyuq'] # bar->baz
526 assert_not_nil resp['zzzzz-8i9sb-aceg2bnq7jt7kon'] # foo->bar
529 test "get no provenance for foo file" do
530 # spectator user cannot even see baz collection
531 authorize_with :spectator
532 get :provenance, params: {id: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'}
536 test "get partial provenance for baz file" do
537 # spectator user can see bar->baz job, but not foo->bar job
538 authorize_with :spectator
539 get :provenance, params: {id: 'ea10d51bcf88862dbcc36eb292017dfd+45'}
540 assert_response :success
541 resp = JSON.parse(@response.body)
542 assert_not_nil resp['ea10d51bcf88862dbcc36eb292017dfd+45'] # baz
543 assert_not_nil resp['fa7aeb5140e2848d39b416daeef4ffc5+45'] # bar
544 assert_not_nil resp['zzzzz-8i9sb-cjs4pklxxjykyuq'] # bar->baz
545 assert_nil resp['zzzzz-8i9sb-aceg2bnq7jt7kon'] # foo->bar
546 assert_nil resp['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'] # foo
549 test "search collections with 'any' operator" do
550 expect_pdh = collections(:docker_image).portable_data_hash
551 authorize_with :active
552 get :index, params: {
553 where: { any: ['contains', expect_pdh[5..25]] }
555 assert_response :success
556 found = assigns(:objects)
557 assert_equal 1, found.count
558 assert_equal expect_pdh, found.first.portable_data_hash
561 [false, true].each do |permit_unsigned|
562 test "create collection with signed manifest, permit_unsigned=#{permit_unsigned}" do
563 permit_unsigned_manifests permit_unsigned
564 authorize_with :active
566 d41d8cd98f00b204e9800998ecf8427e+0
567 acbd18db4cc2f85cedef654fccc4a4d8+3
568 ea10d51bcf88862dbcc36eb292017dfd+45)
570 unsigned_manifest = locators.map { |loc|
571 ". " + loc + " 0:0:foo.txt\n"
573 manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest) +
575 unsigned_manifest.length.to_s
577 # Build a manifest with both signed and unsigned locators.
579 key: Rails.configuration.Collections.BlobSigningKey,
580 api_token: api_token(:active),
582 signed_locators = locators.collect do |x|
583 Blob.sign_locator x, signing_opts
586 # Leave a non-empty blob unsigned.
587 signed_locators[1] = locators[1]
589 # Leave the empty blob unsigned. This should still be allowed.
590 signed_locators[0] = locators[0]
593 ". " + signed_locators[0] + " 0:0:foo.txt\n" +
594 ". " + signed_locators[1] + " 0:0:foo.txt\n" +
595 ". " + signed_locators[2] + " 0:0:foo.txt\n"
597 post :create, params: {
599 manifest_text: signed_manifest,
600 portable_data_hash: manifest_uuid,
603 assert_response :success
604 assert_not_nil assigns(:object)
605 resp = JSON.parse(@response.body)
606 assert_equal manifest_uuid, resp['portable_data_hash']
607 # All of the signatures in the output must be valid.
608 resp['manifest_text'].lines.each do |entry|
609 m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
610 if m && m[0].index('+A')
611 assert Blob.verify_signature m[0], signing_opts
617 test "create collection with signed manifest and explicit TTL" do
618 authorize_with :active
620 d41d8cd98f00b204e9800998ecf8427e+0
621 acbd18db4cc2f85cedef654fccc4a4d8+3
622 ea10d51bcf88862dbcc36eb292017dfd+45)
624 unsigned_manifest = locators.map { |loc|
625 ". " + loc + " 0:0:foo.txt\n"
627 manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest) +
629 unsigned_manifest.length.to_s
631 # build a manifest with both signed and unsigned locators.
632 # TODO(twp): in phase 4, all locators will need to be signed, so
633 # this test should break and will need to be rewritten. Issue #2755.
635 key: Rails.configuration.Collections.BlobSigningKey,
636 api_token: api_token(:active),
640 ". " + locators[0] + " 0:0:foo.txt\n" +
641 ". " + Blob.sign_locator(locators[1], signing_opts) + " 0:0:foo.txt\n" +
642 ". " + Blob.sign_locator(locators[2], signing_opts) + " 0:0:foo.txt\n"
644 post :create, params: {
646 manifest_text: signed_manifest,
647 portable_data_hash: manifest_uuid,
650 assert_response :success
651 assert_not_nil assigns(:object)
652 resp = JSON.parse(@response.body)
653 assert_equal manifest_uuid, resp['portable_data_hash']
654 # All of the signatures in the output must be valid.
655 resp['manifest_text'].lines.each do |entry|
656 m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
657 if m && m[0].index('+A')
658 assert Blob.verify_signature m[0], signing_opts
663 test "create fails with invalid signature" do
664 authorize_with :active
666 key: Rails.configuration.Collections.BlobSigningKey,
667 api_token: api_token(:active),
670 # Generate a locator with a bad signature.
671 unsigned_locator = "acbd18db4cc2f85cedef654fccc4a4d8+3"
672 bad_locator = unsigned_locator + "+Affffffffffffffffffffffffffffffffffffffff@ffffffff"
673 assert !Blob.verify_signature(bad_locator, signing_opts)
675 # Creating a collection with this locator should
676 # produce 403 Permission denied.
677 unsigned_manifest = ". #{unsigned_locator} 0:0:foo.txt\n"
678 manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest) +
680 unsigned_manifest.length.to_s
682 bad_manifest = ". #{bad_locator} 0:0:foo.txt\n"
683 post :create, params: {
685 manifest_text: bad_manifest,
686 portable_data_hash: manifest_uuid
693 test "create fails with uuid of signed manifest" do
694 authorize_with :active
696 key: Rails.configuration.Collections.BlobSigningKey,
697 api_token: api_token(:active),
700 unsigned_locator = "d41d8cd98f00b204e9800998ecf8427e+0"
701 signed_locator = Blob.sign_locator(unsigned_locator, signing_opts)
702 signed_manifest = ". #{signed_locator} 0:0:foo.txt\n"
703 manifest_uuid = Digest::MD5.hexdigest(signed_manifest) +
705 signed_manifest.length.to_s
707 post :create, params: {
709 manifest_text: signed_manifest,
710 portable_data_hash: manifest_uuid
717 test "reject manifest with unsigned block as stream name" do
718 authorize_with :active
719 post :create, params: {
721 manifest_text: "00000000000000000000000000000000+1234 d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo.txt\n"
724 assert_includes [422, 403], response.code.to_i
727 test "multiple locators per line" do
728 permit_unsigned_manifests
729 authorize_with :active
731 d41d8cd98f00b204e9800998ecf8427e+0
732 acbd18db4cc2f85cedef654fccc4a4d8+3
733 ea10d51bcf88862dbcc36eb292017dfd+45)
735 manifest_text = [".", *locators, "0:0:foo.txt\n"].join(" ")
736 manifest_uuid = Digest::MD5.hexdigest(manifest_text) +
738 manifest_text.length.to_s
741 manifest_text: manifest_text,
742 portable_data_hash: manifest_uuid,
744 post_collection = Marshal.load(Marshal.dump(test_collection))
745 post :create, params: {
746 collection: post_collection
748 assert_response :success
749 assert_not_nil assigns(:object)
750 resp = JSON.parse(@response.body)
751 assert_equal manifest_uuid, resp['portable_data_hash']
753 # The manifest in the response will have had permission hints added.
754 # Remove any permission hints in the response before comparing it to the source.
755 stripped_manifest = resp['manifest_text'].gsub(/\+A[A-Za-z0-9@_-]+/, '')
756 assert_equal manifest_text, stripped_manifest
759 test 'Reject manifest with unsigned blob' do
760 permit_unsigned_manifests false
761 authorize_with :active
762 unsigned_manifest = ". 0cc175b9c0f1b6a831c399e269772661+1 0:1:a.txt\n"
763 manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest)
764 post :create, params: {
766 manifest_text: unsigned_manifest,
767 portable_data_hash: manifest_uuid,
771 "Creating a collection with unsigned blobs should respond 403"
772 assert_empty Collection.where('uuid like ?', manifest_uuid+'%'),
773 "Collection should not exist in database after failed create"
776 test 'List expired collection returns empty list' do
777 authorize_with :active
778 get :index, params: {
779 where: {name: 'expired_collection'},
781 assert_response :success
782 found = assigns(:objects)
783 assert_equal 0, found.count
786 test 'Show expired collection returns 404' do
787 authorize_with :active
789 id: 'zzzzz-4zz18-mto52zx1s7sn3ih',
794 test 'Update expired collection returns 404' do
795 authorize_with :active
796 post :update, params: {
797 id: 'zzzzz-4zz18-mto52zx1s7sn3ih',
799 name: "still expired"
805 test 'List collection with future expiration time succeeds' do
806 authorize_with :active
807 get :index, params: {
808 where: {name: 'collection_expires_in_future'},
810 found = assigns(:objects)
811 assert_equal 1, found.count
815 test 'Show collection with future expiration time succeeds' do
816 authorize_with :active
818 id: 'zzzzz-4zz18-padkqo7yb8d9i3j',
820 assert_response :success
823 test 'Update collection with future expiration time succeeds' do
824 authorize_with :active
825 post :update, params: {
826 id: 'zzzzz-4zz18-padkqo7yb8d9i3j',
828 name: "still not expired"
831 assert_response :success
834 test "get collection and verify that file_names is not included" do
835 authorize_with :active
836 get :show, params: {id: collections(:foo_file).uuid}
837 assert_response :success
838 assert_equal collections(:foo_file).uuid, json_response['uuid']
839 assert_nil json_response['file_names']
840 assert json_response['manifest_text']
846 ].each do |description_size, expected_response|
847 # Descriptions are not part of search indexes. Skip until
848 # full-text search is implemented, at which point replace with a
849 # search in description.
850 skip "create collection with description size #{description_size}
851 and expect response #{expected_response}" do
852 authorize_with :active
854 description = 'here is a collection with a very large description'
855 while description.length < description_size
856 description = description + description
859 post :create, params: {
861 manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo.txt\n",
862 description: description,
866 assert_response expected_response
870 [1, 5, nil].each do |ask|
871 test "Set replication_desired=#{ask.inspect}" do
872 Rails.configuration.Collections.DefaultReplication = 2
873 authorize_with :active
874 put :update, params: {
875 id: collections(:replication_undesired_unconfirmed).uuid,
877 replication_desired: ask,
880 assert_response :success
881 assert_equal ask, json_response['replication_desired']
885 test "get collection with properties" do
886 authorize_with :active
887 get :show, params: {id: collections(:collection_with_one_property).uuid}
888 assert_response :success
889 assert_not_nil json_response['uuid']
890 assert_equal 'value1', json_response['properties']['property1']
894 {'property_1' => 'value_1'},
895 "{\"property_1\":\"value_1\"}",
897 test "create collection with valid properties param #{p.inspect}" do
898 authorize_with :active
899 manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
900 post :create, params: {
902 manifest_text: manifest_text,
903 portable_data_hash: "d30fe8ae534397864cb96c544f4cf102+47",
907 assert_response :success
908 assert_not_nil json_response['uuid']
909 assert_equal Hash, json_response['properties'].class, 'Collection properties attribute should be of type hash'
910 assert_equal 'value_1', json_response['properties']['property_1']
919 '["json", "encoded", "array"]',
921 test "create collection with non-valid properties param #{p.inspect}" do
922 authorize_with :active
923 post :create, params: {
925 name: "test collection with non-valid properties param '#{p.inspect}'",
931 response_errors = json_response['errors']
932 assert_not_nil response_errors, 'Expected error in response'
937 [". d41d8cd98f00b204e9800998ecf8427e 0:34:foo.txt\n", 1, 34],
938 [". d41d8cd98f00b204e9800998ecf8427e 0:34:foo.txt 0:30:foo.txt 0:30:foo1.txt 0:30:foo2.txt 0:30:foo3.txt 0:30:foo4.txt\n", 5, 184],
939 [". d41d8cd98f00b204e9800998ecf8427e 0:0:.\n", 0, 0]
940 ].each do |manifest, count, size|
941 test "create collection with valid manifest #{manifest} and expect file stats" do
942 authorize_with :active
943 post :create, params: {
945 manifest_text: manifest
949 assert_equal count, json_response['file_count']
950 assert_equal size, json_response['file_size_total']
954 test "update collection manifest and expect new file stats" do
955 authorize_with :active
956 post :update, params: {
957 id: collections(:collection_owned_by_active_with_file_stats).uuid,
959 manifest_text: ". d41d8cd98f00b204e9800998ecf8427e 0:34:foo.txt\n"
963 assert_equal 1, json_response['file_count']
964 assert_equal 34, json_response['file_size_total']
969 ['file_size_total', 34]
970 ].each do |attribute, val|
971 test "create collection with #{attribute} and expect overwrite" do
972 authorize_with :active
973 post :create, params: {
975 manifest_text: ". d41d8cd98f00b204e9800998ecf8427e 0:34:foo.txt\n",
980 assert_equal val, json_response[attribute]
986 ['file_size_total', 3]
987 ].each do |attribute, val|
988 test "update collection with #{attribute} and expect ignore" do
989 authorize_with :active
990 post :update, params: {
991 id: collections(:collection_owned_by_active_with_file_stats).uuid,
997 assert_equal val, json_response[attribute]
1003 ['file_size_total', 34]
1004 ].each do |attribute, val|
1005 test "update collection with #{attribute} and manifest and expect manifest values" do
1006 authorize_with :active
1007 post :update, params: {
1008 id: collections(:collection_owned_by_active_with_file_stats).uuid,
1010 manifest_text: ". d41d8cd98f00b204e9800998ecf8427e 0:34:foo.txt\n",
1015 assert_equal val, json_response[attribute]
1021 ". d41d8cd98f00b204e9800998ecf8427e foo.txt",
1022 "d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt",
1023 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt",
1024 ].each do |manifest_text|
1025 test "create collection with invalid manifest #{manifest_text} and expect error" do
1026 authorize_with :active
1027 post :create, params: {
1029 manifest_text: manifest_text,
1030 portable_data_hash: "d41d8cd98f00b204e9800998ecf8427e+0"
1034 response_errors = json_response['errors']
1035 assert_not_nil response_errors, 'Expected error in response'
1036 assert(response_errors.first.include?('Invalid manifest'),
1037 "Expected 'Invalid manifest' error in #{response_errors.first}")
1042 [nil, "d41d8cd98f00b204e9800998ecf8427e+0"],
1043 ["", "d41d8cd98f00b204e9800998ecf8427e+0"],
1044 [". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n", "d30fe8ae534397864cb96c544f4cf102+47"],
1045 ].each do |manifest_text, pdh|
1046 test "create collection with valid manifest #{manifest_text.inspect} and expect success" do
1047 authorize_with :active
1048 post :create, params: {
1050 manifest_text: manifest_text,
1051 portable_data_hash: pdh
1060 ". d41d8cd98f00b204e9800998ecf8427e foo.txt",
1061 "d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt",
1062 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt",
1063 ].each do |manifest_text|
1064 test "update collection with invalid manifest #{manifest_text} and expect error" do
1065 authorize_with :active
1066 post :update, params: {
1067 id: 'zzzzz-4zz18-bv31uwvy3neko21',
1069 manifest_text: manifest_text,
1073 response_errors = json_response['errors']
1074 assert_not_nil response_errors, 'Expected error in response'
1075 assert(response_errors.first.include?('Invalid manifest'),
1076 "Expected 'Invalid manifest' error in #{response_errors.first}")
1083 ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n",
1084 ].each do |manifest_text|
1085 test "update collection with valid manifest #{manifest_text.inspect} and expect success" do
1086 authorize_with :active
1087 post :update, params: {
1088 id: 'zzzzz-4zz18-bv31uwvy3neko21',
1090 manifest_text: manifest_text,
1097 [true, false].each do |include_trash|
1098 test "get trashed collection with include_trash=#{include_trash}" do
1099 uuid = 'zzzzz-4zz18-mto52zx1s7sn3ih' # expired_collection
1100 authorize_with :active
1101 get :show, params: {
1103 include_trash: include_trash,
1113 [:admin, :active].each do |user|
1114 test "get trashed collection via filters and #{user} user without including its past versions" do
1115 uuid = 'zzzzz-4zz18-mto52zx1s7sn3ih' # expired_collection
1117 get :index, params: {
1118 filters: [["current_version_uuid", "=", uuid]],
1119 include_trash: true,
1122 # Only the current version is returned
1123 assert_equal 1, json_response["items"].size
1127 [:admin, :active].each do |user|
1128 test "get trashed collection via filters and #{user} user, including its past versions" do
1129 uuid = 'zzzzz-4zz18-mto52zx1s7sn3ih' # expired_collection
1130 authorize_with :admin
1131 get :index, params: {
1132 filters: [["current_version_uuid", "=", uuid]],
1133 include_trash: true,
1134 include_old_versions: true,
1137 # Both current & past version are returned
1138 assert_equal 2, json_response["items"].size
1142 test "trash collection also trash its past versions" do
1143 uuid = collections(:collection_owned_by_active).uuid
1144 authorize_with :active
1145 versions = Collection.where(current_version_uuid: uuid)
1146 assert_equal 2, versions.size
1147 versions.each do |col|
1148 refute col.is_trashed
1150 post :trash, params: {
1154 versions = Collection.where(current_version_uuid: uuid)
1155 assert_equal 2, versions.size
1156 versions.each do |col|
1157 assert col.is_trashed
1161 test 'get trashed collection without include_trash' do
1162 uuid = 'zzzzz-4zz18-mto52zx1s7sn3ih' # expired_collection
1163 authorize_with :active
1164 get :show, params: {
1170 test 'trash collection using http DELETE verb' do
1171 uuid = collections(:collection_owned_by_active).uuid
1172 authorize_with :active
1173 delete :destroy, params: {
1177 c = Collection.find_by_uuid(uuid)
1178 assert_operator c.trash_at, :<, db_current_time
1179 assert_equal c.delete_at, c.trash_at + Rails.configuration.Collections.BlobSigningTTL
1182 test 'delete long-trashed collection immediately using http DELETE verb' do
1183 uuid = 'zzzzz-4zz18-mto52zx1s7sn3ih' # expired_collection
1184 authorize_with :active
1185 delete :destroy, params: {
1189 c = Collection.find_by_uuid(uuid)
1190 assert_operator c.trash_at, :<, db_current_time
1191 assert_operator c.delete_at, :<, db_current_time
1194 ['zzzzz-4zz18-mto52zx1s7sn3ih', # expired_collection
1195 :empty_collection_name_in_active_user_home_project,
1197 test "trash collection #{fixture} via trash action with grace period" do
1198 if fixture.is_a? String
1201 uuid = collections(fixture).uuid
1203 authorize_with :active
1204 time_before_trashing = db_current_time
1205 post :trash, params: {
1209 c = Collection.find_by_uuid(uuid)
1210 assert_operator c.trash_at, :<, db_current_time
1211 assert_operator c.delete_at, :>=, time_before_trashing + Rails.configuration.Collections.DefaultTrashLifetime
1215 test 'untrash a trashed collection' do
1216 authorize_with :active
1217 post :untrash, params: {
1218 id: collections(:expired_collection).uuid,
1221 assert_equal false, json_response['is_trashed']
1222 assert_nil json_response['trash_at']
1225 test 'untrash a trashed collection by assigning nil to trash_at' do
1226 authorize_with :active
1227 post :update, params: {
1228 id: collections(:expired_collection).uuid,
1232 include_trash: true,
1235 assert_equal false, json_response['is_trashed']
1236 assert_nil json_response['trash_at']
1239 test 'untrash error on not trashed collection' do
1240 authorize_with :active
1241 post :untrash, params: {
1242 id: collections(:collection_owned_by_active).uuid,
1247 [:active, :admin].each do |user|
1248 test "get trashed collections as #{user}" do
1250 get :index, params: {
1251 filters: [["is_trashed", "=", true]],
1252 include_trash: true,
1254 assert_response :success
1257 json_response["items"].each do |coll|
1258 items << coll['uuid']
1261 assert_includes(items, collections('unique_expired_collection')['uuid'])
1263 assert_includes(items, collections('unique_expired_collection2')['uuid'])
1265 assert_not_includes(items, collections('unique_expired_collection2')['uuid'])
1270 test 'untrash collection with same name as another with no ensure unique name' do
1271 authorize_with :active
1272 post :untrash, params: {
1273 id: collections(:trashed_collection_to_test_name_conflict_on_untrash).uuid,
1278 test 'untrash collection with same name as another with ensure unique name' do
1279 authorize_with :active
1280 post :untrash, params: {
1281 id: collections(:trashed_collection_to_test_name_conflict_on_untrash).uuid,
1282 ensure_unique_name: true
1285 assert_equal false, json_response['is_trashed']
1286 assert_nil json_response['trash_at']
1287 assert_nil json_response['delete_at']
1288 assert_match /^same name for trashed and persisted collections \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
1291 test 'cannot show collection in trashed subproject' do
1292 authorize_with :active
1293 get :show, params: {
1294 id: collections(:collection_in_trashed_subproject).uuid,
1300 test 'can show collection in untrashed subproject' do
1301 authorize_with :active
1302 Group.find_by_uuid(groups(:trashed_project).uuid).update! is_trashed: false
1303 get :show, params: {
1304 id: collections(:collection_in_trashed_subproject).uuid,
1307 assert_response :success
1310 test 'cannot index collection in trashed subproject' do
1311 authorize_with :active
1312 get :index, params: { limit: 1000 }
1313 assert_response :success
1314 item_uuids = json_response['items'].map do |item|
1317 assert_not_includes(item_uuids, collections(:collection_in_trashed_subproject).uuid)
1320 test 'can index collection in untrashed subproject' do
1321 authorize_with :active
1322 Group.find_by_uuid(groups(:trashed_project).uuid).update! is_trashed: false
1323 get :index, params: { limit: 1000 }
1324 assert_response :success
1325 item_uuids = json_response['items'].map do |item|
1328 assert_includes(item_uuids, collections(:collection_in_trashed_subproject).uuid)
1331 test 'can index trashed subproject collection with include_trash' do
1332 authorize_with :active
1333 get :index, params: {
1334 include_trash: true,
1337 assert_response :success
1338 item_uuids = json_response['items'].map do |item|
1341 assert_includes(item_uuids, collections(:collection_in_trashed_subproject).uuid)
1344 test 'can get collection with past versions' do
1345 authorize_with :active
1346 get :index, params: {
1347 filters: [['current_version_uuid','=',collections(:collection_owned_by_active).uuid]],
1348 include_old_versions: true
1350 assert_response :success
1351 assert_equal 2, assigns(:objects).length
1352 assert_equal 2, json_response['items_available']
1353 assert_equal 2, json_response['items'].count
1354 json_response['items'].each do |c|
1355 assert_equal collections(:collection_owned_by_active).uuid,
1356 c['current_version_uuid'],
1357 'response includes a version from a different collection'
1361 test 'can get old version collection by uuid' do
1362 authorize_with :active
1363 get :show, params: {
1364 id: collections(:collection_owned_by_active_past_version_1).uuid,
1366 assert_response :success
1367 assert_equal collections(:collection_owned_by_active_past_version_1).name,
1368 json_response['name']
1371 test 'can get old version collection by PDH' do
1372 authorize_with :active
1373 get :show, params: {
1374 id: collections(:collection_owned_by_active_past_version_1).portable_data_hash,
1376 assert_response :success
1377 assert_equal collections(:collection_owned_by_active_past_version_1).portable_data_hash,
1378 json_response['portable_data_hash']
1381 test 'version and current_version_uuid are ignored at creation time' do
1382 permit_unsigned_manifests
1383 authorize_with :active
1384 manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
1385 post :create, params: {
1387 name: 'Test collection',
1389 current_version_uuid: collections(:collection_owned_by_active).uuid,
1390 manifest_text: manifest_text,
1393 assert_response :success
1394 resp = JSON.parse(@response.body)
1395 assert_equal 1, resp['version']
1396 assert_equal resp['uuid'], resp['current_version_uuid']
1399 test "update collection with versioning enabled" do
1400 Rails.configuration.Collections.CollectionVersioning = true
1401 Rails.configuration.Collections.PreserveVersionIfIdle = 1 # 1 second
1403 col = collections(:collection_owned_by_active)
1404 assert_equal 2, col.version
1405 assert col.modified_at < Time.now - 1.second
1407 token = api_client_authorizations(:active).v2token
1408 signed = Blob.sign_locator(
1409 'acbd18db4cc2f85cedef654fccc4a4d8+3',
1410 key: Rails.configuration.Collections.BlobSigningKey,
1412 authorize_with_token token
1413 put :update, params: {
1416 manifest_text: ". #{signed} 0:3:foo.txt\n",
1419 assert_response :success
1420 assert_equal 3, json_response['version']
1423 test "delete collection with versioning enabled" do
1424 Rails.configuration.Collections.CollectionVersioning = true
1425 Rails.configuration.Collections.PreserveVersionIfIdle = 1 # 1 second
1427 col = collections(:collection_owned_by_active)
1428 assert_equal 2, col.version
1429 assert col.modified_at < Time.now - 1.second
1431 authorize_with(:active)
1432 post :trash, params: {
1435 assert_response :success
1436 assert_equal col.version, json_response['version'], 'Trashing a collection should not create a new version'
1443 ['=', :==]].each do |op, rubyop|
1444 test "filter collections by replication_desired #{op} replication_confirmed" do
1445 authorize_with(:active)
1446 get :index, params: {
1447 filters: [["(replication_desired #{op} replication_confirmed)", "=", true]],
1449 assert_response :success
1450 json_response["items"].each do |c|
1451 assert_operator(c["replication_desired"], rubyop, c["replication_confirmed"])
1456 ["(replication_desired < bogus)",
1457 "replication_desired < replication_confirmed",
1458 "(replication_desired < replication_confirmed",
1459 "(replication_desired ! replication_confirmed)",
1460 "(replication_desired <)",
1461 "(replication_desired < manifest_text)",
1462 "(manifest_text < manifest_text)", # currently only numeric attrs are supported
1463 "(replication_desired < 2)", # currently only attrs are supported, not literals
1466 test "invalid filter expression #{expr}" do
1467 authorize_with(:active)
1468 get :index, params: {
1469 filters: [[expr, "=", true]],
1475 test "invalid op/arg with filter expression" do
1476 authorize_with(:active)
1477 get :index, params: {
1478 filters: [["replication_desired < replication_confirmed", "!=", false]],
1483 ["storage_classes_desired", "storage_classes_confirmed"].each do |attr|
1484 test "filter collections by #{attr}" do
1485 authorize_with(:active)
1486 get :index, params: {
1487 filters: [[attr, "=", '["default"]']]
1489 assert_response :success
1490 assert_not_equal 0, json_response["items"].length
1491 json_response["items"].each do |c|
1492 assert_equal ["default"], c[attr]
1497 test "select param is respected in 'show' response" do
1498 authorize_with :active
1499 get :show, params: {
1500 id: collections(:collection_owned_by_active).uuid,
1503 assert_response :success
1504 assert_raises ActiveModel::MissingAttributeError do
1505 assigns(:object).manifest_text
1507 assert_nil json_response["manifest_text"]
1508 assert_nil json_response["properties"]
1509 assert_equal collections(:collection_owned_by_active).name, json_response["name"]
1512 test "select param is respected in 'update' response" do
1513 authorize_with :active
1514 post :update, params: {
1515 id: collections(:collection_owned_by_active).uuid,
1517 manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foobar.txt\n",
1521 assert_response :success
1522 assert_nil json_response["manifest_text"]
1523 assert_nil json_response["properties"]
1524 assert_equal collections(:collection_owned_by_active).name, json_response["name"]
1529 ["is_trashed", "trash_at"],
1530 ["is_trashed", "trash_at", "portable_data_hash"],
1531 ["portable_data_hash"],
1532 ["portable_data_hash", "manifest_text"],
1534 test "select=#{select.inspect} param is respected in 'get by pdh' response" do
1535 authorize_with :active
1536 get :show, params: {
1537 id: collections(:collection_owned_by_active).portable_data_hash,
1540 assert_response :success
1541 if !select || select.index("manifest_text")
1542 assert_not_nil json_response["manifest_text"]
1544 assert_nil json_response["manifest_text"]