Merge branch 'master' into 7490-datamanager-dont-die-return-error
[arvados.git] / apps / workbench / test / controllers / collections_controller_test.rb
1 require 'test_helper'
2
3 class CollectionsControllerTest < ActionController::TestCase
4   # These tests don't do state-changing API calls. Save some time by
5   # skipping the database reset.
6   reset_api_fixtures :after_each_test, false
7   reset_api_fixtures :after_suite, true
8
9   include PipelineInstancesHelper
10
11   NONEXISTENT_COLLECTION = "ffffffffffffffffffffffffffffffff+0"
12
13   def config_anonymous enable
14     Rails.configuration.anonymous_user_token =
15       if enable
16         api_fixture('api_client_authorizations')['anonymous']['api_token']
17       else
18         false
19       end
20   end
21
22   def stub_file_content
23     # For the duration of the current test case, stub file download
24     # content with a randomized (but recognizable) string. Return the
25     # string, the test case can use it in assertions.
26     txt = 'the quick brown fox ' + rand(2**32).to_s
27     @controller.stubs(:file_enumerator).returns([txt])
28     txt
29   end
30
31   def collection_params(collection_name, file_name=nil)
32     uuid = api_fixture('collections')[collection_name.to_s]['uuid']
33     params = {uuid: uuid, id: uuid}
34     params[:file] = file_name if file_name
35     params
36   end
37
38   def assert_hash_includes(actual_hash, expected_hash, msg=nil)
39     expected_hash.each do |key, value|
40       assert_equal(value, actual_hash[key], msg)
41     end
42   end
43
44   def assert_no_session
45     assert_hash_includes(session, {arvados_api_token: nil},
46                          "session includes unexpected API token")
47   end
48
49   def assert_session_for_auth(client_auth)
50     api_token =
51       api_fixture('api_client_authorizations')[client_auth.to_s]['api_token']
52     assert_hash_includes(session, {arvados_api_token: api_token},
53                          "session token does not belong to #{client_auth}")
54   end
55
56   def show_collection(params, session={}, response=:success)
57     params = collection_params(params) if not params.is_a? Hash
58     session = session_for(session) if not session.is_a? Hash
59     get(:show, params, session)
60     assert_response response
61   end
62
63   test "viewing a collection" do
64     show_collection(:foo_file, :active)
65     assert_equal([['.', 'foo', 3]], assigns(:object).files)
66   end
67
68   test "viewing a collection with spaces in filename" do
69     show_collection(:w_a_z_file, :active)
70     assert_equal([['.', 'w a z', 5]], assigns(:object).files)
71   end
72
73   test "download a file with spaces in filename" do
74     collection = api_fixture('collections')['w_a_z_file']
75     fakepipe = IO.popen(['echo', '-n', 'w a z'], 'rb')
76     IO.expects(:popen).with { |cmd, mode|
77       cmd.include? "#{collection['uuid']}/w a z"
78     }.returns(fakepipe)
79     get :show_file, {
80       uuid: collection['uuid'],
81       file: 'w a z'
82     }, session_for(:active)
83     assert_response :success
84     assert_equal 'w a z', response.body
85   end
86
87   test "viewing a collection fetches related projects" do
88     show_collection({id: api_fixture('collections')["foo_file"]['portable_data_hash']}, :active)
89     assert_includes(assigns(:same_pdh).map(&:owner_uuid),
90                     api_fixture('groups')['aproject']['uuid'],
91                     "controller did not find linked project")
92   end
93
94   test "viewing a collection fetches related permissions" do
95     show_collection(:bar_file, :active)
96     assert_includes(assigns(:permissions).map(&:uuid),
97                     api_fixture('links')['bar_file_readable_by_active']['uuid'],
98                     "controller did not find permission link")
99   end
100
101   test "viewing a collection fetches jobs that output it" do
102     show_collection(:bar_file, :active)
103     assert_includes(assigns(:output_of).map(&:uuid),
104                     api_fixture('jobs')['foobar']['uuid'],
105                     "controller did not find output job")
106   end
107
108   test "viewing a collection fetches jobs that logged it" do
109     show_collection(:baz_file, :active)
110     assert_includes(assigns(:log_of).map(&:uuid),
111                     api_fixture('jobs')['foobar']['uuid'],
112                     "controller did not find logger job")
113   end
114
115   test "viewing a collection fetches logs about it" do
116     show_collection(:foo_file, :active)
117     assert_includes(assigns(:logs).map(&:uuid),
118                     api_fixture('logs')['system_adds_foo_file']['uuid'],
119                     "controller did not find related log")
120   end
121
122   test "sharing auths available to admin" do
123     show_collection("collection_owned_by_active", "admin_trustedclient")
124     assert_not_nil assigns(:search_sharing)
125   end
126
127   test "sharing auths available to owner" do
128     show_collection("collection_owned_by_active", "active_trustedclient")
129     assert_not_nil assigns(:search_sharing)
130   end
131
132   test "sharing auths available to reader" do
133     show_collection("foo_collection_in_aproject",
134                     "project_viewer_trustedclient")
135     assert_not_nil assigns(:search_sharing)
136   end
137
138   test "viewing collection files with a reader token" do
139     params = collection_params(:foo_file)
140     params[:reader_token] = api_fixture("api_client_authorizations",
141                                         "active_all_collections", "api_token")
142     get(:show_file_links, params)
143     assert_response :success
144     assert_equal([['.', 'foo', 3]], assigns(:object).files)
145     assert_no_session
146   end
147
148   test "fetching collection file with reader token" do
149     expected = stub_file_content
150     params = collection_params(:foo_file, "foo")
151     params[:reader_token] = api_fixture("api_client_authorizations",
152                                         "active_all_collections", "api_token")
153     get(:show_file, params)
154     assert_response :success
155     assert_equal(expected, @response.body,
156                  "failed to fetch a Collection file with a reader token")
157     assert_no_session
158   end
159
160   test "reader token Collection links end with trailing slash" do
161     # Testing the fix for #2937.
162     session = session_for(:active_trustedclient)
163     post(:share, collection_params(:foo_file), session)
164     assert(@controller.download_link.ends_with? '/',
165            "Collection share link does not end with slash for wget")
166   end
167
168   test "getting a file from Keep" do
169     params = collection_params(:foo_file, 'foo')
170     sess = session_for(:active)
171     expect_content = stub_file_content
172     get(:show_file, params, sess)
173     assert_response :success
174     assert_equal(expect_content, @response.body,
175                  "failed to get a correct file from Keep")
176   end
177
178   test 'anonymous download' do
179     config_anonymous true
180     expect_content = stub_file_content
181     get :show_file, {
182       uuid: api_fixture('collections')['user_agreement_in_anonymously_accessible_project']['uuid'],
183       file: 'GNU_General_Public_License,_version_3.pdf',
184     }
185     assert_response :success
186     assert_equal expect_content, response.body
187   end
188
189   test "can't get a file from Keep without permission" do
190     params = collection_params(:foo_file, 'foo')
191     sess = session_for(:spectator)
192     get(:show_file, params, sess)
193     assert_response 404
194   end
195
196   test "trying to get a nonexistent file from Keep returns a 404" do
197     params = collection_params(:foo_file, 'gone')
198     sess = session_for(:admin)
199     get(:show_file, params, sess)
200     assert_response 404
201   end
202
203   test "getting a file from Keep with a good reader token" do
204     params = collection_params(:foo_file, 'foo')
205     read_token = api_fixture('api_client_authorizations')['active']['api_token']
206     params[:reader_token] = read_token
207     expect_content = stub_file_content
208     get(:show_file, params)
209     assert_response :success
210     assert_equal(expect_content, @response.body,
211                  "failed to get a correct file from Keep using a reader token")
212     assert_not_equal(read_token, session[:arvados_api_token],
213                      "using a reader token set the session's API token")
214   end
215
216   [false, true].each do |anon|
217     test "download a file using a reader token with insufficient scope, anon #{anon}" do
218       config_anonymous anon
219       params = collection_params(:foo_file, 'foo')
220       params[:reader_token] =
221         api_fixture('api_client_authorizations')['active_noscope']['api_token']
222       get(:show_file, params)
223       if anon
224         # Some files can be shown without a valid token, but not this one.
225         assert_response 404
226       else
227         # No files will ever be shown without a valid token. You
228         # should log in and try again.
229         assert_response :redirect
230       end
231     end
232   end
233
234   test "can get a file with an unpermissioned auth but in-scope reader token" do
235     params = collection_params(:foo_file, 'foo')
236     sess = session_for(:expired)
237     read_token = api_fixture('api_client_authorizations')['active']['api_token']
238     params[:reader_token] = read_token
239     expect_content = stub_file_content
240     get(:show_file, params, sess)
241     assert_response :success
242     assert_equal(expect_content, @response.body,
243                  "failed to get a correct file from Keep using a reader token")
244     assert_not_equal(read_token, session[:arvados_api_token],
245                      "using a reader token set the session's API token")
246   end
247
248   test "inactive user can retrieve user agreement" do
249     ua_collection = api_fixture('collections')['user_agreement']
250     # Here we don't test whether the agreement can be retrieved from
251     # Keep. We only test that show_file decides to send file content,
252     # so we use the file content stub.
253     stub_file_content
254     get :show_file, {
255       uuid: ua_collection['uuid'],
256       file: ua_collection['manifest_text'].match(/ \d+:\d+:(\S+)/)[1]
257     }, session_for(:inactive)
258     assert_nil(assigns(:unsigned_user_agreements),
259                "Did not skip check_user_agreements filter " +
260                "when showing the user agreement.")
261     assert_response :success
262   end
263
264   test "requesting nonexistent Collection returns 404" do
265     show_collection({uuid: NONEXISTENT_COLLECTION, id: NONEXISTENT_COLLECTION},
266                     :active, 404)
267   end
268
269   test "use a reasonable read buffer even if client requests a huge range" do
270     fakefiledata = mock
271     IO.expects(:popen).returns(fakefiledata)
272     fakefiledata.expects(:read).twice.with() do |length|
273       # Fail the test if read() is called with length>1MiB:
274       length < 2**20
275       ## Force the ActionController::Live thread to lose the race to
276       ## verify that @response.body.length actually waits for the
277       ## response (see below):
278       # sleep 3
279     end.returns("foo\n", nil)
280     fakefiledata.expects(:close)
281     foo_file = api_fixture('collections')['foo_file']
282     @request.headers['Range'] = 'bytes=0-4294967296/*'
283     get :show_file, {
284       uuid: foo_file['uuid'],
285       file: foo_file['manifest_text'].match(/ \d+:\d+:(\S+)/)[1]
286     }, session_for(:active)
287     # Wait for the whole response to arrive before deciding whether
288     # mocks' expectations were met. Otherwise, Mocha will fail the
289     # test depending on how slowly the ActionController::Live thread
290     # runs.
291     @response.body.length
292   end
293
294   test "show file in a subdirectory of a collection" do
295     params = collection_params(:collection_with_files_in_subdir, 'subdir2/subdir3/subdir4/file1_in_subdir4.txt')
296     expect_content = stub_file_content
297     get(:show_file, params, session_for(:user1_with_load))
298     assert_response :success
299     assert_equal(expect_content, @response.body, "failed to get a correct file from Keep")
300   end
301
302   test 'provenance graph' do
303     use_token 'admin'
304
305     obj = find_fixture Collection, "graph_test_collection3"
306
307     provenance = obj.provenance.stringify_keys
308
309     [obj[:portable_data_hash]].each do |k|
310       assert_not_nil provenance[k], "Expected key #{k} in provenance set"
311     end
312
313     prov_svg = ProvenanceHelper::create_provenance_graph(provenance, "provenance_svg",
314                                                          {:request => RequestDuck,
315                                                            :direction => :bottom_up,
316                                                            :combine_jobs => :script_only})
317
318     stage1 = find_fixture Job, "graph_stage1"
319     stage3 = find_fixture Job, "graph_stage3"
320     previous_job_run = find_fixture Job, "previous_job_run"
321
322     obj_id = obj.portable_data_hash.gsub('+', '\\\+')
323     stage1_out = stage1.output.gsub('+', '\\\+')
324     stage1_id = "#{stage1.script}_#{Digest::MD5.hexdigest(stage1[:script_parameters].to_json)}"
325     stage3_id = "#{stage3.script}_#{Digest::MD5.hexdigest(stage3[:script_parameters].to_json)}"
326
327     assert /#{obj_id}&#45;&gt;#{stage3_id}/.match(prov_svg)
328
329     assert /#{stage3_id}&#45;&gt;#{stage1_out}/.match(prov_svg)
330
331     assert /#{stage1_out}&#45;&gt;#{stage1_id}/.match(prov_svg)
332
333   end
334
335   test 'used_by graph' do
336     use_token 'admin'
337     obj = find_fixture Collection, "graph_test_collection1"
338
339     used_by = obj.used_by.stringify_keys
340
341     used_by_svg = ProvenanceHelper::create_provenance_graph(used_by, "used_by_svg",
342                                                             {:request => RequestDuck,
343                                                               :direction => :top_down,
344                                                               :combine_jobs => :script_only,
345                                                               :pdata_only => true})
346
347     stage2 = find_fixture Job, "graph_stage2"
348     stage3 = find_fixture Job, "graph_stage3"
349
350     stage2_id = "#{stage2.script}_#{Digest::MD5.hexdigest(stage2[:script_parameters].to_json)}"
351     stage3_id = "#{stage3.script}_#{Digest::MD5.hexdigest(stage3[:script_parameters].to_json)}"
352
353     obj_id = obj.portable_data_hash.gsub('+', '\\\+')
354     stage3_out = stage3.output.gsub('+', '\\\+')
355
356     assert /#{obj_id}&#45;&gt;#{stage2_id}/.match(used_by_svg)
357
358     assert /#{obj_id}&#45;&gt;#{stage3_id}/.match(used_by_svg)
359
360     assert /#{stage3_id}&#45;&gt;#{stage3_out}/.match(used_by_svg)
361
362     assert /#{stage3_id}&#45;&gt;#{stage3_out}/.match(used_by_svg)
363
364   end
365
366   test "view collection with empty properties" do
367     fixture_name = :collection_with_empty_properties
368     show_collection(fixture_name, :active)
369     assert_equal(api_fixture('collections')[fixture_name.to_s]['name'], assigns(:object).name)
370     assert_not_nil(assigns(:object).properties)
371     assert_empty(assigns(:object).properties)
372   end
373
374   test "view collection with one property" do
375     fixture_name = :collection_with_one_property
376     show_collection(fixture_name, :active)
377     fixture = api_fixture('collections')[fixture_name.to_s]
378     assert_equal(fixture['name'], assigns(:object).name)
379     assert_equal(fixture['properties'][0], assigns(:object).properties[0])
380   end
381
382   test "create collection with properties" do
383     post :create, {
384       collection: {
385         name: 'collection created with properties',
386         manifest_text: '',
387         properties: {
388           property_1: 'value_1'
389         },
390       },
391       format: :json
392     }, session_for(:active)
393     assert_response :success
394     assert_not_nil assigns(:object).uuid
395     assert_equal 'collection created with properties', assigns(:object).name
396     assert_equal 'value_1', assigns(:object).properties[:property_1]
397   end
398
399   test "update description and check manifest_text is not lost" do
400     collection = api_fixture("collections")["multilevel_collection_1"]
401     post :update, {
402       id: collection["uuid"],
403       collection: {
404         description: 'test description update'
405       },
406       format: :json
407     }, session_for(:active)
408     assert_response :success
409     assert_not_nil assigns(:object)
410     # Ensure the Workbench response still has the original manifest_text
411     assert_equal 'test description update', assigns(:object).description
412     assert_equal true, strip_signatures_and_compare(collection['manifest_text'], assigns(:object).manifest_text)
413     # Ensure the API server still has the original manifest_text after
414     # we called arvados.v1.collections.update
415     use_token :active do
416       assert_equal true, strip_signatures_and_compare(Collection.find(collection['uuid']).manifest_text,
417                                                       collection['manifest_text'])
418     end
419   end
420
421   # Since we got the initial collection from fixture, there are no signatures in manifest_text.
422   # However, after update or find, the collection retrieved will have singed manifest_text.
423   # Hence, let's compare each line after excluding signatures.
424   def strip_signatures_and_compare m1, m2
425     m1_lines = m1.split "\n"
426     m2_lines = m2.split "\n"
427
428     return false if m1_lines.size != m2_lines.size
429
430     m1_lines.each_with_index do |line, i|
431       m1_words = []
432       line.split.each do |word|
433         m1_words << word.split('+A')[0]
434       end
435       m2_words = []
436       m2_lines[i].split.each do |word|
437         m2_words << word.split('+A')[0]
438       end
439       return false if !m1_words.join(' ').eql?(m2_words.join(' '))
440     end
441
442     return true
443   end
444
445   test "view collection and verify none of the file types listed are disabled" do
446     show_collection(:collection_with_several_supported_file_types, :active)
447
448     files = assigns(:object).files
449     assert_equal true, files.length>0, "Expected one or more files in collection"
450
451     disabled = css_select('[disabled="disabled"]').collect do |el|
452       el
453     end
454     assert_equal 0, disabled.length, "Expected no disabled files in collection viewables list"
455   end
456
457   test "view collection and verify file types listed are all disabled" do
458     show_collection(:collection_with_several_unsupported_file_types, :active)
459
460     files = assigns(:object).files.collect do |_, file, _|
461       file
462     end
463     assert_equal true, files.length>0, "Expected one or more files in collection"
464
465     disabled = css_select('[disabled="disabled"]').collect do |el|
466       el.attributes['title'].split[-1]
467     end
468
469     assert_equal files.sort, disabled.sort, "Expected to see all collection files in disabled list of files"
470   end
471
472   test "anonymous user accesses collection in shared project" do
473     config_anonymous true
474     collection = api_fixture('collections')['public_text_file']
475     get(:show, {id: collection['uuid']})
476
477     response_object = assigns(:object)
478     assert_equal collection['name'], response_object['name']
479     assert_equal collection['uuid'], response_object['uuid']
480     assert_includes @response.body, 'Hello world'
481     assert_includes @response.body, 'Content address'
482     refute_nil css_select('[href="#Advanced"]')
483   end
484
485   test "can view empty collection" do
486     get :show, {id: 'd41d8cd98f00b204e9800998ecf8427e+0'}, session_for(:active)
487     assert_includes @response.body, 'The following collections have this content'
488   end
489
490   test "collection portable data hash redirect" do
491     di = api_fixture('collections')['docker_image']
492     get :show, {id: di['portable_data_hash']}, session_for(:active)
493     assert_match /\/collections\/#{di['uuid']}/, @response.redirect_url
494   end
495
496   test "collection portable data hash with multiple matches" do
497     pdh = api_fixture('collections')['foo_file']['portable_data_hash']
498     get :show, {id: pdh}, session_for(:admin)
499     matches = api_fixture('collections').select {|k,v| v["portable_data_hash"] == pdh}
500     assert matches.size > 1
501
502     matches.each do |k,v|
503       assert_match /href="\/collections\/#{v['uuid']}">.*#{v['name']}<\/a>/, @response.body
504     end
505
506     assert_includes @response.body, 'The following collections have this content:'
507     assert_not_includes @response.body, 'more results are not shown'
508     assert_not_includes @response.body, 'Activity'
509     assert_not_includes @response.body, 'Sharing and permissions'
510   end
511
512   test "collection page renders name" do
513     collection = api_fixture('collections')['foo_file']
514     get :show, {id: collection['uuid']}, session_for(:active)
515     assert_includes @response.body, collection['name']
516     assert_match /href="#{collection['uuid']}\/foo" ><\/i> foo</, @response.body
517   end
518
519   test "No Upload tab on non-writable collection" do
520     get :show, {id: api_fixture('collections')['user_agreement']['uuid']}, session_for(:active)
521     assert_not_includes @response.body, '<a href="#Upload"'
522   end
523
524   def setup_for_keep_web cfg='https://%{uuid_or_pdh}.example', dl_cfg=false
525     Rails.configuration.keep_web_url = cfg
526     Rails.configuration.keep_web_download_url = dl_cfg
527     @controller.expects(:file_enumerator).never
528   end
529
530   %w(uuid portable_data_hash).each do |id_type|
531     test "Redirect to keep_web_url via #{id_type}" do
532       setup_for_keep_web
533       tok = api_fixture('api_client_authorizations')['active']['api_token']
534       id = api_fixture('collections')['w_a_z_file'][id_type]
535       get :show_file, {uuid: id, file: "w a z"}, session_for(:active)
536       assert_response :redirect
537       assert_equal "https://#{id.sub '+', '-'}.example/_/w+a+z?api_token=#{tok}", @response.redirect_url
538     end
539
540     test "Redirect to keep_web_url via #{id_type} with reader token" do
541       setup_for_keep_web
542       tok = api_fixture('api_client_authorizations')['active']['api_token']
543       id = api_fixture('collections')['w_a_z_file'][id_type]
544       get :show_file, {uuid: id, file: "w a z", reader_token: tok}, session_for(:expired)
545       assert_response :redirect
546       assert_equal "https://#{id.sub '+', '-'}.example/t=#{tok}/_/w+a+z", @response.redirect_url
547     end
548
549     test "Redirect to keep_web_url via #{id_type} with no token" do
550       setup_for_keep_web
551       config_anonymous true
552       id = api_fixture('collections')['public_text_file'][id_type]
553       get :show_file, {uuid: id, file: "Hello World.txt"}
554       assert_response :redirect
555       assert_equal "https://#{id.sub '+', '-'}.example/_/Hello+World.txt", @response.redirect_url
556     end
557
558     test "Redirect to keep_web_url via #{id_type} with disposition param" do
559       setup_for_keep_web
560       config_anonymous true
561       id = api_fixture('collections')['public_text_file'][id_type]
562       get :show_file, {
563         uuid: id,
564         file: "Hello World.txt",
565         disposition: 'attachment',
566       }
567       assert_response :redirect
568       assert_equal "https://#{id.sub '+', '-'}.example/_/Hello+World.txt?disposition=attachment", @response.redirect_url
569     end
570
571     test "Redirect to keep_web_download_url via #{id_type}" do
572       setup_for_keep_web('https://collections.example/c=%{uuid_or_pdh}',
573                          'https://download.example/c=%{uuid_or_pdh}')
574       tok = api_fixture('api_client_authorizations')['active']['api_token']
575       id = api_fixture('collections')['w_a_z_file'][id_type]
576       get :show_file, {uuid: id, file: "w a z"}, session_for(:active)
577       assert_response :redirect
578       assert_equal "https://download.example/c=#{id.sub '+', '-'}/_/w+a+z?api_token=#{tok}", @response.redirect_url
579     end
580   end
581
582   [false, true].each do |anon|
583     test "No redirect to keep_web_url if collection not found, anon #{anon}" do
584       setup_for_keep_web
585       config_anonymous anon
586       id = api_fixture('collections')['w_a_z_file']['uuid']
587       get :show_file, {uuid: id, file: "w a z"}, session_for(:spectator)
588       assert_response 404
589     end
590
591     test "Redirect download to keep_web_download_url, anon #{anon}" do
592       config_anonymous anon
593       setup_for_keep_web('https://collections.example/c=%{uuid_or_pdh}',
594                          'https://download.example/c=%{uuid_or_pdh}')
595       tok = api_fixture('api_client_authorizations')['active']['api_token']
596       id = api_fixture('collections')['public_text_file']['uuid']
597       get :show_file, {
598         uuid: id,
599         file: 'Hello world.txt',
600         disposition: 'attachment',
601       }, session_for(:active)
602       assert_response :redirect
603       expect_url = "https://download.example/c=#{id.sub '+', '-'}/_/Hello+world.txt"
604       if not anon
605         expect_url += "?api_token=#{tok}"
606       end
607       assert_equal expect_url, @response.redirect_url
608     end
609   end
610
611   test "Error if file is impossible to retrieve from keep_web_url" do
612     # Cannot pass a session token using a single-origin keep-web URL,
613     # cannot read this collection without a session token.
614     setup_for_keep_web 'https://collections.example/c=%{uuid_or_pdh}', false
615     id = api_fixture('collections')['w_a_z_file']['uuid']
616     get :show_file, {uuid: id, file: "w a z"}, session_for(:active)
617     assert_response 422
618   end
619
620   test "Redirect preview to keep_web_download_url when preview is disabled" do
621     setup_for_keep_web false, 'https://download.example/c=%{uuid_or_pdh}'
622     tok = api_fixture('api_client_authorizations')['active']['api_token']
623     id = api_fixture('collections')['w_a_z_file']['uuid']
624     get :show_file, {uuid: id, file: "w a z"}, session_for(:active)
625     assert_response :redirect
626     assert_equal "https://download.example/c=#{id.sub '+', '-'}/_/w+a+z?api_token=#{tok}", @response.redirect_url
627   end
628 end