Merge branch '9865-cwl-fix-ignored-exceptions'
[arvados.git] / services / api / test / unit / container_test.rb
1 # Copyright (C) The Arvados Authors. All rights reserved.
2 #
3 # SPDX-License-Identifier: AGPL-3.0
4
5 require 'test_helper'
6 require 'helpers/container_test_helper'
7
8 class ContainerTest < ActiveSupport::TestCase
9   include DbCurrentTime
10   include ContainerTestHelper
11
12   DEFAULT_ATTRS = {
13     command: ['echo', 'foo'],
14     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
15     output_path: '/tmp',
16     priority: 1,
17     runtime_constraints: {"vcpus" => 1, "ram" => 1},
18   }
19
20   REUSABLE_COMMON_ATTRS = {
21     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
22     cwd: "test",
23     command: ["echo", "hello"],
24     output_path: "test",
25     runtime_constraints: {
26       "ram" => 12000000000,
27       "vcpus" => 4,
28     },
29     mounts: {
30       "test" => {"kind" => "json"},
31     },
32     environment: {
33       "var" => "val",
34     },
35     secret_mounts: {},
36     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
37     runtime_auth_scopes: ["all"]
38   }
39
40   def request_only attrs
41     attrs.reject {|k| [:runtime_user_uuid, :runtime_auth_scopes].include? k}
42   end
43
44   def minimal_new attrs={}
45     cr = ContainerRequest.new request_only(DEFAULT_ATTRS.merge(attrs))
46     cr.state = ContainerRequest::Committed
47     cr.save!
48     c = Container.find_by_uuid cr.container_uuid
49     assert_not_nil c
50     return c, cr
51   end
52
53   def check_illegal_updates c, bad_updates
54     bad_updates.each do |u|
55       refute c.update_attributes(u), u.inspect
56       refute c.valid?, u.inspect
57       c.reload
58     end
59   end
60
61   def check_illegal_modify c
62     check_illegal_updates c, [{command: ["echo", "bar"]},
63                               {container_image: "arvados/apitestfixture:june10"},
64                               {cwd: "/tmp2"},
65                               {environment: {"FOO" => "BAR"}},
66                               {mounts: {"FOO" => "BAR"}},
67                               {output_path: "/tmp3"},
68                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
69                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
70                               {runtime_constraints: {"FOO" => "BAR"}}]
71   end
72
73   def check_bogus_states c
74     check_illegal_updates c, [{state: nil},
75                               {state: "Flubber"}]
76   end
77
78   def check_no_change_from_cancelled c
79     check_illegal_modify c
80     check_bogus_states c
81     check_illegal_updates c, [{ priority: 3 },
82                               { state: Container::Queued },
83                               { state: Container::Locked },
84                               { state: Container::Running },
85                               { state: Container::Complete }]
86   end
87
88   test "Container create" do
89     act_as_system_user do
90       c, _ = minimal_new(environment: {},
91                       mounts: {"BAR" => {"kind" => "FOO"}},
92                       output_path: "/tmp",
93                       priority: 1,
94                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
95
96       check_illegal_modify c
97       check_bogus_states c
98
99       c.reload
100       c.priority = 2
101       c.save!
102     end
103   end
104
105   test "Container valid priority" do
106     act_as_system_user do
107       c, _ = minimal_new(environment: {},
108                       mounts: {"BAR" => {"kind" => "FOO"}},
109                       output_path: "/tmp",
110                       priority: 1,
111                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
112
113       assert_raises(ActiveRecord::RecordInvalid) do
114         c.priority = -1
115         c.save!
116       end
117
118       c.priority = 0
119       c.save!
120
121       c.priority = 1
122       c.save!
123
124       c.priority = 500
125       c.save!
126
127       c.priority = 999
128       c.save!
129
130       c.priority = 1000
131       c.save!
132
133       c.priority = 1000 << 50
134       c.save!
135     end
136   end
137
138   test "Container runtime_status data types" do
139     set_user_from_auth :active
140     attrs = {
141       environment: {},
142       mounts: {"BAR" => {"kind" => "FOO"}},
143       output_path: "/tmp",
144       priority: 1,
145       runtime_constraints: {"vcpus" => 1, "ram" => 1}
146     }
147     c, _ = minimal_new(attrs)
148     assert_equal c.runtime_status, {}
149     assert_equal Container::Queued, c.state
150
151     set_user_from_auth :dispatch1
152     c.update_attributes! state: Container::Locked
153     c.update_attributes! state: Container::Running
154
155     [
156       'error', 'errorDetail', 'warning', 'warningDetail', 'activity'
157     ].each do |k|
158       # String type is allowed
159       string_val = 'A string is accepted'
160       c.update_attributes! runtime_status: {k => string_val}
161       assert_equal string_val, c.runtime_status[k]
162
163       # Other types aren't allowed
164       [
165         42, false, [], {}, nil
166       ].each do |unallowed_val|
167         assert_raises ActiveRecord::RecordInvalid do
168           c.update_attributes! runtime_status: {k => unallowed_val}
169         end
170       end
171     end
172   end
173
174   test "Container runtime_status updates" do
175     set_user_from_auth :active
176     attrs = {
177       environment: {},
178       mounts: {"BAR" => {"kind" => "FOO"}},
179       output_path: "/tmp",
180       priority: 1,
181       runtime_constraints: {"vcpus" => 1, "ram" => 1}
182     }
183     c1, _ = minimal_new(attrs)
184     assert_equal c1.runtime_status, {}
185
186     assert_equal Container::Queued, c1.state
187     assert_raises ActiveRecord::RecordInvalid do
188       c1.update_attributes! runtime_status: {'error' => 'Oops!'}
189     end
190
191     set_user_from_auth :dispatch1
192
193     # Allow updates when state = Locked
194     c1.update_attributes! state: Container::Locked
195     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
196     assert c1.runtime_status.key? 'error'
197
198     # Reset when transitioning from Locked to Queued
199     c1.update_attributes! state: Container::Queued
200     assert_equal c1.runtime_status, {}
201
202     # Allow updates when state = Running
203     c1.update_attributes! state: Container::Locked
204     c1.update_attributes! state: Container::Running
205     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
206     assert c1.runtime_status.key? 'error'
207
208     # Don't allow updates on other states
209     c1.update_attributes! state: Container::Complete
210     assert_raises ActiveRecord::RecordInvalid do
211       c1.update_attributes! runtime_status: {'error' => 'Some other error'}
212     end
213
214     set_user_from_auth :active
215     c2, _ = minimal_new(attrs)
216     assert_equal c2.runtime_status, {}
217     set_user_from_auth :dispatch1
218     c2.update_attributes! state: Container::Locked
219     c2.update_attributes! state: Container::Running
220     c2.update_attributes! state: Container::Cancelled
221     assert_raises ActiveRecord::RecordInvalid do
222       c2.update_attributes! runtime_status: {'error' => 'Oops!'}
223     end
224   end
225
226   test "Container serialized hash attributes sorted before save" do
227     set_user_from_auth :active
228     env = {"C" => "3", "B" => "2", "A" => "1"}
229     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
230     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
231     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
232     assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
233     assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
234     assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
235   end
236
237   test 'deep_sort_hash on array of hashes' do
238     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
239     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
240     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
241   end
242
243   test "find_reusable method should select higher priority queued container" do
244         Rails.configuration.log_reuse_decisions = true
245     set_user_from_auth :active
246     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
247     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
248     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
249     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
250     assert_equal Container::Queued, c_low_priority.state
251     assert_equal Container::Queued, c_high_priority.state
252     reused = Container.find_reusable(common_attrs)
253     assert_not_nil reused
254     assert_equal reused.uuid, c_high_priority.uuid
255   end
256
257   test "find_reusable method should select latest completed container" do
258     set_user_from_auth :active
259     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
260     completed_attrs = {
261       state: Container::Complete,
262       exit_code: 0,
263       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
264       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
265     }
266
267     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
268     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
269     assert_not_equal c_older.uuid, c_recent.uuid
270
271     set_user_from_auth :dispatch1
272     c_older.update_attributes!({state: Container::Locked})
273     c_older.update_attributes!({state: Container::Running})
274     c_older.update_attributes!(completed_attrs)
275
276     c_recent.update_attributes!({state: Container::Locked})
277     c_recent.update_attributes!({state: Container::Running})
278     c_recent.update_attributes!(completed_attrs)
279
280     reused = Container.find_reusable(common_attrs)
281     assert_not_nil reused
282     assert_equal reused.uuid, c_older.uuid
283   end
284
285   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
286     set_user_from_auth :active
287     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
288     completed_attrs = {
289       state: Container::Complete,
290       exit_code: 0,
291       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
292     }
293
294     cr = ContainerRequest.new request_only(common_attrs)
295     cr.use_existing = false
296     cr.state = ContainerRequest::Committed
297     cr.save!
298     c_output1 = Container.where(uuid: cr.container_uuid).first
299
300     cr = ContainerRequest.new request_only(common_attrs)
301     cr.use_existing = false
302     cr.state = ContainerRequest::Committed
303     cr.save!
304     c_output2 = Container.where(uuid: cr.container_uuid).first
305
306     assert_not_equal c_output1.uuid, c_output2.uuid
307
308     set_user_from_auth :dispatch1
309
310     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
311     log1 = collections(:real_log_collection).portable_data_hash
312     c_output1.update_attributes!({state: Container::Locked})
313     c_output1.update_attributes!({state: Container::Running})
314     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
315
316     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
317     c_output2.update_attributes!({state: Container::Locked})
318     c_output2.update_attributes!({state: Container::Running})
319     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
320
321     set_user_from_auth :active
322     reused = Container.resolve(ContainerRequest.new(request_only(common_attrs)))
323     assert_equal c_output1.uuid, reused.uuid
324   end
325
326   test "find_reusable method should select running container by start date" do
327     set_user_from_auth :active
328     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
329     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
330     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
331     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
332     # Confirm the 3 container UUIDs are different.
333     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
334     set_user_from_auth :dispatch1
335     c_slower.update_attributes!({state: Container::Locked})
336     c_slower.update_attributes!({state: Container::Running,
337                                  progress: 0.1})
338     c_faster_started_first.update_attributes!({state: Container::Locked})
339     c_faster_started_first.update_attributes!({state: Container::Running,
340                                                progress: 0.15})
341     c_faster_started_second.update_attributes!({state: Container::Locked})
342     c_faster_started_second.update_attributes!({state: Container::Running,
343                                                 progress: 0.15})
344     reused = Container.find_reusable(common_attrs)
345     assert_not_nil reused
346     # Selected container is the one that started first
347     assert_equal reused.uuid, c_faster_started_first.uuid
348   end
349
350   test "find_reusable method should select running container by progress" do
351     set_user_from_auth :active
352     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
353     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
354     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
355     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
356     # Confirm the 3 container UUIDs are different.
357     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
358     set_user_from_auth :dispatch1
359     c_slower.update_attributes!({state: Container::Locked})
360     c_slower.update_attributes!({state: Container::Running,
361                                  progress: 0.1})
362     c_faster_started_first.update_attributes!({state: Container::Locked})
363     c_faster_started_first.update_attributes!({state: Container::Running,
364                                                progress: 0.15})
365     c_faster_started_second.update_attributes!({state: Container::Locked})
366     c_faster_started_second.update_attributes!({state: Container::Running,
367                                                 progress: 0.2})
368     reused = Container.find_reusable(common_attrs)
369     assert_not_nil reused
370     # Selected container is the one with most progress done
371     assert_equal reused.uuid, c_faster_started_second.uuid
372   end
373
374   test "find_reusable method should select non-failing running container" do
375     set_user_from_auth :active
376     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
377     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
378     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
379     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
380     # Confirm the 3 container UUIDs are different.
381     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
382     set_user_from_auth :dispatch1
383     c_slower.update_attributes!({state: Container::Locked})
384     c_slower.update_attributes!({state: Container::Running,
385                                  progress: 0.1})
386     c_faster_started_first.update_attributes!({state: Container::Locked})
387     c_faster_started_first.update_attributes!({state: Container::Running,
388                                                runtime_status: {'warning' => 'This is not an error'},
389                                                progress: 0.15})
390     c_faster_started_second.update_attributes!({state: Container::Locked})
391     c_faster_started_second.update_attributes!({state: Container::Running,
392                                                 runtime_status: {'error' => 'Something bad happened'},
393                                                 progress: 0.2})
394     reused = Container.find_reusable(common_attrs)
395     assert_not_nil reused
396     # Selected the non-failing container even if it's the one with less progress done
397     assert_equal reused.uuid, c_faster_started_first.uuid
398   end
399
400   test "find_reusable method should select locked container most likely to start sooner" do
401     set_user_from_auth :active
402     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
403     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
404     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
405     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
406     # Confirm the 3 container UUIDs are different.
407     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
408     set_user_from_auth :dispatch1
409     c_low_priority.update_attributes!({state: Container::Locked,
410                                        priority: 1})
411     c_high_priority_older.update_attributes!({state: Container::Locked,
412                                               priority: 2})
413     c_high_priority_newer.update_attributes!({state: Container::Locked,
414                                               priority: 2})
415     reused = Container.find_reusable(common_attrs)
416     assert_not_nil reused
417     assert_equal reused.uuid, c_high_priority_older.uuid
418   end
419
420   test "find_reusable method should select running over failed container" do
421     set_user_from_auth :active
422     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
423     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
424     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
425     assert_not_equal c_failed.uuid, c_running.uuid
426     set_user_from_auth :dispatch1
427     c_failed.update_attributes!({state: Container::Locked})
428     c_failed.update_attributes!({state: Container::Running})
429     c_failed.update_attributes!({state: Container::Complete,
430                                  exit_code: 42,
431                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
432                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
433     c_running.update_attributes!({state: Container::Locked})
434     c_running.update_attributes!({state: Container::Running,
435                                   progress: 0.15})
436     reused = Container.find_reusable(common_attrs)
437     assert_not_nil reused
438     assert_equal reused.uuid, c_running.uuid
439   end
440
441   test "find_reusable method should select complete over running container" do
442     set_user_from_auth :active
443     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
444     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
445     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
446     assert_not_equal c_completed.uuid, c_running.uuid
447     set_user_from_auth :dispatch1
448     c_completed.update_attributes!({state: Container::Locked})
449     c_completed.update_attributes!({state: Container::Running})
450     c_completed.update_attributes!({state: Container::Complete,
451                                     exit_code: 0,
452                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
453                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
454     c_running.update_attributes!({state: Container::Locked})
455     c_running.update_attributes!({state: Container::Running,
456                                   progress: 0.15})
457     reused = Container.find_reusable(common_attrs)
458     assert_not_nil reused
459     assert_equal c_completed.uuid, reused.uuid
460   end
461
462   test "find_reusable method should select running over locked container" do
463     set_user_from_auth :active
464     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
465     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
466     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
467     assert_not_equal c_running.uuid, c_locked.uuid
468     set_user_from_auth :dispatch1
469     c_locked.update_attributes!({state: Container::Locked})
470     c_running.update_attributes!({state: Container::Locked})
471     c_running.update_attributes!({state: Container::Running,
472                                   progress: 0.15})
473     reused = Container.find_reusable(common_attrs)
474     assert_not_nil reused
475     assert_equal reused.uuid, c_running.uuid
476   end
477
478   test "find_reusable method should select locked over queued container" do
479     set_user_from_auth :active
480     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
481     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
482     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
483     assert_not_equal c_queued.uuid, c_locked.uuid
484     set_user_from_auth :dispatch1
485     c_locked.update_attributes!({state: Container::Locked})
486     reused = Container.find_reusable(common_attrs)
487     assert_not_nil reused
488     assert_equal reused.uuid, c_locked.uuid
489   end
490
491   test "find_reusable method should not select failed container" do
492     set_user_from_auth :active
493     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
494     c, _ = minimal_new(attrs)
495     set_user_from_auth :dispatch1
496     c.update_attributes!({state: Container::Locked})
497     c.update_attributes!({state: Container::Running})
498     c.update_attributes!({state: Container::Complete,
499                           exit_code: 33})
500     reused = Container.find_reusable(attrs)
501     assert_nil reused
502   end
503
504   test "find_reusable with logging disabled" do
505     set_user_from_auth :active
506     Rails.logger.expects(:info).never
507     Container.find_reusable(REUSABLE_COMMON_ATTRS)
508   end
509
510   test "find_reusable with logging enabled" do
511     set_user_from_auth :active
512     Rails.configuration.log_reuse_decisions = true
513     Rails.logger.expects(:info).at_least(3)
514     Container.find_reusable(REUSABLE_COMMON_ATTRS)
515   end
516
517   def runtime_token_attr tok
518     auth = api_client_authorizations(tok)
519     {runtime_user_uuid: User.find_by_id(auth.user_id).uuid,
520      runtime_auth_scopes: auth.scopes,
521      runtime_token: auth.token}
522   end
523
524   test "find_reusable method with same runtime_token" do
525     set_user_from_auth :active
526     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
527     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:container_runtime_token).token}))
528     assert_equal Container::Queued, c1.state
529     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
530     assert_not_nil reused
531     assert_equal reused.uuid, c1.uuid
532   end
533
534   test "find_reusable method with different runtime_token, same user" do
535     set_user_from_auth :active
536     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
537     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:crt_user).token}))
538     assert_equal Container::Queued, c1.state
539     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
540     assert_not_nil reused
541     assert_equal reused.uuid, c1.uuid
542   end
543
544   test "find_reusable method with nil runtime_token, then runtime_token with same user" do
545     set_user_from_auth :crt_user
546     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
547     c1, _ = minimal_new(common_attrs)
548     assert_equal Container::Queued, c1.state
549     assert_equal users(:container_runtime_token_user).uuid, c1.runtime_user_uuid
550     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
551     assert_not_nil reused
552     assert_equal reused.uuid, c1.uuid
553   end
554
555   test "find_reusable method with different runtime_token, different user" do
556     set_user_from_auth :crt_user
557     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
558     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
559     assert_equal Container::Queued, c1.state
560     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
561     # See #14584
562     assert_equal c1.uuid, reused.uuid
563   end
564
565   test "find_reusable method with nil runtime_token, then runtime_token with different user" do
566     set_user_from_auth :active
567     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
568     c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
569     assert_equal Container::Queued, c1.state
570     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
571     # See #14584
572     assert_equal c1.uuid, reused.uuid
573   end
574
575   test "find_reusable method with different runtime_token, different scope, same user" do
576     set_user_from_auth :active
577     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
578     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
579     assert_equal Container::Queued, c1.state
580     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
581     # See #14584
582     assert_equal c1.uuid, reused.uuid
583   end
584
585   test "Container running" do
586     set_user_from_auth :active
587     c, _ = minimal_new priority: 1
588
589     set_user_from_auth :dispatch1
590     check_illegal_updates c, [{state: Container::Running},
591                               {state: Container::Complete}]
592
593     c.lock
594     c.update_attributes! state: Container::Running
595
596     check_illegal_modify c
597     check_bogus_states c
598
599     check_illegal_updates c, [{state: Container::Queued}]
600     c.reload
601
602     c.update_attributes! priority: 3
603   end
604
605   test "Lock and unlock" do
606     set_user_from_auth :active
607     c, cr = minimal_new priority: 0
608
609     set_user_from_auth :dispatch1
610     assert_equal Container::Queued, c.state
611
612     assert_raise(ArvadosModel::LockFailedError) do
613       # "no priority"
614       c.lock
615     end
616     c.reload
617     assert cr.update_attributes priority: 1
618
619     refute c.update_attributes(state: Container::Running), "not locked"
620     c.reload
621     refute c.update_attributes(state: Container::Complete), "not locked"
622     c.reload
623
624     assert c.lock, show_errors(c)
625     assert c.locked_by_uuid
626     assert c.auth_uuid
627
628     assert_raise(ArvadosModel::LockFailedError) {c.lock}
629     c.reload
630
631     assert c.unlock, show_errors(c)
632     refute c.locked_by_uuid
633     refute c.auth_uuid
634
635     refute c.update_attributes(state: Container::Running), "not locked"
636     c.reload
637     refute c.locked_by_uuid
638     refute c.auth_uuid
639
640     assert c.lock, show_errors(c)
641     assert c.update_attributes(state: Container::Running), show_errors(c)
642     assert c.locked_by_uuid
643     assert c.auth_uuid
644
645     auth_uuid_was = c.auth_uuid
646
647     assert_raise(ArvadosModel::LockFailedError) do
648       # Running to Locked is not allowed
649       c.lock
650     end
651     c.reload
652     assert_raise(ArvadosModel::InvalidStateTransitionError) do
653       # Running to Queued is not allowed
654       c.unlock
655     end
656     c.reload
657
658     assert c.update_attributes(state: Container::Complete), show_errors(c)
659     refute c.locked_by_uuid
660     refute c.auth_uuid
661
662     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
663     assert_operator auth_exp, :<, db_current_time
664   end
665
666   test "Exceed maximum lock-unlock cycles" do
667     Rails.configuration.max_container_dispatch_attempts = 3
668
669     set_user_from_auth :active
670     c, cr = minimal_new
671
672     set_user_from_auth :dispatch1
673     assert_equal Container::Queued, c.state
674     assert_equal 0, c.lock_count
675
676     c.lock
677     c.reload
678     assert_equal 1, c.lock_count
679     assert_equal Container::Locked, c.state
680
681     c.unlock
682     c.reload
683     assert_equal 1, c.lock_count
684     assert_equal Container::Queued, c.state
685
686     c.lock
687     c.reload
688     assert_equal 2, c.lock_count
689     assert_equal Container::Locked, c.state
690
691     c.unlock
692     c.reload
693     assert_equal 2, c.lock_count
694     assert_equal Container::Queued, c.state
695
696     c.lock
697     c.reload
698     assert_equal 3, c.lock_count
699     assert_equal Container::Locked, c.state
700
701     c.unlock
702     c.reload
703     assert_equal 3, c.lock_count
704     assert_equal Container::Cancelled, c.state
705
706     assert_raise(ArvadosModel::LockFailedError) do
707       # Cancelled to Locked is not allowed
708       c.lock
709     end
710   end
711
712   test "Container queued cancel" do
713     set_user_from_auth :active
714     c, cr = minimal_new({container_count_max: 1})
715     set_user_from_auth :dispatch1
716     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
717     check_no_change_from_cancelled c
718     cr.reload
719     assert_equal ContainerRequest::Final, cr.state
720   end
721
722   test "Container queued count" do
723     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
724   end
725
726   test "Containers with no matching request are readable by admin" do
727     uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid)
728     assert_not_empty uuids
729     assert_empty Container.readable_by(users(:active)).where(uuid: uuids)
730     assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids)
731     assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count
732   end
733
734   test "Container locked cancel" do
735     set_user_from_auth :active
736     c, _ = minimal_new
737     set_user_from_auth :dispatch1
738     assert c.lock, show_errors(c)
739     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
740     check_no_change_from_cancelled c
741   end
742
743   test "Container locked cancel with log" do
744     set_user_from_auth :active
745     c, _ = minimal_new
746     set_user_from_auth :dispatch1
747     assert c.lock, show_errors(c)
748     assert c.update_attributes(
749              state: Container::Cancelled,
750              log: collections(:real_log_collection).portable_data_hash,
751            ), show_errors(c)
752     check_no_change_from_cancelled c
753   end
754
755   test "Container running cancel" do
756     set_user_from_auth :active
757     c, _ = minimal_new
758     set_user_from_auth :dispatch1
759     c.lock
760     c.update_attributes! state: Container::Running
761     c.update_attributes! state: Container::Cancelled
762     check_no_change_from_cancelled c
763   end
764
765   test "Container create forbidden for non-admin" do
766     set_user_from_auth :active_trustedclient
767     c = Container.new DEFAULT_ATTRS
768     c.environment = {}
769     c.mounts = {"BAR" => "FOO"}
770     c.output_path = "/tmp"
771     c.priority = 1
772     c.runtime_constraints = {}
773     assert_raises(ArvadosModel::PermissionDeniedError) do
774       c.save!
775     end
776   end
777
778   test "Container only set exit code on complete" do
779     set_user_from_auth :active
780     c, _ = minimal_new
781     set_user_from_auth :dispatch1
782     c.lock
783     c.update_attributes! state: Container::Running
784
785     check_illegal_updates c, [{exit_code: 1},
786                               {exit_code: 1, state: Container::Cancelled}]
787
788     assert c.update_attributes(exit_code: 1, state: Container::Complete)
789   end
790
791   test "locked_by_uuid can update log when locked/running, and output when running" do
792     set_user_from_auth :active
793     logcoll = collections(:real_log_collection)
794     c, cr1 = minimal_new
795     cr2 = ContainerRequest.new(DEFAULT_ATTRS)
796     cr2.state = ContainerRequest::Committed
797     act_as_user users(:active) do
798       cr2.save!
799     end
800     assert_equal cr1.container_uuid, cr2.container_uuid
801
802     logpdh_time1 = logcoll.portable_data_hash
803
804     set_user_from_auth :dispatch1
805     c.lock
806     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
807     c.update_attributes!(log: logpdh_time1)
808     c.update_attributes!(state: Container::Running)
809     cr1.reload
810     cr2.reload
811     cr1log_uuid = cr1.log_uuid
812     cr2log_uuid = cr2.log_uuid
813     assert_not_nil cr1log_uuid
814     assert_not_nil cr2log_uuid
815     assert_not_equal logcoll.uuid, cr1log_uuid
816     assert_not_equal logcoll.uuid, cr2log_uuid
817     assert_not_equal cr1log_uuid, cr2log_uuid
818
819     logcoll.update_attributes!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
820     logpdh_time2 = logcoll.portable_data_hash
821
822     assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
823     assert c.update_attributes(log: logpdh_time2)
824     assert c.update_attributes(state: Container::Complete, log: logcoll.portable_data_hash)
825     c.reload
826     assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
827     assert_equal logpdh_time2, c.log
828     refute c.update_attributes(output: nil)
829     refute c.update_attributes(log: nil)
830     cr1.reload
831     cr2.reload
832     assert_equal cr1log_uuid, cr1.log_uuid
833     assert_equal cr2log_uuid, cr2.log_uuid
834     assert_equal [logpdh_time2], Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq
835   end
836
837   ["auth_uuid", "runtime_token"].each do |tok|
838     test "#{tok} can set output, progress, runtime_status, state on running container -- but not log" do
839       if tok == "runtime_token"
840         set_user_from_auth :spectator
841         c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
842                            runtime_token: api_client_authorizations(:active).token)
843       else
844         set_user_from_auth :active
845         c, _ = minimal_new
846       end
847       set_user_from_auth :dispatch1
848       c.lock
849       c.update_attributes! state: Container::Running
850
851       if tok == "runtime_token"
852         auth = ApiClientAuthorization.validate(token: c.runtime_token)
853         Thread.current[:api_client_authorization] = auth
854         Thread.current[:api_client] = auth.api_client
855         Thread.current[:token] = auth.token
856         Thread.current[:user] = auth.user
857       else
858         auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
859         Thread.current[:api_client_authorization] = auth
860         Thread.current[:api_client] = auth.api_client
861         Thread.current[:token] = auth.token
862         Thread.current[:user] = auth.user
863       end
864
865       assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
866       assert c.update_attributes(runtime_status: {'warning' => 'something happened'})
867       assert c.update_attributes(progress: 0.5)
868       refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
869       c.reload
870       assert c.update_attributes(state: Container::Complete, exit_code: 0)
871     end
872   end
873
874   test "not allowed to set output that is not readable by current user" do
875     set_user_from_auth :active
876     c, _ = minimal_new
877     set_user_from_auth :dispatch1
878     c.lock
879     c.update_attributes! state: Container::Running
880
881     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
882     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
883
884     assert_raises ActiveRecord::RecordInvalid do
885       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
886     end
887   end
888
889   test "other token cannot set output on running container" do
890     set_user_from_auth :active
891     c, _ = minimal_new
892     set_user_from_auth :dispatch1
893     c.lock
894     c.update_attributes! state: Container::Running
895
896     set_user_from_auth :running_to_be_deleted_container_auth
897     refute c.update_attributes(output: collections(:foo_file).portable_data_hash)
898   end
899
900   test "can set trashed output on running container" do
901     set_user_from_auth :active
902     c, _ = minimal_new
903     set_user_from_auth :dispatch1
904     c.lock
905     c.update_attributes! state: Container::Running
906
907     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
908
909     assert output.is_trashed
910     assert c.update_attributes output: output.portable_data_hash
911     assert c.update_attributes! state: Container::Complete
912   end
913
914   test "not allowed to set trashed output that is not readable by current user" do
915     set_user_from_auth :active
916     c, _ = minimal_new
917     set_user_from_auth :dispatch1
918     c.lock
919     c.update_attributes! state: Container::Running
920
921     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
922
923     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
924     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
925
926     assert_raises ActiveRecord::RecordInvalid do
927       c.update_attributes! output: output.portable_data_hash
928     end
929   end
930
931   [
932     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
933     {state: Container::Cancelled},
934   ].each do |final_attrs|
935     test "secret_mounts and runtime_token are null after container is #{final_attrs[:state]}" do
936       set_user_from_auth :active
937       c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
938                           container_count_max: 1, runtime_token: api_client_authorizations(:active).token)
939       set_user_from_auth :dispatch1
940       c.lock
941       c.update_attributes!(state: Container::Running)
942       c.reload
943       assert c.secret_mounts.has_key?('/secret')
944       assert_equal api_client_authorizations(:active).token, c.runtime_token
945
946       c.update_attributes!(final_attrs)
947       c.reload
948       assert_equal({}, c.secret_mounts)
949       assert_nil c.runtime_token
950       cr.reload
951       assert_equal({}, cr.secret_mounts)
952       assert_nil cr.runtime_token
953       assert_no_secrets_logged
954     end
955   end
956 end