18948: Update test.
[arvados.git] / services / api / test / unit / container_test.rb
1 # Copyright (C) The Arvados Authors. All rights reserved.
2 #
3 # SPDX-License-Identifier: AGPL-3.0
4
5 require 'test_helper'
6 require 'helpers/container_test_helper'
7
8 class ContainerTest < ActiveSupport::TestCase
9   include DbCurrentTime
10   include ContainerTestHelper
11
12   DEFAULT_ATTRS = {
13     command: ['echo', 'foo'],
14     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
15     output_path: '/tmp',
16     priority: 1,
17     runtime_constraints: {"vcpus" => 1, "ram" => 1, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},
18   }
19
20   REUSABLE_COMMON_ATTRS = {
21     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
22     cwd: "test",
23     command: ["echo", "hello"],
24     output_path: "test",
25     runtime_constraints: {
26       "API" => false,
27       "keep_cache_ram" => 0,
28       "ram" => 12000000000,
29       "vcpus" => 4
30     },
31     mounts: {
32       "test" => {"kind" => "json"},
33     },
34     environment: {
35       "var" => "val",
36     },
37     secret_mounts: {},
38     runtime_user_uuid: "zzzzz-tpzed-xurymjxw79nv3jz",
39     runtime_auth_scopes: ["all"]
40   }
41
42   def request_only attrs
43     attrs.reject {|k| [:runtime_user_uuid, :runtime_auth_scopes].include? k}
44   end
45
46   def minimal_new attrs={}
47     cr = ContainerRequest.new request_only(DEFAULT_ATTRS.merge(attrs))
48     cr.state = ContainerRequest::Committed
49     cr.save!
50     c = Container.find_by_uuid cr.container_uuid
51     assert_not_nil c
52     return c, cr
53   end
54
55   def check_illegal_updates c, bad_updates
56     bad_updates.each do |u|
57       refute c.update_attributes(u), u.inspect
58       refute c.valid?, u.inspect
59       c.reload
60     end
61   end
62
63   def check_illegal_modify c
64     check_illegal_updates c, [{command: ["echo", "bar"]},
65                               {container_image: "arvados/apitestfixture:june10"},
66                               {cwd: "/tmp2"},
67                               {environment: {"FOO" => "BAR"}},
68                               {mounts: {"FOO" => "BAR"}},
69                               {output_path: "/tmp3"},
70                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
71                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
72                               {runtime_constraints: {"FOO" => "BAR"}}]
73   end
74
75   def check_bogus_states c
76     check_illegal_updates c, [{state: nil},
77                               {state: "Flubber"}]
78   end
79
80   def check_no_change_from_cancelled c
81     check_illegal_modify c
82     check_bogus_states c
83     check_illegal_updates c, [{ priority: 3 },
84                               { state: Container::Queued },
85                               { state: Container::Locked },
86                               { state: Container::Running },
87                               { state: Container::Complete }]
88   end
89
90   test "Container create" do
91     act_as_system_user do
92       c, _ = minimal_new(environment: {},
93                       mounts: {"BAR" => {"kind" => "FOO"}},
94                       output_path: "/tmp",
95                       priority: 1,
96                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
97
98       check_illegal_modify c
99       check_bogus_states c
100
101       c.reload
102       c.priority = 2
103       c.save!
104     end
105   end
106
107   test "Container valid priority" do
108     act_as_system_user do
109       c, _ = minimal_new(environment: {},
110                       mounts: {"BAR" => {"kind" => "FOO"}},
111                       output_path: "/tmp",
112                       priority: 1,
113                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
114
115       assert_raises(ActiveRecord::RecordInvalid) do
116         c.priority = -1
117         c.save!
118       end
119
120       c.priority = 0
121       c.save!
122
123       c.priority = 1
124       c.save!
125
126       c.priority = 500
127       c.save!
128
129       c.priority = 999
130       c.save!
131
132       c.priority = 1000
133       c.save!
134
135       c.priority = 1000 << 50
136       c.save!
137     end
138   end
139
140   test "Container runtime_status data types" do
141     set_user_from_auth :active
142     attrs = {
143       environment: {},
144       mounts: {"BAR" => {"kind" => "FOO"}},
145       output_path: "/tmp",
146       priority: 1,
147       runtime_constraints: {"vcpus" => 1, "ram" => 1}
148     }
149     c, _ = minimal_new(attrs)
150     assert_equal c.runtime_status, {}
151     assert_equal Container::Queued, c.state
152
153     set_user_from_auth :dispatch1
154     c.update_attributes! state: Container::Locked
155     c.update_attributes! state: Container::Running
156
157     [
158       'error', 'errorDetail', 'warning', 'warningDetail', 'activity'
159     ].each do |k|
160       # String type is allowed
161       string_val = 'A string is accepted'
162       c.update_attributes! runtime_status: {k => string_val}
163       assert_equal string_val, c.runtime_status[k]
164
165       # Other types aren't allowed
166       [
167         42, false, [], {}, nil
168       ].each do |unallowed_val|
169         assert_raises ActiveRecord::RecordInvalid do
170           c.update_attributes! runtime_status: {k => unallowed_val}
171         end
172       end
173     end
174   end
175
176   test "Container runtime_status updates" do
177     set_user_from_auth :active
178     attrs = {
179       environment: {},
180       mounts: {"BAR" => {"kind" => "FOO"}},
181       output_path: "/tmp",
182       priority: 1,
183       runtime_constraints: {"vcpus" => 1, "ram" => 1}
184     }
185     c1, _ = minimal_new(attrs)
186     assert_equal c1.runtime_status, {}
187
188     assert_equal Container::Queued, c1.state
189     assert_raises ArvadosModel::PermissionDeniedError do
190       c1.update_attributes! runtime_status: {'error' => 'Oops!'}
191     end
192
193     set_user_from_auth :dispatch1
194
195     # Allow updates when state = Locked
196     c1.update_attributes! state: Container::Locked
197     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
198     assert c1.runtime_status.key? 'error'
199
200     # Reset when transitioning from Locked to Queued
201     c1.update_attributes! state: Container::Queued
202     assert_equal c1.runtime_status, {}
203
204     # Allow updates when state = Running
205     c1.update_attributes! state: Container::Locked
206     c1.update_attributes! state: Container::Running
207     c1.update_attributes! runtime_status: {'error' => 'Oops!'}
208     assert c1.runtime_status.key? 'error'
209
210     # Don't allow updates on other states
211     c1.update_attributes! state: Container::Complete
212     assert_raises ActiveRecord::RecordInvalid do
213       c1.update_attributes! runtime_status: {'error' => 'Some other error'}
214     end
215
216     set_user_from_auth :active
217     c2, _ = minimal_new(attrs)
218     assert_equal c2.runtime_status, {}
219     set_user_from_auth :dispatch1
220     c2.update_attributes! state: Container::Locked
221     c2.update_attributes! state: Container::Running
222     c2.update_attributes! state: Container::Cancelled
223     assert_raises ActiveRecord::RecordInvalid do
224       c2.update_attributes! runtime_status: {'error' => 'Oops!'}
225     end
226   end
227
228   test "Container serialized hash attributes sorted before save" do
229     set_user_from_auth :active
230     env = {"C" => "3", "B" => "2", "A" => "1"}
231     m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
232     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "API" => true, "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}}
233     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
234     c.reload
235     assert_equal Container.deep_sort_hash(env).to_json, c.environment.to_json
236     assert_equal Container.deep_sort_hash(m).to_json, c.mounts.to_json
237     assert_equal Container.deep_sort_hash(rc).to_json, c.runtime_constraints.to_json
238   end
239
240   test 'deep_sort_hash on array of hashes' do
241     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
242     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
243     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
244   end
245
246   test "find_reusable method should select higher priority queued container" do
247         Rails.configuration.Containers.LogReuseDecisions = true
248     set_user_from_auth :active
249     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
250     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
251     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
252     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
253     assert_equal Container::Queued, c_low_priority.state
254     assert_equal Container::Queued, c_high_priority.state
255     reused = Container.find_reusable(common_attrs)
256     assert_not_nil reused
257     assert_equal reused.uuid, c_high_priority.uuid
258   end
259
260   test "find_reusable method should select latest completed container" do
261     set_user_from_auth :active
262     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
263     completed_attrs = {
264       state: Container::Complete,
265       exit_code: 0,
266       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
267       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
268     }
269
270     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
271     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
272     assert_not_equal c_older.uuid, c_recent.uuid
273
274     set_user_from_auth :dispatch1
275     c_older.update_attributes!({state: Container::Locked})
276     c_older.update_attributes!({state: Container::Running})
277     c_older.update_attributes!(completed_attrs)
278
279     c_recent.update_attributes!({state: Container::Locked})
280     c_recent.update_attributes!({state: Container::Running})
281     c_recent.update_attributes!(completed_attrs)
282
283     reused = Container.find_reusable(common_attrs)
284     assert_not_nil reused
285     assert_equal reused.uuid, c_older.uuid
286   end
287
288   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
289     set_user_from_auth :active
290     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
291     completed_attrs = {
292       state: Container::Complete,
293       exit_code: 0,
294       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
295     }
296
297     cr = ContainerRequest.new request_only(common_attrs)
298     cr.use_existing = false
299     cr.state = ContainerRequest::Committed
300     cr.save!
301     c_output1 = Container.where(uuid: cr.container_uuid).first
302
303     cr = ContainerRequest.new request_only(common_attrs)
304     cr.use_existing = false
305     cr.state = ContainerRequest::Committed
306     cr.save!
307     c_output2 = Container.where(uuid: cr.container_uuid).first
308
309     assert_not_equal c_output1.uuid, c_output2.uuid
310
311     set_user_from_auth :dispatch1
312
313     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
314     log1 = collections(:real_log_collection).portable_data_hash
315     c_output1.update_attributes!({state: Container::Locked})
316     c_output1.update_attributes!({state: Container::Running})
317     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
318
319     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
320     c_output2.update_attributes!({state: Container::Locked})
321     c_output2.update_attributes!({state: Container::Running})
322     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
323
324     set_user_from_auth :active
325     reused = Container.resolve(ContainerRequest.new(request_only(common_attrs)))
326     assert_equal c_output1.uuid, reused.uuid
327   end
328
329   test "find_reusable method should select running container by start date" do
330     set_user_from_auth :active
331     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
332     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
333     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
334     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
335     # Confirm the 3 container UUIDs are different.
336     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
337     set_user_from_auth :dispatch1
338     c_slower.update_attributes!({state: Container::Locked})
339     c_slower.update_attributes!({state: Container::Running,
340                                  progress: 0.1})
341     c_faster_started_first.update_attributes!({state: Container::Locked})
342     c_faster_started_first.update_attributes!({state: Container::Running,
343                                                progress: 0.15})
344     c_faster_started_second.update_attributes!({state: Container::Locked})
345     c_faster_started_second.update_attributes!({state: Container::Running,
346                                                 progress: 0.15})
347     reused = Container.find_reusable(common_attrs)
348     assert_not_nil reused
349     # Selected container is the one that started first
350     assert_equal reused.uuid, c_faster_started_first.uuid
351   end
352
353   test "find_reusable method should select running container by progress" do
354     set_user_from_auth :active
355     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
356     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
357     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
358     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
359     # Confirm the 3 container UUIDs are different.
360     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
361     set_user_from_auth :dispatch1
362     c_slower.update_attributes!({state: Container::Locked})
363     c_slower.update_attributes!({state: Container::Running,
364                                  progress: 0.1})
365     c_faster_started_first.update_attributes!({state: Container::Locked})
366     c_faster_started_first.update_attributes!({state: Container::Running,
367                                                progress: 0.15})
368     c_faster_started_second.update_attributes!({state: Container::Locked})
369     c_faster_started_second.update_attributes!({state: Container::Running,
370                                                 progress: 0.2})
371     reused = Container.find_reusable(common_attrs)
372     assert_not_nil reused
373     # Selected container is the one with most progress done
374     assert_equal reused.uuid, c_faster_started_second.uuid
375   end
376
377   test "find_reusable method should select non-failing running container" do
378     set_user_from_auth :active
379     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
380     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
381     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
382     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
383     # Confirm the 3 container UUIDs are different.
384     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
385     set_user_from_auth :dispatch1
386     c_slower.update_attributes!({state: Container::Locked})
387     c_slower.update_attributes!({state: Container::Running,
388                                  progress: 0.1})
389     c_faster_started_first.update_attributes!({state: Container::Locked})
390     c_faster_started_first.update_attributes!({state: Container::Running,
391                                                runtime_status: {'warning' => 'This is not an error'},
392                                                progress: 0.15})
393     c_faster_started_second.update_attributes!({state: Container::Locked})
394     assert_equal 0, Container.where("runtime_status->'error' is not null").count
395     c_faster_started_second.update_attributes!({state: Container::Running,
396                                                 runtime_status: {'error' => 'Something bad happened'},
397                                                 progress: 0.2})
398     assert_equal 1, Container.where("runtime_status->'error' is not null").count
399     reused = Container.find_reusable(common_attrs)
400     assert_not_nil reused
401     # Selected the non-failing container even if it's the one with less progress done
402     assert_equal reused.uuid, c_faster_started_first.uuid
403   end
404
405   test "find_reusable method should select locked container most likely to start sooner" do
406     set_user_from_auth :active
407     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
408     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
409     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
410     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
411     # Confirm the 3 container UUIDs are different.
412     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
413     set_user_from_auth :dispatch1
414     c_low_priority.update_attributes!({state: Container::Locked,
415                                        priority: 1})
416     c_high_priority_older.update_attributes!({state: Container::Locked,
417                                               priority: 2})
418     c_high_priority_newer.update_attributes!({state: Container::Locked,
419                                               priority: 2})
420     reused = Container.find_reusable(common_attrs)
421     assert_not_nil reused
422     assert_equal reused.uuid, c_high_priority_older.uuid
423   end
424
425   test "find_reusable method should select running over failed container" do
426     set_user_from_auth :active
427     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
428     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
429     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
430     assert_not_equal c_failed.uuid, c_running.uuid
431     set_user_from_auth :dispatch1
432     c_failed.update_attributes!({state: Container::Locked})
433     c_failed.update_attributes!({state: Container::Running})
434     c_failed.update_attributes!({state: Container::Complete,
435                                  exit_code: 42,
436                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
437                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
438     c_running.update_attributes!({state: Container::Locked})
439     c_running.update_attributes!({state: Container::Running,
440                                   progress: 0.15})
441     reused = Container.find_reusable(common_attrs)
442     assert_not_nil reused
443     assert_equal reused.uuid, c_running.uuid
444   end
445
446   test "find_reusable method should select complete over running container" do
447     set_user_from_auth :active
448     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
449     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
450     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
451     assert_not_equal c_completed.uuid, c_running.uuid
452     set_user_from_auth :dispatch1
453     c_completed.update_attributes!({state: Container::Locked})
454     c_completed.update_attributes!({state: Container::Running})
455     c_completed.update_attributes!({state: Container::Complete,
456                                     exit_code: 0,
457                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
458                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
459     c_running.update_attributes!({state: Container::Locked})
460     c_running.update_attributes!({state: Container::Running,
461                                   progress: 0.15})
462     reused = Container.find_reusable(common_attrs)
463     assert_not_nil reused
464     assert_equal c_completed.uuid, reused.uuid
465   end
466
467   test "find_reusable method should select running over locked container" do
468     set_user_from_auth :active
469     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
470     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
471     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
472     assert_not_equal c_running.uuid, c_locked.uuid
473     set_user_from_auth :dispatch1
474     c_locked.update_attributes!({state: Container::Locked})
475     c_running.update_attributes!({state: Container::Locked})
476     c_running.update_attributes!({state: Container::Running,
477                                   progress: 0.15})
478     reused = Container.find_reusable(common_attrs)
479     assert_not_nil reused
480     assert_equal reused.uuid, c_running.uuid
481   end
482
483   test "find_reusable method should select locked over queued container" do
484     set_user_from_auth :active
485     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
486     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
487     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
488     assert_not_equal c_queued.uuid, c_locked.uuid
489     set_user_from_auth :dispatch1
490     c_locked.update_attributes!({state: Container::Locked})
491     reused = Container.find_reusable(common_attrs)
492     assert_not_nil reused
493     assert_equal reused.uuid, c_locked.uuid
494   end
495
496   test "find_reusable method should not select failed container" do
497     set_user_from_auth :active
498     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
499     c, _ = minimal_new(attrs)
500     set_user_from_auth :dispatch1
501     c.update_attributes!({state: Container::Locked})
502     c.update_attributes!({state: Container::Running})
503     c.update_attributes!({state: Container::Complete,
504                           exit_code: 33})
505     reused = Container.find_reusable(attrs)
506     assert_nil reused
507   end
508
509   test "find_reusable with logging disabled" do
510     set_user_from_auth :active
511     Rails.logger.expects(:info).never
512     Container.find_reusable(REUSABLE_COMMON_ATTRS)
513   end
514
515   test "find_reusable with logging enabled" do
516     set_user_from_auth :active
517     Rails.configuration.Containers.LogReuseDecisions = true
518     Rails.logger.expects(:info).at_least(3)
519     Container.find_reusable(REUSABLE_COMMON_ATTRS)
520   end
521
522   def runtime_token_attr tok
523     auth = api_client_authorizations(tok)
524     {runtime_user_uuid: User.find_by_id(auth.user_id).uuid,
525      runtime_auth_scopes: auth.scopes,
526      runtime_token: auth.token}
527   end
528
529   test "find_reusable method with same runtime_token" do
530     set_user_from_auth :active
531     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
532     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:container_runtime_token).token}))
533     assert_equal Container::Queued, c1.state
534     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
535     assert_not_nil reused
536     assert_equal reused.uuid, c1.uuid
537   end
538
539   test "find_reusable method with different runtime_token, same user" do
540     set_user_from_auth :active
541     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
542     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:crt_user).token}))
543     assert_equal Container::Queued, c1.state
544     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
545     assert_not_nil reused
546     assert_equal reused.uuid, c1.uuid
547   end
548
549   test "find_reusable method with nil runtime_token, then runtime_token with same user" do
550     set_user_from_auth :crt_user
551     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
552     c1, _ = minimal_new(common_attrs)
553     assert_equal Container::Queued, c1.state
554     assert_equal users(:container_runtime_token_user).uuid, c1.runtime_user_uuid
555     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
556     assert_not_nil reused
557     assert_equal reused.uuid, c1.uuid
558   end
559
560   test "find_reusable method with different runtime_token, different user" do
561     set_user_from_auth :crt_user
562     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
563     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
564     assert_equal Container::Queued, c1.state
565     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
566     # See #14584
567     assert_not_nil reused
568     assert_equal c1.uuid, reused.uuid
569   end
570
571   test "find_reusable method with nil runtime_token, then runtime_token with different user" do
572     set_user_from_auth :active
573     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
574     c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
575     assert_equal Container::Queued, c1.state
576     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
577     # See #14584
578     assert_not_nil reused
579     assert_equal c1.uuid, reused.uuid
580   end
581
582   test "find_reusable method with different runtime_token, different scope, same user" do
583     set_user_from_auth :active
584     common_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"}})
585     c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
586     assert_equal Container::Queued, c1.state
587     reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
588     # See #14584
589     assert_not_nil reused
590     assert_equal c1.uuid, reused.uuid
591   end
592
593   test "find_reusable method with cuda" do
594     set_user_from_auth :active
595     # No cuda
596     no_cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
597                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_ram"=>268435456, "API" => false,
598                                                                       "cuda" => {"device_count":0, "driver_version": "", "hardware_capability": ""}},})
599     c1, _ = minimal_new(no_cuda_attrs)
600     assert_equal Container::Queued, c1.state
601
602     # has cuda
603     cuda_attrs = REUSABLE_COMMON_ATTRS.merge({use_existing:false, priority:1, environment:{"var" => "queued"},
604                                                 runtime_constraints: {"vcpus" => 1, "ram" => 1, "keep_cache_ram"=>268435456, "API" => false,
605                                                                       "cuda" => {"device_count":1, "driver_version": "11.0", "hardware_capability": "9.0"}},})
606     c2, _ = minimal_new(cuda_attrs)
607     assert_equal Container::Queued, c2.state
608
609     # should find the no cuda one
610     reused = Container.find_reusable(no_cuda_attrs)
611     assert_not_nil reused
612     assert_equal reused.uuid, c1.uuid
613
614     # should find the cuda one
615     reused = Container.find_reusable(cuda_attrs)
616     assert_not_nil reused
617     assert_equal reused.uuid, c2.uuid
618   end
619
620   test "Container running" do
621     set_user_from_auth :active
622     c, _ = minimal_new priority: 1
623
624     set_user_from_auth :dispatch1
625     check_illegal_updates c, [{state: Container::Running},
626                               {state: Container::Complete}]
627
628     c.lock
629     c.update_attributes! state: Container::Running
630
631     check_illegal_modify c
632     check_bogus_states c
633
634     check_illegal_updates c, [{state: Container::Queued}]
635     c.reload
636
637     c.update_attributes! priority: 3
638   end
639
640   test "Lock and unlock" do
641     set_user_from_auth :active
642     c, cr = minimal_new priority: 0
643
644     set_user_from_auth :dispatch1
645     assert_equal Container::Queued, c.state
646
647     assert_raise(ArvadosModel::LockFailedError) do
648       # "no priority"
649       c.lock
650     end
651     c.reload
652     assert cr.update_attributes priority: 1
653
654     refute c.update_attributes(state: Container::Running), "not locked"
655     c.reload
656     refute c.update_attributes(state: Container::Complete), "not locked"
657     c.reload
658
659     assert c.lock, show_errors(c)
660     assert c.locked_by_uuid
661     assert c.auth_uuid
662
663     assert_raise(ArvadosModel::LockFailedError) {c.lock}
664     c.reload
665
666     assert c.unlock, show_errors(c)
667     refute c.locked_by_uuid
668     refute c.auth_uuid
669
670     refute c.update_attributes(state: Container::Running), "not locked"
671     c.reload
672     refute c.locked_by_uuid
673     refute c.auth_uuid
674
675     assert c.lock, show_errors(c)
676     assert c.update_attributes(state: Container::Running), show_errors(c)
677     assert c.locked_by_uuid
678     assert c.auth_uuid
679
680     auth_uuid_was = c.auth_uuid
681
682     assert_raise(ArvadosModel::LockFailedError) do
683       # Running to Locked is not allowed
684       c.lock
685     end
686     c.reload
687     assert_raise(ArvadosModel::InvalidStateTransitionError) do
688       # Running to Queued is not allowed
689       c.unlock
690     end
691     c.reload
692
693     assert c.update_attributes(state: Container::Complete), show_errors(c)
694     refute c.locked_by_uuid
695     refute c.auth_uuid
696
697     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
698     assert_operator auth_exp, :<, db_current_time
699
700     assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token)
701   end
702
703   test "Exceed maximum lock-unlock cycles" do
704     Rails.configuration.Containers.MaxDispatchAttempts = 3
705
706     set_user_from_auth :active
707     c, cr = minimal_new
708
709     set_user_from_auth :dispatch1
710     assert_equal Container::Queued, c.state
711     assert_equal 0, c.lock_count
712
713     c.lock
714     c.reload
715     assert_equal 1, c.lock_count
716     assert_equal Container::Locked, c.state
717
718     c.unlock
719     c.reload
720     assert_equal 1, c.lock_count
721     assert_equal Container::Queued, c.state
722
723     c.lock
724     c.reload
725     assert_equal 2, c.lock_count
726     assert_equal Container::Locked, c.state
727
728     c.unlock
729     c.reload
730     assert_equal 2, c.lock_count
731     assert_equal Container::Queued, c.state
732
733     c.lock
734     c.reload
735     assert_equal 3, c.lock_count
736     assert_equal Container::Locked, c.state
737
738     c.unlock
739     c.reload
740     assert_equal 3, c.lock_count
741     assert_equal Container::Cancelled, c.state
742
743     assert_raise(ArvadosModel::LockFailedError) do
744       # Cancelled to Locked is not allowed
745       c.lock
746     end
747   end
748
749   test "Container queued cancel" do
750     set_user_from_auth :active
751     c, cr = minimal_new({container_count_max: 1})
752     set_user_from_auth :dispatch1
753     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
754     check_no_change_from_cancelled c
755     cr.reload
756     assert_equal ContainerRequest::Final, cr.state
757   end
758
759   test "Container queued count" do
760     assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
761   end
762
763   test "Containers with no matching request are readable by admin" do
764     uuids = Container.includes('container_requests').where(container_requests: {uuid: nil}).collect(&:uuid)
765     assert_not_empty uuids
766     assert_empty Container.readable_by(users(:active)).where(uuid: uuids)
767     assert_not_empty Container.readable_by(users(:admin)).where(uuid: uuids)
768     assert_equal uuids.count, Container.readable_by(users(:admin)).where(uuid: uuids).count
769   end
770
771   test "Container locked cancel" do
772     set_user_from_auth :active
773     c, _ = minimal_new
774     set_user_from_auth :dispatch1
775     assert c.lock, show_errors(c)
776     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
777     check_no_change_from_cancelled c
778   end
779
780   test "Container locked with non-expiring token" do
781     Rails.configuration.API.TokenMaxLifetime = 1.hour
782     set_user_from_auth :active
783     c, _ = minimal_new
784     set_user_from_auth :dispatch1
785     assert c.lock, show_errors(c)
786     refute c.auth.nil?
787     assert c.auth.expires_at.nil?
788     assert c.auth.user_id == User.find_by_uuid(users(:active).uuid).id
789   end
790
791   test "Container locked cancel with log" do
792     set_user_from_auth :active
793     c, _ = minimal_new
794     set_user_from_auth :dispatch1
795     assert c.lock, show_errors(c)
796     assert c.update_attributes(
797              state: Container::Cancelled,
798              log: collections(:real_log_collection).portable_data_hash,
799            ), show_errors(c)
800     check_no_change_from_cancelled c
801   end
802
803   test "Container running cancel" do
804     set_user_from_auth :active
805     c, _ = minimal_new
806     set_user_from_auth :dispatch1
807     c.lock
808     c.update_attributes! state: Container::Running
809     c.update_attributes! state: Container::Cancelled
810     check_no_change_from_cancelled c
811   end
812
813   test "Container create forbidden for non-admin" do
814     set_user_from_auth :active_trustedclient
815     c = Container.new DEFAULT_ATTRS
816     c.environment = {}
817     c.mounts = {"BAR" => "FOO"}
818     c.output_path = "/tmp"
819     c.priority = 1
820     c.runtime_constraints = {}
821     assert_raises(ArvadosModel::PermissionDeniedError) do
822       c.save!
823     end
824   end
825
826   [
827     [Container::Queued, {state: Container::Locked}],
828     [Container::Queued, {state: Container::Running}],
829     [Container::Queued, {state: Container::Complete}],
830     [Container::Queued, {state: Container::Cancelled}],
831     [Container::Queued, {priority: 123456789}],
832     [Container::Queued, {runtime_status: {'error' => 'oops'}}],
833     [Container::Queued, {cwd: '/'}],
834     [Container::Locked, {state: Container::Running}],
835     [Container::Locked, {state: Container::Queued}],
836     [Container::Locked, {priority: 123456789}],
837     [Container::Locked, {runtime_status: {'error' => 'oops'}}],
838     [Container::Locked, {cwd: '/'}],
839     [Container::Running, {state: Container::Complete}],
840     [Container::Running, {state: Container::Cancelled}],
841     [Container::Running, {priority: 123456789}],
842     [Container::Running, {runtime_status: {'error' => 'oops'}}],
843     [Container::Running, {cwd: '/'}],
844     [Container::Running, {gateway_address: "172.16.0.1:12345"}],
845     [Container::Running, {interactive_session_started: true}],
846     [Container::Complete, {state: Container::Cancelled}],
847     [Container::Complete, {priority: 123456789}],
848     [Container::Complete, {runtime_status: {'error' => 'oops'}}],
849     [Container::Complete, {cwd: '/'}],
850     [Container::Cancelled, {cwd: '/'}],
851   ].each do |start_state, updates|
852     test "Container update #{updates.inspect} when #{start_state} forbidden for non-admin" do
853       set_user_from_auth :active
854       c, _ = minimal_new
855       if start_state != Container::Queued
856         set_user_from_auth :dispatch1
857         c.lock
858         if start_state != Container::Locked
859           c.update_attributes! state: Container::Running
860           if start_state != Container::Running
861             c.update_attributes! state: start_state
862           end
863         end
864       end
865       assert_equal c.state, start_state
866       set_user_from_auth :active
867       assert_raises(ArvadosModel::PermissionDeniedError) do
868         c.update_attributes! updates
869       end
870     end
871   end
872
873   test "can only change exit code while running and at completion" do
874     set_user_from_auth :active
875     c, _ = minimal_new
876     set_user_from_auth :dispatch1
877     c.lock
878     check_illegal_updates c, [{exit_code: 1}]
879     c.update_attributes! state: Container::Running
880     assert c.update_attributes(exit_code: 1)
881     assert c.update_attributes(exit_code: 1, state: Container::Complete)
882   end
883
884   test "locked_by_uuid can update log when locked/running, and output when running" do
885     set_user_from_auth :active
886     logcoll = collections(:real_log_collection)
887     c, cr1 = minimal_new
888     cr2 = ContainerRequest.new(DEFAULT_ATTRS)
889     cr2.state = ContainerRequest::Committed
890     act_as_user users(:active) do
891       cr2.save!
892     end
893     assert_equal cr1.container_uuid, cr2.container_uuid
894
895     logpdh_time1 = logcoll.portable_data_hash
896
897     set_user_from_auth :dispatch1
898     c.lock
899     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
900     c.update_attributes!(log: logpdh_time1)
901     c.update_attributes!(state: Container::Running)
902     cr1.reload
903     cr2.reload
904     cr1log_uuid = cr1.log_uuid
905     cr2log_uuid = cr2.log_uuid
906     assert_not_nil cr1log_uuid
907     assert_not_nil cr2log_uuid
908     assert_not_equal logcoll.uuid, cr1log_uuid
909     assert_not_equal logcoll.uuid, cr2log_uuid
910     assert_not_equal cr1log_uuid, cr2log_uuid
911
912     logcoll.update_attributes!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
913     logpdh_time2 = logcoll.portable_data_hash
914
915     assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
916     assert c.update_attributes(log: logpdh_time2)
917     assert c.update_attributes(state: Container::Complete, log: logcoll.portable_data_hash)
918     c.reload
919     assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
920     assert_equal logpdh_time2, c.log
921     refute c.update_attributes(output: nil)
922     refute c.update_attributes(log: nil)
923     cr1.reload
924     cr2.reload
925     assert_equal cr1log_uuid, cr1.log_uuid
926     assert_equal cr2log_uuid, cr2.log_uuid
927     assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length
928     assert_equal ". acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
929 ./log\\040for\\040container\\040#{cr1.container_uuid} acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
930 ", Collection.find_by_uuid(cr1log_uuid).manifest_text
931   end
932
933   ["auth_uuid", "runtime_token"].each do |tok|
934     test "#{tok} can set output, progress, runtime_status, state, exit_code on running container -- but not log" do
935       if tok == "runtime_token"
936         set_user_from_auth :spectator
937         c, _ = minimal_new(container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
938                            runtime_token: api_client_authorizations(:active).token)
939       else
940         set_user_from_auth :active
941         c, _ = minimal_new
942       end
943       set_user_from_auth :dispatch1
944       c.lock
945       c.update_attributes! state: Container::Running
946
947       if tok == "runtime_token"
948         auth = ApiClientAuthorization.validate(token: c.runtime_token)
949         Thread.current[:api_client_authorization] = auth
950         Thread.current[:api_client] = auth.api_client
951         Thread.current[:token] = auth.token
952         Thread.current[:user] = auth.user
953       else
954         auth = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
955         Thread.current[:api_client_authorization] = auth
956         Thread.current[:api_client] = auth.api_client
957         Thread.current[:token] = auth.token
958         Thread.current[:user] = auth.user
959       end
960
961       assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
962       assert c.update_attributes(runtime_status: {'warning' => 'something happened'})
963       assert c.update_attributes(progress: 0.5)
964       assert c.update_attributes(exit_code: 0)
965       refute c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
966       c.reload
967       assert c.update_attributes(state: Container::Complete, exit_code: 0)
968     end
969   end
970
971   test "not allowed to set output that is not readable by current user" do
972     set_user_from_auth :active
973     c, _ = minimal_new
974     set_user_from_auth :dispatch1
975     c.lock
976     c.update_attributes! state: Container::Running
977
978     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
979     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
980
981     assert_raises ActiveRecord::RecordInvalid do
982       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
983     end
984   end
985
986   test "other token cannot set output on running container" do
987     set_user_from_auth :active
988     c, _ = minimal_new
989     set_user_from_auth :dispatch1
990     c.lock
991     c.update_attributes! state: Container::Running
992
993     set_user_from_auth :running_to_be_deleted_container_auth
994     assert_raises(ArvadosModel::PermissionDeniedError) do
995       c.update_attributes(output: collections(:foo_file).portable_data_hash)
996     end
997   end
998
999   test "can set trashed output on running container" do
1000     set_user_from_auth :active
1001     c, _ = minimal_new
1002     set_user_from_auth :dispatch1
1003     c.lock
1004     c.update_attributes! state: Container::Running
1005
1006     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
1007
1008     assert output.is_trashed
1009     assert c.update_attributes output: output.portable_data_hash
1010     assert c.update_attributes! state: Container::Complete
1011   end
1012
1013   test "not allowed to set trashed output that is not readable by current user" do
1014     set_user_from_auth :active
1015     c, _ = minimal_new
1016     set_user_from_auth :dispatch1
1017     c.lock
1018     c.update_attributes! state: Container::Running
1019
1020     output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
1021
1022     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
1023     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
1024
1025     assert_raises ActiveRecord::RecordInvalid do
1026       c.update_attributes! output: output.portable_data_hash
1027     end
1028   end
1029
1030   test "user cannot delete" do
1031     set_user_from_auth :active
1032     c, _ = minimal_new
1033     assert_raises ArvadosModel::PermissionDeniedError do
1034       c.destroy
1035     end
1036     assert Container.find_by_uuid(c.uuid)
1037   end
1038
1039   [
1040     {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
1041     {state: Container::Cancelled},
1042   ].each do |final_attrs|
1043     test "secret_mounts and runtime_token are null after container is #{final_attrs[:state]}" do
1044       set_user_from_auth :active
1045       c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
1046                           container_count_max: 1, runtime_token: api_client_authorizations(:active).token)
1047       set_user_from_auth :dispatch1
1048       c.lock
1049       c.update_attributes!(state: Container::Running)
1050       c.reload
1051       assert c.secret_mounts.has_key?('/secret')
1052       assert_equal api_client_authorizations(:active).token, c.runtime_token
1053
1054       c.update_attributes!(final_attrs)
1055       c.reload
1056       assert_equal({}, c.secret_mounts)
1057       assert_nil c.runtime_token
1058       cr.reload
1059       assert_equal({}, cr.secret_mounts)
1060       assert_nil cr.runtime_token
1061       assert_no_secrets_logged
1062     end
1063   end
1064 end