1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: AGPL-3.0
6 require 'helpers/container_test_helper'
8 class ContainerTest < ActiveSupport::TestCase
10 include ContainerTestHelper
13 command: ['echo', 'foo'],
14 container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
17 runtime_constraints: {"vcpus" => 1, "ram" => 1},
20 REUSABLE_COMMON_ATTRS = {
21 container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
23 command: ["echo", "hello"],
25 runtime_constraints: {
30 "test" => {"kind" => "json"},
38 def minimal_new attrs={}
39 cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
40 cr.state = ContainerRequest::Committed
41 act_as_user users(:active) do
44 c = Container.find_by_uuid cr.container_uuid
49 def check_illegal_updates c, bad_updates
50 bad_updates.each do |u|
51 refute c.update_attributes(u), u.inspect
52 refute c.valid?, u.inspect
57 def check_illegal_modify c
58 check_illegal_updates c, [{command: ["echo", "bar"]},
59 {container_image: "arvados/apitestfixture:june10"},
61 {environment: {"FOO" => "BAR"}},
62 {mounts: {"FOO" => "BAR"}},
63 {output_path: "/tmp3"},
64 {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
65 {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
66 {runtime_constraints: {"FOO" => "BAR"}}]
69 def check_bogus_states c
70 check_illegal_updates c, [{state: nil},
74 def check_no_change_from_cancelled c
75 check_illegal_modify c
77 check_illegal_updates c, [{ priority: 3 },
78 { state: Container::Queued },
79 { state: Container::Locked },
80 { state: Container::Running },
81 { state: Container::Complete }]
84 test "Container create" do
86 c, _ = minimal_new(environment: {},
87 mounts: {"BAR" => {"kind" => "FOO"}},
90 runtime_constraints: {"vcpus" => 1, "ram" => 1})
92 check_illegal_modify c
101 test "Container valid priority" do
102 act_as_system_user do
103 c, _ = minimal_new(environment: {},
104 mounts: {"BAR" => {"kind" => "FOO"}},
107 runtime_constraints: {"vcpus" => 1, "ram" => 1})
109 assert_raises(ActiveRecord::RecordInvalid) do
129 c.priority = 1000 << 50
135 test "Container serialized hash attributes sorted before save" do
136 env = {"C" => "3", "B" => "2", "A" => "1"}
137 m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
138 rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
139 c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
140 assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
141 assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
142 assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
145 test 'deep_sort_hash on array of hashes' do
146 a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
147 b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
148 assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
151 test "find_reusable method should select higher priority queued container" do
152 set_user_from_auth :active
153 common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
154 c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
155 c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
156 assert_not_equal c_low_priority.uuid, c_high_priority.uuid
157 assert_equal Container::Queued, c_low_priority.state
158 assert_equal Container::Queued, c_high_priority.state
159 reused = Container.find_reusable(common_attrs)
160 assert_not_nil reused
161 assert_equal reused.uuid, c_high_priority.uuid
164 test "find_reusable method should select latest completed container" do
165 set_user_from_auth :active
166 common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
168 state: Container::Complete,
170 log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
171 output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
174 c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
175 c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
176 assert_not_equal c_older.uuid, c_recent.uuid
178 set_user_from_auth :dispatch1
179 c_older.update_attributes!({state: Container::Locked})
180 c_older.update_attributes!({state: Container::Running})
181 c_older.update_attributes!(completed_attrs)
183 c_recent.update_attributes!({state: Container::Locked})
184 c_recent.update_attributes!({state: Container::Running})
185 c_recent.update_attributes!(completed_attrs)
187 reused = Container.find_reusable(common_attrs)
188 assert_not_nil reused
189 assert_equal reused.uuid, c_older.uuid
192 test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
193 set_user_from_auth :active
194 common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
196 state: Container::Complete,
198 log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
201 cr = ContainerRequest.new common_attrs
202 cr.use_existing = false
203 cr.state = ContainerRequest::Committed
205 c_output1 = Container.where(uuid: cr.container_uuid).first
207 cr = ContainerRequest.new common_attrs
208 cr.use_existing = false
209 cr.state = ContainerRequest::Committed
211 c_output2 = Container.where(uuid: cr.container_uuid).first
213 assert_not_equal c_output1.uuid, c_output2.uuid
215 set_user_from_auth :dispatch1
217 out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
218 log1 = collections(:real_log_collection).portable_data_hash
219 c_output1.update_attributes!({state: Container::Locked})
220 c_output1.update_attributes!({state: Container::Running})
221 c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
223 out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
224 c_output2.update_attributes!({state: Container::Locked})
225 c_output2.update_attributes!({state: Container::Running})
226 c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
228 reused = Container.resolve(ContainerRequest.new(common_attrs))
229 assert_equal c_output1.uuid, reused.uuid
232 test "find_reusable method should select running container by start date" do
233 set_user_from_auth :active
234 common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
235 c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
236 c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
237 c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
238 # Confirm the 3 container UUIDs are different.
239 assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
240 set_user_from_auth :dispatch1
241 c_slower.update_attributes!({state: Container::Locked})
242 c_slower.update_attributes!({state: Container::Running,
244 c_faster_started_first.update_attributes!({state: Container::Locked})
245 c_faster_started_first.update_attributes!({state: Container::Running,
247 c_faster_started_second.update_attributes!({state: Container::Locked})
248 c_faster_started_second.update_attributes!({state: Container::Running,
250 reused = Container.find_reusable(common_attrs)
251 assert_not_nil reused
252 # Selected container is the one that started first
253 assert_equal reused.uuid, c_faster_started_first.uuid
256 test "find_reusable method should select running container by progress" do
257 set_user_from_auth :active
258 common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
259 c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
260 c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
261 c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
262 # Confirm the 3 container UUIDs are different.
263 assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
264 set_user_from_auth :dispatch1
265 c_slower.update_attributes!({state: Container::Locked})
266 c_slower.update_attributes!({state: Container::Running,
268 c_faster_started_first.update_attributes!({state: Container::Locked})
269 c_faster_started_first.update_attributes!({state: Container::Running,
271 c_faster_started_second.update_attributes!({state: Container::Locked})
272 c_faster_started_second.update_attributes!({state: Container::Running,
274 reused = Container.find_reusable(common_attrs)
275 assert_not_nil reused
276 # Selected container is the one with most progress done
277 assert_equal reused.uuid, c_faster_started_second.uuid
280 test "find_reusable method should select locked container most likely to start sooner" do
281 set_user_from_auth :active
282 common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
283 c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
284 c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
285 c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
286 # Confirm the 3 container UUIDs are different.
287 assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
288 set_user_from_auth :dispatch1
289 c_low_priority.update_attributes!({state: Container::Locked,
291 c_high_priority_older.update_attributes!({state: Container::Locked,
293 c_high_priority_newer.update_attributes!({state: Container::Locked,
295 reused = Container.find_reusable(common_attrs)
296 assert_not_nil reused
297 assert_equal reused.uuid, c_high_priority_older.uuid
300 test "find_reusable method should select running over failed container" do
301 set_user_from_auth :active
302 common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
303 c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
304 c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
305 assert_not_equal c_failed.uuid, c_running.uuid
306 set_user_from_auth :dispatch1
307 c_failed.update_attributes!({state: Container::Locked})
308 c_failed.update_attributes!({state: Container::Running})
309 c_failed.update_attributes!({state: Container::Complete,
311 log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
312 output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
313 c_running.update_attributes!({state: Container::Locked})
314 c_running.update_attributes!({state: Container::Running,
316 reused = Container.find_reusable(common_attrs)
317 assert_not_nil reused
318 assert_equal reused.uuid, c_running.uuid
321 test "find_reusable method should select complete over running container" do
322 set_user_from_auth :active
323 common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
324 c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
325 c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
326 assert_not_equal c_completed.uuid, c_running.uuid
327 set_user_from_auth :dispatch1
328 c_completed.update_attributes!({state: Container::Locked})
329 c_completed.update_attributes!({state: Container::Running})
330 c_completed.update_attributes!({state: Container::Complete,
332 log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
333 output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
334 c_running.update_attributes!({state: Container::Locked})
335 c_running.update_attributes!({state: Container::Running,
337 reused = Container.find_reusable(common_attrs)
338 assert_not_nil reused
339 assert_equal c_completed.uuid, reused.uuid
342 test "find_reusable method should select running over locked container" do
343 set_user_from_auth :active
344 common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
345 c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
346 c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
347 assert_not_equal c_running.uuid, c_locked.uuid
348 set_user_from_auth :dispatch1
349 c_locked.update_attributes!({state: Container::Locked})
350 c_running.update_attributes!({state: Container::Locked})
351 c_running.update_attributes!({state: Container::Running,
353 reused = Container.find_reusable(common_attrs)
354 assert_not_nil reused
355 assert_equal reused.uuid, c_running.uuid
358 test "find_reusable method should select locked over queued container" do
359 set_user_from_auth :active
360 common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
361 c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
362 c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
363 assert_not_equal c_queued.uuid, c_locked.uuid
364 set_user_from_auth :dispatch1
365 c_locked.update_attributes!({state: Container::Locked})
366 reused = Container.find_reusable(common_attrs)
367 assert_not_nil reused
368 assert_equal reused.uuid, c_locked.uuid
371 test "find_reusable method should not select failed container" do
372 set_user_from_auth :active
373 attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
374 c, _ = minimal_new(attrs)
375 set_user_from_auth :dispatch1
376 c.update_attributes!({state: Container::Locked})
377 c.update_attributes!({state: Container::Running})
378 c.update_attributes!({state: Container::Complete,
380 reused = Container.find_reusable(attrs)
384 test "find_reusable with logging disabled" do
385 set_user_from_auth :active
386 Rails.logger.expects(:info).never
387 Container.find_reusable(REUSABLE_COMMON_ATTRS)
390 test "find_reusable with logging enabled" do
391 set_user_from_auth :active
392 Rails.configuration.log_reuse_decisions = true
393 Rails.logger.expects(:info).at_least(3)
394 Container.find_reusable(REUSABLE_COMMON_ATTRS)
397 test "Container running" do
398 c, _ = minimal_new priority: 1
400 set_user_from_auth :dispatch1
401 check_illegal_updates c, [{state: Container::Running},
402 {state: Container::Complete}]
405 c.update_attributes! state: Container::Running
407 check_illegal_modify c
410 check_illegal_updates c, [{state: Container::Queued}]
413 c.update_attributes! priority: 3
416 test "Lock and unlock" do
417 c, cr = minimal_new priority: 0
419 set_user_from_auth :dispatch1
420 assert_equal Container::Queued, c.state
422 assert_raise(ArvadosModel::LockFailedError) do
427 assert cr.update_attributes priority: 1
429 refute c.update_attributes(state: Container::Running), "not locked"
431 refute c.update_attributes(state: Container::Complete), "not locked"
434 assert c.lock, show_errors(c)
435 assert c.locked_by_uuid
438 assert_raise(ArvadosModel::LockFailedError) {c.lock}
441 assert c.unlock, show_errors(c)
442 refute c.locked_by_uuid
445 refute c.update_attributes(state: Container::Running), "not locked"
447 refute c.locked_by_uuid
450 assert c.lock, show_errors(c)
451 assert c.update_attributes(state: Container::Running), show_errors(c)
452 assert c.locked_by_uuid
455 auth_uuid_was = c.auth_uuid
457 assert_raise(ArvadosModel::LockFailedError) do
458 # Running to Locked is not allowed
462 assert_raise(ArvadosModel::InvalidStateTransitionError) do
463 # Running to Queued is not allowed
468 assert c.update_attributes(state: Container::Complete), show_errors(c)
469 refute c.locked_by_uuid
472 auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
473 assert_operator auth_exp, :<, db_current_time
476 test "Container queued cancel" do
477 c, cr = minimal_new({container_count_max: 1})
478 set_user_from_auth :dispatch1
479 assert c.update_attributes(state: Container::Cancelled), show_errors(c)
480 check_no_change_from_cancelled c
482 assert_equal ContainerRequest::Final, cr.state
485 test "Container queued count" do
486 assert_equal 1, Container.readable_by(users(:active)).where(state: "Queued").count
489 test "Container locked cancel" do
491 set_user_from_auth :dispatch1
492 assert c.lock, show_errors(c)
493 assert c.update_attributes(state: Container::Cancelled), show_errors(c)
494 check_no_change_from_cancelled c
497 test "Container locked cancel with log" do
499 set_user_from_auth :dispatch1
500 assert c.lock, show_errors(c)
501 assert c.update_attributes(
502 state: Container::Cancelled,
503 log: collections(:real_log_collection).portable_data_hash,
505 check_no_change_from_cancelled c
508 test "Container running cancel" do
510 set_user_from_auth :dispatch1
512 c.update_attributes! state: Container::Running
513 c.update_attributes! state: Container::Cancelled
514 check_no_change_from_cancelled c
517 test "Container create forbidden for non-admin" do
518 set_user_from_auth :active_trustedclient
519 c = Container.new DEFAULT_ATTRS
521 c.mounts = {"BAR" => "FOO"}
522 c.output_path = "/tmp"
524 c.runtime_constraints = {}
525 assert_raises(ArvadosModel::PermissionDeniedError) do
530 test "Container only set exit code on complete" do
532 set_user_from_auth :dispatch1
534 c.update_attributes! state: Container::Running
536 check_illegal_updates c, [{exit_code: 1},
537 {exit_code: 1, state: Container::Cancelled}]
539 assert c.update_attributes(exit_code: 1, state: Container::Complete)
542 test "locked_by_uuid can set output on running container" do
544 set_user_from_auth :dispatch1
546 c.update_attributes! state: Container::Running
548 assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
550 assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
551 assert c.update_attributes! state: Container::Complete
554 test "auth_uuid can set output on running container, but not change container state" do
556 set_user_from_auth :dispatch1
558 c.update_attributes! state: Container::Running
560 Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
561 Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
562 assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
564 assert_raises ArvadosModel::PermissionDeniedError do
565 # auth_uuid cannot set container state
566 c.update_attributes state: Container::Complete
570 test "not allowed to set output that is not readable by current user" do
572 set_user_from_auth :dispatch1
574 c.update_attributes! state: Container::Running
576 Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
577 Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
579 assert_raises ActiveRecord::RecordInvalid do
580 c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
584 test "other token cannot set output on running container" do
586 set_user_from_auth :dispatch1
588 c.update_attributes! state: Container::Running
590 set_user_from_auth :running_to_be_deleted_container_auth
591 assert_raises ArvadosModel::PermissionDeniedError do
592 c.update_attributes! output: collections(:foo_file).portable_data_hash
596 test "can set trashed output on running container" do
598 set_user_from_auth :dispatch1
600 c.update_attributes! state: Container::Running
602 output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
604 assert output.is_trashed
605 assert c.update_attributes output: output.portable_data_hash
606 assert c.update_attributes! state: Container::Complete
609 test "not allowed to set trashed output that is not readable by current user" do
611 set_user_from_auth :dispatch1
613 c.update_attributes! state: Container::Running
615 output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
617 Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
618 Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
620 assert_raises ActiveRecord::RecordInvalid do
621 c.update_attributes! output: output.portable_data_hash
626 {state: Container::Complete, exit_code: 0, output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'},
627 {state: Container::Cancelled},
628 ].each do |final_attrs|
629 test "secret_mounts is null after container is #{final_attrs[:state]}" do
630 c, cr = minimal_new(secret_mounts: {'/secret' => {'kind' => 'text', 'content' => 'foo'}},
631 container_count_max: 1)
632 set_user_from_auth :dispatch1
634 c.update_attributes!(state: Container::Running)
636 assert c.secret_mounts.has_key?('/secret')
638 c.update_attributes!(final_attrs)
640 assert_equal({}, c.secret_mounts)
642 assert_equal({}, cr.secret_mounts)
643 assert_no_secrets_logged